{% for source in sources %}
- {{ source.icon|safe }} {{ source.name }} ({{ source.get_source_type_display }})
+ {{ source.icon|safe }} {{ source.name }} ({{ source.get_source_type_display }} "{{ source.key }}")
{{ source.format_summary }}
+ {% if source.has_failed %}
+ Source has permanent failures
+ {% else %}
{{ source.media_count }} media items{% if source.delete_old_media and source.days_to_keep > 0 %}, keep {{ source.days_to_keep }} days of media{% endif %}
+ {% endif %}
{% empty %}
You haven't added any sources.
diff --git a/app/sync/templates/sync/tasks-completed.html b/app/sync/templates/sync/tasks-completed.html
new file mode 100644
index 0000000..3a6977b
--- /dev/null
+++ b/app/sync/templates/sync/tasks-completed.html
@@ -0,0 +1,33 @@
+{% extends 'base.html' %}
+
+{% block headtitle %}Tasks - Completed{% endblock %}
+
+{% block content %}
+
+{% include 'infobox.html' with message=message %}
+
+
+
+ {% for task in tasks %}
+
+ {% if task.has_error %}
+ {{ task.verbose_name }}
+ Error: "{{ task.error_message }}"
+ Task started at {{ task.run_at|date:'Y-m-d H:i:s' }}
+ {% else %}
+ {{ task.verbose_name }}
+ Task started at {{ task.run_at|date:'Y-m-d H:i:s' }}
+ {% endif %}
+
+ {% empty %}
+ There have been no completed tasks.
+ {% endfor %}
+
+
+
+{% include 'pagination.html' with pagination=sources.paginator filter=source.pk %}
+{% endblock %}
diff --git a/app/sync/templates/sync/tasks.html b/app/sync/templates/sync/tasks.html
index ecb202a..5c9a7e9 100644
--- a/app/sync/templates/sync/tasks.html
+++ b/app/sync/templates/sync/tasks.html
@@ -5,7 +5,83 @@
{% block content %}
- tasks
+
Tasks
+
+ Tasks are the background work that TubeSync undertakes to index and download
+ media. This page allows you to see basic overview of what is running and what is
+ scheduled to perform in the future as well as check up on any errors that might
+ have occured.
+
+
+
+
+
+
{{ running|length }} Running
+
+ Running tasks are tasks which currently being worked on right now.
+
+
+
+
+
+
+
{{ errors|length }} Error{{ errors|length|pluralize }}
+
+ Tasks which generated an error are shown here. Tasks are retried a couple of
+ times, so if there was an intermittent error such as a download got interrupted
+ it will be scheduled to run again.
+
+
+
+
+
+
+
{{ scheduled|length }} Scheduled
+
+ Tasks which are scheduled to run in the future or are waiting in a queue to be
+ processed. They can be waiting for an available worker to run immediately, or
+ run in the future at the specified "run at" time.
+
+
+
+
+
+
+
Completed
+
+ A record of recently completed tasks is kept for a few days. You can use the button
+ below to view recent tasks which have completed successfully.
+
+
View Completed tasks
{% endblock %}
diff --git a/app/sync/urls.py b/app/sync/urls.py
index 4769b55..def8e06 100644
--- a/app/sync/urls.py
+++ b/app/sync/urls.py
@@ -1,7 +1,7 @@
from django.urls import path
from .views import (DashboardView, SourcesView, ValidateSourceView, AddSourceView,
SourceView, UpdateSourceView, DeleteSourceView, MediaView,
- MediaThumbView, MediaItemView, TasksView)
+ MediaThumbView, MediaItemView, TasksView, CompletedTasksView)
app_name = 'sync'
@@ -61,4 +61,8 @@ urlpatterns = [
TasksView.as_view(),
name='tasks'),
+ path('tasks-completed',
+ CompletedTasksView.as_view(),
+ name='tasks-completed'),
+
]
diff --git a/app/sync/utils.py b/app/sync/utils.py
index a9e3ca4..4bc0e47 100644
--- a/app/sync/utils.py
+++ b/app/sync/utils.py
@@ -14,25 +14,31 @@ def validate_url(url, validator):
Validate a URL against a dict of validation requirements. Returns an extracted
part of the URL if the URL is valid, if invalid raises a ValidationError.
'''
- valid_scheme, valid_netloc, valid_path, valid_query, extract_parts = (
- validator['scheme'], validator['domain'], validator['path_regex'],
- validator['qs_args'], validator['extract_key'])
+ valid_scheme, valid_netloc, valid_path, invalid_paths, valid_query, \
+ extract_parts = (
+ validator['scheme'], validator['domain'], validator['path_regex'],
+ validator['path_must_not_match'], validator['qs_args'],
+ validator['extract_key']
+ )
url_parts = urlsplit(str(url).strip())
url_scheme = str(url_parts.scheme).strip().lower()
if url_scheme != valid_scheme:
- raise ValidationError(f'scheme "{url_scheme}" must be "{valid_scheme}"')
+ raise ValidationError(f'invalid scheme "{url_scheme}" must be "{valid_scheme}"')
url_netloc = str(url_parts.netloc).strip().lower()
if url_netloc != valid_netloc:
- raise ValidationError(f'domain "{url_netloc}" must be "{valid_netloc}"')
+ raise ValidationError(f'invalid domain "{url_netloc}" must be "{valid_netloc}"')
url_path = str(url_parts.path).strip()
matches = re.findall(valid_path, url_path)
if not matches:
- raise ValidationError(f'path "{url_path}" must match "{valid_path}"')
+ raise ValidationError(f'invalid path "{url_path}" must match "{valid_path}"')
+ for invalid_path in invalid_paths:
+ if url_path.lower() == invalid_path.lower():
+ raise ValidationError(f'path "{url_path}" is not valid')
url_query = str(url_parts.query).strip()
url_query_parts = parse_qs(url_query)
for required_query in valid_query:
if required_query not in url_query_parts:
- raise ValidationError(f'query string "{url_query}" must '
+ raise ValidationError(f'invalid query string "{url_query}" must '
f'contain the parameter "{required_query}"')
extract_from, extract_param = extract_parts
extract_value = ''
@@ -76,7 +82,7 @@ def resize_image_to_height(image, width, height):
if scaled_width > width:
# Width too large, crop it
delta = scaled_width - width
- left, upper = (delta / 2), 0
+ left, upper = round(delta / 2), 0
right, lower = (left + width), height
image = image.crop((left, upper, right, lower))
return image
diff --git a/app/sync/views.py b/app/sync/views.py
index 995c41d..e22fdd7 100644
--- a/app/sync/views.py
+++ b/app/sync/views.py
@@ -9,11 +9,14 @@ from django.urls import reverse_lazy
from django.db.models import Count
from django.forms import ValidationError
from django.utils.text import slugify
+from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from common.utils import append_uri_params
+from background_task.models import Task, CompletedTask
from .models import Source, Media
from .forms import ValidateSourceForm, ConfirmDeleteSourceForm
from .utils import validate_url
+from .tasks import map_task_to_instance, get_error_message, get_source_completed_tasks
from . import signals
from . import youtube
@@ -108,6 +111,7 @@ class ValidateSourceView(FormView):
'scheme': 'https',
'domain': 'www.youtube.com',
'path_regex': '^\/(c\/)?([^\/]+)$',
+ 'path_must_not_match': ('/playlist',),
'qs_args': [],
'extract_key': ('path_regex', 1),
'example': 'https://www.youtube.com/SOMECHANNEL'
@@ -116,7 +120,8 @@ class ValidateSourceView(FormView):
'scheme': 'https',
'domain': 'www.youtube.com',
'path_regex': '^\/(playlist|watch)$',
- 'qs_args': ['list'],
+ 'path_must_not_match': (),
+ 'qs_args': ('list',),
'extract_key': ('qs_args', 'list'),
'example': 'https://www.youtube.com/playlist?list=PLAYLISTID'
},
@@ -239,6 +244,16 @@ class SourceView(DetailView):
template_name = 'sync/source.html'
model = Source
+ def get_context_data(self, *args, **kwargs):
+ data = super().get_context_data(*args, **kwargs)
+ data['errors'] = []
+ for error in get_source_completed_tasks(self.object.pk, only_errors=True):
+ error_message = get_error_message(error)
+ setattr(error, 'error_message', error_message)
+ data['errors'].append(error)
+ data['media'] = Media.objects.filter(source=self.object).order_by('-published')
+ return data
+
class UpdateSourceView(UpdateView):
@@ -286,7 +301,7 @@ class MediaView(ListView):
context_object_name = 'media'
paginate_by = settings.MEDIA_PER_PAGE
messages = {
- 'filter': _('Viewing media for source:
{name}'),
+ 'filter': _('Viewing media filtered for source:
{name}'),
}
def __init__(self, *args, **kwargs):
@@ -352,13 +367,90 @@ class MediaItemView(DetailView):
model = Media
-class TasksView(TemplateView):
+class TasksView(ListView):
'''
A list of tasks queued to be completed. Typically, this is scraping for new
media or downloading media.
'''
template_name = 'sync/tasks.html'
+ context_object_name = 'tasks'
+
+ def get_queryset(self):
+ return Task.objects.all().order_by('run_at')
+
+ def get_context_data(self, *args, **kwargs):
+ data = super().get_context_data(*args, **kwargs)
+ data['running'] = []
+ data['errors'] = []
+ data['scheduled'] = []
+ queryset = self.get_queryset()
+ now = timezone.now()
+ for task in queryset:
+ obj, url = map_task_to_instance(task)
+ if not obj:
+ # Orphaned task, ignore it (it will be deleted when it fires)
+ continue
+ setattr(task, 'instance', obj)
+ setattr(task, 'url', url)
+ if task.locked_by_pid_running():
+ data['running'].append(task)
+ elif task.has_error():
+ error_message = get_error_message(task)
+ setattr(task, 'error_message', error_message)
+ data['errors'].append(task)
+ else:
+ data['scheduled'].append(task)
+ return data
+
+
+class CompletedTasksView(ListView):
+ '''
+ List of tasks which have been completed with an optional per-source filter.
+ '''
+
+ template_name = 'sync/tasks-completed.html'
+ context_object_name = 'tasks'
+ paginate_by = settings.TASKS_PER_PAGE
+ messages = {
+ 'filter': _('Viewing tasks filtered for source:
{name}'),
+ }
+
+ def __init__(self, *args, **kwargs):
+ self.filter_source = None
+ super().__init__(*args, **kwargs)
def dispatch(self, request, *args, **kwargs):
+ filter_by = request.GET.get('filter', '')
+ if filter_by:
+ try:
+ self.filter_source = Source.objects.get(pk=filter_by)
+ except Source.DoesNotExist:
+ self.filter_source = None
return super().dispatch(request, *args, **kwargs)
+
+ def get_queryset(self):
+ return CompletedTask.objects.all().order_by('-run_at')
+
+ def get_queryset(self):
+ if self.filter_source:
+ return CompletedTask.objects.all().order_by('-run_at')
+ #tasks = []
+ #for task in CompletedTask.objects.all().order_by('-run_at'):
+ # # ???
+ #q = Media.objects.filter(source=self.filter_source)
+ return CompletedTask.objects.all().order_by('-run_at')
+
+ def get_context_data(self, *args, **kwargs):
+ data = super().get_context_data(*args, **kwargs)
+ for task in data['tasks']:
+ if task.has_error():
+ error_message = get_error_message(task)
+ setattr(task, 'error_message', error_message)
+ data['message'] = ''
+ data['source'] = None
+ if self.filter_source:
+ message = str(self.messages.get('filter', ''))
+ data['message'] = message.format(name=self.filter_source.name)
+ data['source'] = self.filter_source
+ return data
diff --git a/app/tubesync/local_settings.py.container b/app/tubesync/local_settings.py.container
index a281c81..3b71cbc 100644
--- a/app/tubesync/local_settings.py.container
+++ b/app/tubesync/local_settings.py.container
@@ -1,13 +1,15 @@
import os
from pathlib import Path
+from
BASE_DIR = Path(__file__).resolve().parent.parent
ROOT_DIR = Path('/')
-SECRET_KEY = str(os.getenv('DJANGO_SECRET_KEY', ''))
-ALLOWED_HOSTS_STR = str(os.getenv('DJANGO_ALLOWED_HOSTS', ''))
+RANDOM_SECRET = hexlify(os.urandom(32)).decode()
+SECRET_KEY = str(os.getenv('DJANGO_SECRET_KEY', RANDOM_SECRET))
+ALLOWED_HOSTS_STR = str(os.getenv('TUBESYNC_HOSTS', 'localhost'))
ALLOWED_HOSTS = ALLOWED_HOSTS_STR.split(',')
@@ -19,6 +21,9 @@ DATABASES = {
}
+BACKGROUND_TASK_ASYNC_THREADS = int(os.get('TUBESYNC_WORKERS', 2))
+
+
MEDIA_ROOT = ROOT_DIR / 'config' / 'media'
SYNC_VIDEO_ROOT = ROOT_DIR / 'downloads' / 'video'
SYNC_AUDIO_ROOT = ROOT_DIR / 'downloads' / 'audio'
diff --git a/app/tubesync/settings.py b/app/tubesync/settings.py
index bc7eb99..57c660a 100644
--- a/app/tubesync/settings.py
+++ b/app/tubesync/settings.py
@@ -118,10 +118,12 @@ MAX_RUN_TIME = 1800 # Maximum amount of time in seconds
BACKGROUND_TASK_RUN_ASYNC = True # Run tasks async in the background
BACKGROUND_TASK_ASYNC_THREADS = 2 # Number of async tasks to run at once
BACKGROUND_TASK_PRIORITY_ORDERING = 'DESC' # Process high priority tasks first
+COMPLETED_TASKS_DAYS_TO_KEEP = 30 # Number of days to keep completed tasks
SOURCES_PER_PAGE = 36
MEDIA_PER_PAGE = 36
+TASKS_PER_PAGE = 100
MEDIA_THUMBNAIL_WIDTH = 430 # Width in pixels to resize thumbnails to