better media task management options

tubesync/sync/forms.py

@@ -22,3 +22,8 @@ class ConfirmDeleteSourceForm(forms.Form):
         label=_('Also delete downloaded media'),
         required=False
     )
+
+
+class RedownloadMediaForm(forms.Form):
+
+    pass

tubesync/sync/models.py

@@ -396,6 +396,20 @@ class Media(models.Model):
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats',
         }
     }
+    STATE_UNKNOWN = 'unknown'
+    STATE_SCHEDULED = 'scheduled'
+    STATE_DOWNLOADING = 'downloading'
+    STATE_DOWNLOADED = 'downloaded'
+    STATE_ERROR = 'error'
+    STATES = (STATE_UNKNOWN, STATE_SCHEDULED, STATE_DOWNLOADING, STATE_DOWNLOADED,
+              STATE_ERROR)
+    STATE_ICONS = {
+        STATE_UNKNOWN: '<i class="far fa-question-circle" title="Unknown download state"></i>',
+        STATE_SCHEDULED: '<i class="far fa-clock" title="Scheduled to download"></i>',
+        STATE_DOWNLOADING: '<i class="fas fa-download" title="Downloading now"></i>',
+        STATE_DOWNLOADED: '<i class="far fa-check-circle" title="Downloaded"></i>',
+        STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>',
+    }
 
     uuid = models.UUIDField(
         _('uuid'),
@@ -649,14 +663,17 @@ class Media(models.Model):
         name = slugify(self.name.replace('&', 'and').replace('+', 'and'))[:50]
         key = self.key.strip()
         fmt = self.source.source_resolution.lower()
-        codecs = []
+        if self.source.is_audio():
-        vcodec = self.source.source_vcodec.lower()
+            codecs = self.source.source_acodec.lower()
-        acodec = self.source.source_acodec.lower()
+        else:
-        if vcodec:
+            codecs = []
-            codecs.append(vcodec)
+            vcodec = self.source.source_vcodec.lower()
-        if acodec:
+            acodec = self.source.source_acodec.lower()
-            codecs.append(acodec)
+            if vcodec:
-        codecs = '-'.join(codecs)
+                codecs.append(vcodec)
+            if acodec:
+                codecs.append(acodec)
+            codecs = '-'.join(codecs)
         ext = self.source.extension
         return f'{datestr}_{source_name}_{name}_{key}-{fmt}-{codecs}.{ext}'
 
@@ -664,6 +681,34 @@ class Media(models.Model):
     def filepath(self):
         return self.source.directory_path / self.filename
 
+    @property
+    def thumb_file_exists(self):
+        if not self.thumb:
+            return False
+        return os.path.exists(self.thumb.path)
+
+    @property
+    def media_file_exists(self):
+        if not self.media_file:
+            return False
+        return os.path.exists(self.media_file.path)
+
+    def get_download_state(self, task=None):
+        if self.downloaded:
+            return self.STATE_DOWNLOADED
+        if task:
+            if task.locked_by_pid_running():
+                return self.STATE_DOWNLOADING
+            elif task.has_error():
+                return self.STATE_ERROR
+            else:
+                return self.STATE_SCHEDULED
+        return self.STATE_UNKNOWN
+
+    def get_download_state_icon(self, task=None):
+        state = self.get_download_state(task)
+        return self.STATE_ICONS.get(state, self.STATE_ICONS[self.STATE_UNKNOWN])
+
     def download_media(self):
         format_str = self.get_format_str()
         if not format_str:

tubesync/sync/signals.py

@@ -29,6 +29,7 @@ def source_pre_save(sender, instance, **kwargs):
             str(instance.pk),
             repeat=instance.index_schedule,
             queue=str(instance.pk),
+            priority=5,
             verbose_name=verbose_name.format(instance.name)
         )
 
@@ -46,6 +47,7 @@ def source_post_save(sender, instance, created, **kwargs):
             str(instance.pk),
             repeat=instance.index_schedule,
             queue=str(instance.pk),
+            priority=5,
             verbose_name=verbose_name.format(instance.name)
         )
     # Trigger the post_save signal for each media item linked to this source as various
@@ -82,9 +84,18 @@ def task_task_failed(sender, task_id, completed_task, **kwargs):
 
 @receiver(post_save, sender=Media)
 def media_post_save(sender, instance, created, **kwargs):
-    # Triggered after media is saved
+    # Triggered after media is saved, Recalculate the "can_download" flag, this may
-    if created:
+    # need to change if the source specifications have been changed
-        # If the media is newly created start a task to download its thumbnail
+    if instance.get_format_str():
+        if not instance.can_download:
+            instance.can_download = True
+            instance.save()
+    else:
+        if instance.can_download:
+            instance.can_download = True
+            instance.save()
+    # If the media is missing a thumbnail schedule it to be downloaded
+    if not instance.thumb:
         thumbnail_url = instance.thumbnail
         if thumbnail_url:
             log.info(f'Scheduling task to download thumbnail for: {instance.name} '
@@ -94,18 +105,9 @@ def media_post_save(sender, instance, created, **kwargs):
                 str(instance.pk),
                 thumbnail_url,
                 queue=str(instance.source.pk),
+                priority=10,
                 verbose_name=verbose_name.format(instance.name)
             )
-    # Recalculate the "can_download" flag, this may need to change if the source
-    # specifications have been changed
-    if instance.get_format_str():
-        if not instance.can_download:
-            instance.can_download = True
-            instance.save()
-    else:
-        if instance.can_download:
-            instance.can_download = True
-            instance.save()
     # If the media has not yet been downloaded schedule it to be downloaded
     if not instance.downloaded:
         delete_task_by_media('sync.tasks.download_media', (str(instance.pk),))
@@ -113,6 +115,7 @@ def media_post_save(sender, instance, created, **kwargs):
         download_media(
             str(instance.pk),
             queue=str(instance.source.pk),
+            priority=15,
             verbose_name=verbose_name.format(instance.name)
         )
 

tubesync/sync/tasks.py

@@ -103,6 +103,14 @@ def get_source_completed_tasks(source_id, only_errors=False):
     return CompletedTask.objects.filter(**q).order_by('-failed_at')
 
 
+def get_media_download_task(media_id):
+    try:
+        return Task.objects.get_task('sync.tasks.download_media',
+                                     args=(str(media_id),))[0]
+    except IndexError:
+        return False
+
+
 def delete_task_by_source(task_name, source_id):
     return Task.objects.filter(task_name=task_name, queue=str(source_id)).delete()
 

tubesync/sync/templates/sync/media-item.html

@@ -5,13 +5,14 @@
 {% block content %}
 <div class="row">
   <div class="col s12">
-    <h1 class="truncate">Media <strong>{{ media.key }}</strong></h1>
+    <h1 class="truncate">Media <strong>{{ media.key }}</strong> {{ download_state_icon|safe }}</h1>
     {% if media.title %}<h2 class="truncate"><strong>{{ media.title }}</strong></h2>{% endif %}
     <p class="truncate"><strong><a href="{{ media.url }}" target="_blank"><i class="fas fa-link"></i> {{ media.url }}</a></strong></p>
     <p class="truncate">Downloading to: <strong>{{ media.source.directory_path }}</strong></p>
   </div>
 </div>
 {% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
+{% include 'infobox.html' with message=message %}
 <div class="row">
   <div class="col s12 m7">
     <div><i class="fas fa-quote-left"></i></div>
@@ -26,6 +27,19 @@
     </div>
   </div>
 </div>
+{% if task %}
+<div class="row">
+  <div class="col s12">
+    <div class="collection">
+      <span class="collection-item">
+        <i class="fas fa-stopwatch"></i> <strong>{{ task }}</strong><br>
+        {% if task.instance.index_schedule %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
+        <i class="fas fa-redo"></i> Task will run {% if task.run_now %}<strong>immediately</strong>{% else %}at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>{% endif %}
+      </span>
+    </div>
+  </div>
+</div>
+{% endif %}
 <div class="row">
   <div class="col s12">
     <table class="striped">
@@ -106,4 +120,11 @@
     </table>
   </div>
 </div>
+{% if media.downloaded %}
+<div class="row">
+  <div class="col s12">
+    <a href="{% url 'sync:redownload-media' pk=media.pk %}" class="btn">Delete and redownload media <i class="fas fa-cloud-download-alt"></i></a>
+  </div>
+</div>
+{% endif %}
 {% endblock %}

tubesync/sync/templates/sync/media-redownload.html

@@ -0,0 +1,29 @@
+{% extends 'base.html' %}
+
+{% block headtitle %}Redownload media - {{ media }}{% endblock %}
+
+{% block content %}
+<div class="row no-margin-bottom">
+  <div class="col s12">
+    <h1>Redownload media <strong>{{ media }}</strong></h1>
+    <p>
+      You can delete the downloaded file for your media <strong>{{ media }}</strong> and
+      schedule it to be redownloaded. You might want to use this if you moved the original
+      file on disk and want to download it again, or, if you changed your source settings
+      such as changed the desired resolution and want to redownload the media in a different
+      format.
+    </p>
+  </div>
+</div>
+<div class="row">
+  <form method="post" action="{% url 'sync:redownload-media' pk=media.pk %}" class="col s12 simpleform">
+    {% csrf_token %}
+    {% include 'simpleform.html' with form=form %}
+    <div class="row no-margin-bottom padding-top">
+      <div class="col s12">
+        <button class="btn" type="submit" name="action">Really delete and redownload media <i class="fas fa-trash-alt"></i></button>
+      </div>
+    </div>
+  </form>
+</div>
+{% endblock %}

tubesync/sync/templates/sync/media.html

@@ -18,7 +18,7 @@
           <img src="{% if m.thumb %}{% url 'sync:media-thumb' pk=m.pk %}{% else %}{% static 'images/nothumb.png' %}{% endif %}">
           <span class="card-title truncate">{{ m.source }}<br>
             <span>{{ m.name }}</span><br>
-            <span>{% if m.can_download %}{% if m.downloaded %}<i class="fas fa-check-circle" title="Downloaded"></i>{% else %}<i class="far fa-clock" title="Queued waiting to download"></i>{% endif %} {{ m.published|date:'Y-m-d' }}{% else %}<i class="fas fa-exclamation-triangle"></i> No matching formats{% endif %}</span>
+            <span>{% if m.can_download %}{% if m.downloaded %}<i class="fas fa-check-circle" title="Downloaded"></i>{% else %}<i class="far fa-clock" title="Waiting to download or downloading"></i>{% endif %} {{ m.published|date:'Y-m-d' }}{% else %}<i class="fas fa-exclamation-triangle"></i> No matching formats{% endif %}</span>
           </span>
         </div>
       </a>

tubesync/sync/templates/sync/tasks.html

@@ -66,7 +66,7 @@
         <a href="{% url task.url pk=task.instance.pk %}" class="collection-item">
           <i class="fas fa-stopwatch"></i> <strong>{{ task }}</strong><br>
           {% if task.instance.index_schedule %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
-          <i class="fas fa-redo"></i> Task will next run at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>
+          <i class="fas fa-redo"></i> Task will run {% if task.run_now %}<strong>immediately</strong>{% else %}at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>{% endif %}
         </a>
       {% empty %}
         <span class="collection-item no-items"><i class="fas fa-info-circle"></i> There are no scheduled tasks.</span>

tubesync/sync/urls.py

@@ -1,7 +1,8 @@
 from django.urls import path
 from .views import (DashboardView, SourcesView, ValidateSourceView, AddSourceView,
                     SourceView, UpdateSourceView, DeleteSourceView, MediaView,
-                    MediaThumbView, MediaItemView, TasksView, CompletedTasksView)
+                    MediaThumbView, MediaItemView, MediaRedownloadView, TasksView,
+                    CompletedTasksView)
 
 
 app_name = 'sync'
@@ -55,6 +56,10 @@ urlpatterns = [
          MediaItemView.as_view(),
          name='media-item'),
 
+    path('media-redownload/<uuid:pk>',
+         MediaRedownloadView.as_view(),
+         name='redownload-media'),
+
     # Task URLs
 
     path('tasks',

tubesync/sync/views.py

@@ -4,6 +4,7 @@ from django.http import Http404
 from django.views.generic import TemplateView, ListView, DetailView
 from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateView,
                                        DeleteView)
+from django.views.generic.detail import SingleObjectMixin
 from django.http import HttpResponse
 from django.urls import reverse_lazy
 from django.db.models import Count
@@ -14,9 +15,11 @@ from django.utils.translation import gettext_lazy as _
 from common.utils import append_uri_params
 from background_task.models import Task, CompletedTask
 from .models import Source, Media
-from .forms import ValidateSourceForm, ConfirmDeleteSourceForm
+from .forms import ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm
 from .utils import validate_url, delete_file
-from .tasks import map_task_to_instance, get_error_message, get_source_completed_tasks
+from .tasks import (map_task_to_instance, get_error_message,
+                    get_source_completed_tasks, get_media_download_task,
+                    delete_task_by_media)
 from . import signals
 from . import youtube
 
@@ -295,6 +298,7 @@ class DeleteSourceView(DeleteView, FormMixin):
         delete_media_val = request.POST.get('delete_media', False)
         delete_media = True if delete_media_val is not False else False
         if delete_media:
+            source = self.get_object()
             for media in Media.objects.filter(source=source):
                 if media.media_file:
                     delete_file(media.media_file.name)
@@ -378,12 +382,29 @@ class MediaItemView(DetailView):
 
     template_name = 'sync/media-item.html'
     model = Media
+    messages = {
+        'redownloading': _('Media file has been deleted and scheduled to redownload'),
+    }
+
+    def __init__(self, *args, **kwargs):
+        self.message = None
+        super().__init__(*args, **kwargs)
+
+    def dispatch(self, request, *args, **kwargs):
+        message_key = request.GET.get('message', '')
+        self.message = self.messages.get(message_key, '')
+        return super().dispatch(request, *args, **kwargs)
 
     def get_context_data(self, *args, **kwargs):
         data = super().get_context_data(*args, **kwargs)
+        data['message'] = self.message
         combined_exact, combined_format = self.object.get_best_combined_format()
         audio_exact, audio_format = self.object.get_best_audio_format()
         video_exact, video_format = self.object.get_best_video_format()
+        task = get_media_download_task(self.object.pk)
+        data['task'] = task
+        data['download_state'] = self.object.get_download_state(task)
+        data['download_state_icon'] = self.object.get_download_state_icon(task)
         data['combined_exact'] = combined_exact
         data['combined_format'] = combined_format
         data['audio_exact'] = audio_exact
@@ -394,6 +415,48 @@ class MediaItemView(DetailView):
         return data
 
 
+class MediaRedownloadView(FormView, SingleObjectMixin):
+
+    template_name = 'sync/media-redownload.html'
+    form_class = RedownloadMediaForm
+    model = Media
+
+    def __init__(self, *args, **kwargs):
+        self.object = None
+        super().__init__(*args, **kwargs)
+
+    def dispatch(self, request, *args, **kwargs):
+        self.object = self.get_object()
+        return super().dispatch(request, *args, **kwargs)
+
+    def form_valid(self, form):
+        # Delete any active download tasks for the media
+        delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),))
+        # If the thumbnail file exists on disk, delete it
+        if self.object.thumb_file_exists:
+            delete_file(self.object.thumb.path)
+            self.object.thumb = None
+        # If the media file exists on disk, delete it
+        if self.object.media_file_exists:
+            delete_file(self.object.media_file.path)
+            self.object.media_file = None
+        # Reset all download data
+        self.object.downloaded = False
+        self.object.downloaded_audio_codec = None
+        self.object.downloaded_video_codec = None
+        self.object.downloaded_container = None
+        self.object.downloaded_fps = None
+        self.object.downloaded_hdr = False
+        self.object.downloaded_filesize = None
+        # Saving here will trigger the post_create signals to schedule new tasks
+        self.object.save()
+        return super().form_valid(form)
+
+    def get_success_url(self):
+        url = reverse_lazy('sync:media-item', kwargs={'pk': self.object.pk})
+        return append_uri_params(url, {'message': 'redownloading'})
+
+
 class TasksView(ListView):
     '''
         A list of tasks queued to be completed. This is, for example, scraping for new
@@ -420,6 +483,7 @@ class TasksView(ListView):
                 continue
             setattr(task, 'instance', obj)
             setattr(task, 'url', url)
+            setattr(task, 'run_now', task.run_at < now)
             if task.locked_by_pid_running():
                 data['running'].append(task)
             elif task.has_error():

tubesync/tubesync/settings.py

@@ -117,8 +117,8 @@ MAX_ATTEMPTS = 10                           # Number of times tasks will be retr
 MAX_RUN_TIME = 1800                         # Maximum amount of time in seconds a task can run
 BACKGROUND_TASK_RUN_ASYNC = True            # Run tasks async in the background
 BACKGROUND_TASK_ASYNC_THREADS = 2           # Number of async tasks to run at once
-BACKGROUND_TASK_PRIORITY_ORDERING = 'DESC'  # Process high priority tasks first
+BACKGROUND_TASK_PRIORITY_ORDERING = 'ASC'  # Use 'niceness' task priority ordering
-COMPLETED_TASKS_DAYS_TO_KEEP = 30            # Number of days to keep completed tasks
+COMPLETED_TASKS_DAYS_TO_KEEP = 30           # Number of days to keep completed tasks
 
 
 SOURCES_PER_PAGE = 36