switch source indexing schedule to a per-source setting

app/sync/migrations/0012_auto_20201207_0415.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.4 on 2020-12-07 04:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0011_auto_20201206_0911'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='index_schedule',
+            field=models.IntegerField(db_index=True, default=21600, help_text='Schedule of when to index the source for new media', verbose_name='index schedule'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='source_acodec',
+            field=models.CharField(choices=[('MP4A', 'MP4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'),
+        ),
+    ]

app/sync/migrations/0013_auto_20201207_0439.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.4 on 2020-12-07 04:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0012_auto_20201207_0415'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='index_schedule',
+            field=models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours')], db_index=True, default=21600, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule'),
+        ),
+    ]

app/sync/models.py

@@ -102,6 +102,16 @@ class Source(models.Model):
         SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id',
     }
 
+    class IndexSchedule(models.IntegerChoices):
+        EVERY_HOUR = 3600, _('Every hour')
+        EVERY_2_HOURS = 7200, _('Every 2 hours')
+        EVERY_3_HOURS = 10800, _('Every 3 hours')
+        EVERY_4_HOURS = 14400, _('Every 4 hours')
+        EVERY_5_HOURS = 18000, _('Every 5 hours')
+        EVERY_6_HOURS = 21600, _('Every 6 hours')
+        EVERY_12_HOURS = 43200, _('Every 12 hours')
+        EVERY_24_HOURS = 86400, _('Every 24 hours')
+
     uuid = models.UUIDField(
         _('uuid'),
         primary_key=True,
@@ -151,6 +161,13 @@ class Source(models.Model):
         unique=True,
         help_text=_('Directory name to save the media into')
     )
+    index_schedule = models.IntegerField(
+        _('index schedule'),
+        choices=IndexSchedule.choices,
+        db_index=True,
+        default=IndexSchedule.EVERY_6_HOURS,
+        help_text=_('Schedule of how often to index the source for new media')
+    )
     delete_old_media = models.BooleanField(
         _('delete old media'),
         default=False,

app/sync/signals.py

@@ -9,10 +9,10 @@ from .utils import delete_file
 
 @receiver(post_save, sender=Source)
 def source_post_save(sender, instance, created, **kwargs):
-    # Triggered when a source is saved
+    # Triggered when a source is saved, delete any source tasks that might exist
-    if created:
+    delete_index_source_task(str(instance.pk))
-        # If the source is newly created schedule its indexing
+    # Create a new scheduled indexing task as the repeat schedule may have changed
-        index_source_task(str(instance.pk), repeat=settings.INDEX_SOURCE_EVERY)
+    index_source_task(str(instance.pk), repeat=instance.index_schedule)
 
 
 @receiver(pre_delete, sender=Source)

app/sync/tasks.py

@@ -11,6 +11,7 @@ from PIL import Image
 from django.conf import settings
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.utils import timezone
+from django.db.utils import IntegrityError
 from background_task import background
 from background_task.models import Task
 from common.logger import log
@@ -58,8 +59,11 @@ def index_source_task(source_id):
         upload_date = media.upload_date
         if upload_date:
             media.published = timezone.make_aware(upload_date)
+        try:
             media.save()
             log.info(f'Indexed media: {source} / {media}')
+        except IntegrityError as e:
+            log.error(f'Index media failed: {source} / {media} with "{e}"')
 
 
 @background(schedule=0)

app/sync/templates/sync/source.html

@@ -30,6 +30,10 @@
         <td class="hide-on-small-only">Directory</td>
         <td><span class="hide-on-med-and-up">Directory<br></span><strong>{{ source.directory }}</strong></td>
       </tr>
+      <tr title="Schedule of how often to index the source for new media">
+        <td class="hide-on-small-only">Index schedule</td>
+        <td><span class="hide-on-med-and-up">Index schedule<br></span><strong>{{ source.get_index_schedule_display }}</strong></td>
+      </tr>
       <tr title="When then source was created locally in TubeSync">
         <td class="hide-on-small-only">Created</td>
         <td><span class="hide-on-med-and-up">Created<br></span><strong>{{ source.created|date:'Y-m-d H:i:s' }}</strong></td>

app/sync/views.py

@@ -200,9 +200,9 @@ class AddSourceView(CreateView):
 
     template_name = 'sync/source-add.html'
     model = Source
-    fields = ('source_type', 'key', 'name', 'directory', 'delete_old_media',
+    fields = ('source_type', 'key', 'name', 'directory', 'index_schedule',
-              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
+              'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
-              'prefer_60fps', 'prefer_hdr', 'fallback')
+              'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback')
 
     def __init__(self, *args, **kwargs):
         self.prepopulated_data = {}
@@ -244,9 +244,9 @@ class UpdateSourceView(UpdateView):
 
     template_name = 'sync/source-update.html'
     model = Source
-    fields = ('source_type', 'key', 'name', 'directory', 'delete_old_media',
+    fields = ('source_type', 'key', 'name', 'directory', 'index_schedule',
-              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
+              'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
-              'prefer_60fps', 'prefer_hdr', 'fallback')
+              'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback')
 
     def get_success_url(self):
         url = reverse_lazy('sync:sources')

app/tubesync/settings.py

@@ -124,9 +124,6 @@ SOURCES_PER_PAGE = 36
 MEDIA_PER_PAGE = 36
 
 
-INDEX_SOURCE_EVERY = 21600  # Seconds between indexing sources, 21600 = every 6 hours
-
-
 MEDIA_THUMBNAIL_WIDTH = 430                 # Width in pixels to resize thumbnails to
 MEDIA_THUMBNAIL_HEIGHT = 240                # Height in pixels to resize thumbnails to