bump ffmpeg to 2023-11-14 and yt-dlp to 2023.11.16

Adds workflow to delete local media that no longer exists in the source

Dockerfile

@@ -2,8 +2,8 @@ FROM debian:bookworm-slim
 
 ARG TARGETPLATFORM
 ARG S6_VERSION="3.1.5.0"
-ARG FFMPEG_DATE="autobuild-2023-09-24-14-11"
+ARG FFMPEG_DATE="autobuild-2023-11-14-14-18"
-ARG FFMPEG_VERSION="112171-g13a3e2a9b4"
+ARG FFMPEG_VERSION="112750-g6d60cc7baf"
 
 ENV DEBIAN_FRONTEND="noninteractive" \
   HOME="/root" \
@@ -27,8 +27,8 @@ RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
   "linux/arm64")   echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-aarch64.tar.xz" ;; \
   *)               echo ""        ;; esac) && \
   export FFMPEG_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
-  "linux/amd64")   echo "71cd08ed38c33ff2625dcca68d05efda090bdae455625d3bb1e4be4a53bf7c11" ;; \
+  "linux/amd64")   echo "d905684195f16412d8ee4a61a5a32d4bea530b4f93260e800b5a74904f6a1528" ;; \
-  "linux/arm64")   echo "b6765d97f20cecef0121559ee26a2f0dfbac6aef49c48c71eb703271cb3f527b" ;; \
+  "linux/arm64")   echo "5fdbf8d83d05b39d3e1cd666d485340115bc31cfc686993dcb77f99d1b35751e" ;; \
   *)               echo ""        ;; esac) && \
   export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \
   "linux/amd64")   echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linux64-gpl.tar.xz"   ;; \

docs/other-database-backends.md

@@ -24,7 +24,7 @@ $ docker exec -i tubesync python3 /app/manage.py dumpdata > some-file.json
 Then change you database backend over, then use
 
 ```bash
-$ cat some-file.json | docker exec -i tubesync python3 /app/manage.py loaddata --format=json -
+$ cat some-file.json | docker exec -i tubesync python3 /app/manage.py loaddata - --format=json
 ```
 
 As detailed in the Django documentation:

tubesync/sync/migrations/0019_add_delete_removed_media.py

@@ -0,0 +1,17 @@
+# Generated by pac
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0018_source_subtitles'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='delete_removed_media',
+            field=models.BooleanField(default=False, help_text='Delete media that is no longer on this playlist', verbose_name='delete removed media'),
+        ),
+    ]

tubesync/sync/models.py

@@ -287,6 +287,11 @@ class Source(models.Model):
         help_text=_('If "delete old media" is ticked, the number of days after which '
                     'to automatically delete media')
     )
+    delete_removed_media = models.BooleanField(
+        _('delete removed media'),
+        default=False,
+        help_text=_('Delete media that is no longer on this playlist')
+    )
     source_resolution = models.CharField(
         _('source resolution'),
         max_length=8,
@@ -511,6 +516,7 @@ class Source(models.Model):
             'dd': now.strftime('%d'),
             'source': self.slugname,
             'source_full': self.name,
+            'uploader': 'Some Channel Name',
             'title': 'some-media-title-name',
             'title_full': 'Some Media Title Name',
             'key': 'SoMeUnIqUiD',
@@ -1020,6 +1026,7 @@ class Media(models.Model):
             'acodec': display_format['acodec'],
             'fps': display_format['fps'],
             'hdr': display_format['hdr'],
+            'uploader': self.uploader,
         }
 
     @property

tubesync/sync/signals.py

@@ -156,7 +156,7 @@ def media_post_save(sender, instance, created, **kwargs):
         verbose_name = _('Downloading metadata for "{}"')
         download_media_metadata(
             str(instance.pk),
-            priority=10,
+            priority=5,
             verbose_name=verbose_name.format(instance.pk),
             remove_existing_tasks=True
         )

tubesync/sync/tasks.py

@@ -142,6 +142,15 @@ def cleanup_old_media():
             media.delete()
 
 
+def cleanup_removed_media(source, videos):
+    media_objects = Media.objects.filter(source=source, downloaded=True)
+    for item in media_objects:
+        matching_source_item = [video['id'] for video in videos if video['id'] == item.key]
+        if not matching_source_item:
+            log.info(f'{item.title} is no longer in source, removing')
+            item.delete()
+
+
 @background(schedule=0)
 def index_source_task(source_id):
     '''
@@ -186,6 +195,9 @@ def index_source_task(source_id):
     cleanup_completed_tasks()
     # Tack on a cleanup of old media
     cleanup_old_media()
+    if source.delete_removed_media:
+        log.info(f'Cleaning up media no longer in source {source}')
+        cleanup_removed_media(source, videos)
 
 
 @background(schedule=0)

tubesync/sync/templates/sync/_mediaformatvars.html

@@ -43,6 +43,11 @@
       <td>Full source name</td>
       <td>My Source</td>
     </tr>
+    <tr>
+      <td>{uploader}</td>
+      <td>Uploader name</td>
+      <td>Some Channel Name</td>
+    </tr>
     <tr>
       <td>{title}</td>
       <td>Lower case media title, max 80 chars</td>

tubesync/sync/templates/sync/source.html

@@ -115,6 +115,10 @@
         <td class="hide-on-small-only">Write JSON?</td>
         <td><span class="hide-on-med-and-up">Write JSON?<br></span><strong>{% if source.write_json %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
+      <tr title="Delete media that is no longer on this playlist?">
+        <td class="hide-on-small-only">Delete removed media</td>
+        <td><span class="hide-on-med-and-up">Delete removed media<br></span><strong>{% if source.delete_removed_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
       {% if source.delete_old_media and source.days_to_keep > 0 %}
       <tr title="Days after which your media from this source will be locally deleted">
         <td class="hide-on-small-only">Delete old media</td>

tubesync/sync/views.py

@@ -296,10 +296,11 @@ class EditSourceMixin:
     model = Source
     fields = ('source_type', 'key', 'name', 'directory', 'media_format',
               'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
-              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
+              'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
-              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 
+              'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails',
-              'write_json', 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock',
+              'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
-              'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs')
+              'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
+              'auto_subtitles', 'sub_langs')
     errors = {
         'invalid_media_format': _('Invalid media format, the media format contains '
                                   'errors or is empty. Check the table at the end of '

tubesync/tubesync/settings.py

@@ -6,7 +6,7 @@ CONFIG_BASE_DIR = BASE_DIR
 DOWNLOADS_BASE_DIR = BASE_DIR
 
 
-VERSION = '0.13.0'
+VERSION = '0.13.2'
 SECRET_KEY = ''
 DEBUG = False
 ALLOWED_HOSTS = []