bump to 0.13.3

Fix signals.py mistake

.github/workflows/ci.yaml

@@ -4,6 +4,7 @@ env:
   IMAGE_NAME: tubesync
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - main

Dockerfile

@@ -2,8 +2,8 @@ FROM debian:bookworm-slim
 
 ARG TARGETPLATFORM
 ARG S6_VERSION="3.1.5.0"
-ARG FFMPEG_DATE="autobuild-2023-09-24-14-11"
-ARG FFMPEG_VERSION="112171-g13a3e2a9b4"
+ARG FFMPEG_DATE="autobuild-2023-11-29-14-19"
+ARG FFMPEG_VERSION="112875-g47e214245b"
 
 ENV DEBIAN_FRONTEND="noninteractive" \
   HOME="/root" \
@@ -27,8 +27,8 @@ RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
   "linux/arm64")   echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-aarch64.tar.xz" ;; \
   *)               echo ""        ;; esac) && \
   export FFMPEG_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
-  "linux/amd64")   echo "71cd08ed38c33ff2625dcca68d05efda090bdae455625d3bb1e4be4a53bf7c11" ;; \
-  "linux/arm64")   echo "b6765d97f20cecef0121559ee26a2f0dfbac6aef49c48c71eb703271cb3f527b" ;; \
+  "linux/amd64")   echo "36bac8c527bf390603416f749ab0dd860142b0a66f0865b67366062a9c286c8b" ;; \
+  "linux/arm64")   echo "8f36e45d99d2367a5c0c220ee3164fa48f4f0cec35f78204ccced8dc303bfbdc" ;; \
   *)               echo ""        ;; esac) && \
   export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \
   "linux/amd64")   echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linux64-gpl.tar.xz"   ;; \

Makefile

@@ -29,6 +29,10 @@ runcontainer:
 	$(docker) run --rm --name $(name) --env-file dev.env --log-opt max-size=50m -ti -p 4848:4848 $(image)
 
 
+stopcontainer:
+	$(docker) stop $(name)
+
+
 test: build
 	cd tubesync && $(python) manage.py test --verbosity=2 && cd ..
 

docs/other-database-backends.md

@@ -24,7 +24,7 @@ $ docker exec -i tubesync python3 /app/manage.py dumpdata > some-file.json
 Then change you database backend over, then use
 
 ```bash
-$ cat some-file.json | docker exec -i tubesync python3 /app/manage.py loaddata --format=json -
+$ cat some-file.json | docker exec -i tubesync python3 /app/manage.py loaddata - --format=json
 ```
 
 As detailed in the Django documentation:
@@ -78,3 +78,46 @@ entry in the container or stdout logs:
 
 If you see a line similar to the above and the web interface loads, congratulations,
 you are now using an external database server for your TubeSync data!
+
+## Docker Compose
+
+If you're using Docker Compose and simply want to connect to another container with
+the DB for the performance benefits, a configuration like this would be enough:
+
+```
+ tubesync-db:
+  image: postgres:15.2
+  container_name: tubesync-db
+  restart: unless-stopped
+  volumes:
+   - /<path/to>/init.sql:/docker-entrypoint-initdb.d/init.sql
+   - /<path/to>/tubesync-db:/var/lib/postgresql/data
+  environment:
+   - POSTGRES_USER=postgres
+   - POSTGRES_PASSWORD=testpassword
+
+ tubesync:
+  image: ghcr.io/meeb/tubesync:latest
+  container_name: tubesync
+  restart: unless-stopped
+  ports:
+   - 4848:4848
+  volumes:
+   - /<path/to>/tubesync/config:/config
+   - /<path/to>/YouTube:/downloads
+  environment:
+   - DATABASE_CONNECTION=postgresql://postgres:testpassword@tubesync-db:5432/tubesync
+  depends_on:
+   - tubesync-db
+```
+
+Note that an `init.sql` file is needed to initialize the `tubesync`
+database before it can be written to. This file should contain:
+
+```
+CREATE DATABASE tubesync;
+```
+
+Then it must be mapped to `/docker-entrypoint-initdb.d/init.sql` for it
+to be executed on first startup of the container. See the `tubesync-db`
+volume mapping above for how to do this.

tubesync/common/logger.py

@@ -1,10 +1,14 @@
 import logging
+from django.conf import settings
+
+
+logging_level = logging.DEBUG if settings.DEBUG else logging.INFO
 
 
 log = logging.getLogger('tubesync')
-log.setLevel(logging.DEBUG)
+log.setLevel(logging_level)
 ch = logging.StreamHandler()
-ch.setLevel(logging.DEBUG)
+ch.setLevel(logging_level)
 formatter = logging.Formatter('%(asctime)s [%(name)s/%(levelname)s] %(message)s')
 ch.setFormatter(formatter)
 log.addHandler(ch)

tubesync/common/templates/pagination.html

@@ -3,7 +3,7 @@
   <div class="col s12">
     <div class="pagination">
       {% for i in paginator.page_range %}
-        <a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}{% if show_skipped %}&show_skipped=yes{% endif %}">{{ i }}</a>
+        <a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}{% if show_skipped %}&show_skipped=yes{% endif %}{% if only_skipped %}&only_skipped=yes{% endif %}">{{ i }}</a>
       {% endfor %}
     </div>
   </div>

tubesync/sync/migrations/0019_add_delete_removed_media.py

@@ -0,0 +1,17 @@
+# Generated by pac
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0018_source_subtitles'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='delete_removed_media',
+            field=models.BooleanField(default=False, help_text='Delete media that is no longer on this playlist', verbose_name='delete removed media'),
+        ),
+    ]

tubesync/sync/migrations/0020_auto_20231024_1825.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.22 on 2023-10-24 17:25
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0019_add_delete_removed_media'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='filter_text',
+            field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=100, verbose_name='filter string'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='auto_subtitles',
+            field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto-generated subs'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='sub_langs',
+            field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z]+,)*(\\-?[\\_\\.a-zA-Z]+){1}$')], verbose_name='subs langs'),
+        ),
+    ]

tubesync/sync/models.py

@@ -1,6 +1,7 @@
 import os
 import uuid
 import json
+import re
 from xml.etree import ElementTree
 from collections import OrderedDict
 from datetime import datetime, timedelta
@@ -287,6 +288,18 @@ class Source(models.Model):
         help_text=_('If "delete old media" is ticked, the number of days after which '
                     'to automatically delete media')
     )
+    filter_text = models.CharField(
+        _('filter string'),
+        max_length=100,
+        default='',
+        blank=True,
+        help_text=_('Regex compatible filter string for video titles')
+    )
+    delete_removed_media = models.BooleanField(
+        _('delete removed media'),
+        default=False,
+        help_text=_('Delete media that is no longer on this playlist')
+    )
     source_resolution = models.CharField(
         _('source resolution'),
         max_length=8,
@@ -511,6 +524,7 @@ class Source(models.Model):
             'dd': now.strftime('%d'),
             'source': self.slugname,
             'source_full': self.name,
+            'uploader': 'Some Channel Name',
             'title': 'some-media-title-name',
             'title_full': 'Some Media Title Name',
             'key': 'SoMeUnIqUiD',
@@ -532,6 +546,11 @@ class Source(models.Model):
         except Exception as e:
             return ''
 
+    def is_regex_match(self, media_item_title):
+        if not self.filter_text:
+            return True
+        return bool(re.search(self.filter_text, media_item_title))
+    
     def index_media(self):
         '''
             Index the media source returning a list of media metadata as dicts.
@@ -1020,6 +1039,7 @@ class Media(models.Model):
             'acodec': display_format['acodec'],
             'fps': display_format['fps'],
             'hdr': display_format['hdr'],
+            'uploader': self.uploader,
         }
 
     @property

tubesync/sync/signals.py

@@ -96,14 +96,14 @@ def media_post_save(sender, instance, created, **kwargs):
     # If the media is skipped manually, bail.
     if instance.manual_skip:
         return
-
     # Triggered after media is saved
     cap_changed = False
     can_download_changed = False
     # Reset the skip flag if the download cap has changed if the media has not
     # already been downloaded
-    if not instance.downloaded:
+    if not instance.downloaded and instance.metadata:
         max_cap_age = instance.source.download_cap_date
+        filter_text = instance.source.filter_text.strip()
         published = instance.published
         if not published:
             if not instance.skip:
@@ -117,7 +117,16 @@ def media_post_save(sender, instance, created, **kwargs):
         else:
             if max_cap_age:
                 if published > max_cap_age and instance.skip:
-                    # Media was published after the cap date but is set to be skipped
+                    if filter_text:
+                        if instance.source.is_regex_match(instance.title):
+                            log.info(f'Media: {instance.source} / {instance} has a valid '
+                                    f'publishing date and title filter, marking to be unskipped')
+                            instance.skip = False
+                            cap_changed = True
+                        else:
+                            log.debug(f'Media: {instance.source} / {instance} has a valid publishing date '
+                                      f'but failed the title filter match, already marked skipped')
+                    else:
                         log.info(f'Media: {instance.source} / {instance} has a valid '
                                  f'publishing date, marking to be unskipped')
                         instance.skip = False
@@ -130,10 +139,20 @@ def media_post_save(sender, instance, created, **kwargs):
             else:
                 if instance.skip:
                     # Media marked to be skipped but source download cap removed
+                    if filter_text:
+                        if instance.source.is_regex_match(instance.title):
                             log.info(f'Media: {instance.source} / {instance} has a valid '
-                            f'publishing date, marking to be unskipped')
+                                     f'publishing date and title filter, marking to be unskipped')
                             instance.skip = False
                             cap_changed = True
+                        else:
+                            log.info(f'Media: {instance.source} / {instance} has a valid publishing date '
+                                     f'but failed the title filter match, already marked skipped')
+                else:
+                    log.debug(f'Media: {instance.source} / {instance} has a valid publishing date and '
+                              f'is already marked as not to be skipped')
+
+                    cap_changed = False
     # Recalculate the "can_download" flag, this may
     # need to change if the source specifications have been changed
     if instance.metadata:
@@ -156,7 +175,7 @@ def media_post_save(sender, instance, created, **kwargs):
         verbose_name = _('Downloading metadata for "{}"')
         download_media_metadata(
             str(instance.pk),
-            priority=10,
+            priority=5,
             verbose_name=verbose_name.format(instance.pk),
             remove_existing_tasks=True
         )

tubesync/sync/tasks.py

@@ -142,6 +142,15 @@ def cleanup_old_media():
             media.delete()
 
 
+def cleanup_removed_media(source, videos):
+    media_objects = Media.objects.filter(source=source, downloaded=True)
+    for item in media_objects:
+        matching_source_item = [video['id'] for video in videos if video['id'] == item.key]
+        if not matching_source_item:
+            log.info(f'{item.title} is no longer in source, removing')
+            item.delete()
+
+
 @background(schedule=0)
 def index_source_task(source_id):
     '''
@@ -186,6 +195,9 @@ def index_source_task(source_id):
     cleanup_completed_tasks()
     # Tack on a cleanup of old media
     cleanup_old_media()
+    if source.delete_removed_media:
+        log.info(f'Cleaning up media no longer in source {source}')
+        cleanup_removed_media(source, videos)
 
 
 @background(schedule=0)
@@ -219,11 +231,9 @@ def download_media_metadata(media_id):
         log.error(f'Task download_media_metadata(pk={media_id}) called but no '
                   f'media exists with ID: {media_id}')
         return
-    
     if media.manual_skip:
         log.info(f'Task for ID: {media_id} skipped, due to task being manually skipped.')
         return
-
     source = media.source
     metadata = media.index_metadata()
     media.metadata = json.dumps(metadata, default=json_serial)
@@ -242,6 +252,11 @@ def download_media_metadata(media_id):
             log.warn(f'Media: {source} / {media} is older than cap age '
                      f'{max_cap_age}, skipping')
             media.skip = True
+    # If the source has a search filter, check the video title matches the filter
+    if source.filter_text and not source.is_regex_match(media.title):
+        # Filter text not found in the media title. Accepts regex string, blank search filter results in this returning false
+        log.warn(f'Media: {source} / {media} does not match {source.filter_text}, skipping')
+        media.skip = True
     # If the source has a cut-off check the upload date is within the allowed delta
     if source.delete_old_media and source.days_to_keep > 0:
         if not isinstance(media.published, datetime):

tubesync/sync/templates/sync/_mediaformatvars.html

@@ -43,6 +43,11 @@
       <td>Full source name</td>
       <td>My Source</td>
     </tr>
+    <tr>
+      <td>{uploader}</td>
+      <td>Uploader name</td>
+      <td>Some Channel Name</td>
+    </tr>
     <tr>
       <td>{title}</td>
       <td>Lower case media title, max 80 chars</td>

tubesync/sync/templates/sync/media.html

@@ -64,5 +64,5 @@
   </div>
   {% endfor %}
 </div>
-{% include 'pagination.html' with pagination=sources.paginator filter=source.pk show_skipped=show_skipped %}
+{% include 'pagination.html' with pagination=sources.paginator filter=source.pk show_skipped=show_skipped only_skipped=only_skipped%}
 {% endblock %}

tubesync/sync/templates/sync/source.html

@@ -43,6 +43,10 @@
         <td class="hide-on-small-only">Directory</td>
         <td><span class="hide-on-med-and-up">Directory<br></span><strong>{{ source.directory }}</strong></td>
       </tr>
+      <tr title="Filter text">
+        <td class="hide-on-small-only">Filter text</td>
+        <td><span class="hide-on-med-and-up">Filter text<br></span><strong>{{ source.filter_text }}</strong></td>
+      </tr>
       <tr title="Media file name format to use for saving files">
         <td class="hide-on-small-only">Media format</td>
         <td><span class="hide-on-med-and-up">Media format<br></span><strong>{{ source.media_format }}</strong></td>
@@ -115,6 +119,10 @@
         <td class="hide-on-small-only">Write JSON?</td>
         <td><span class="hide-on-med-and-up">Write JSON?<br></span><strong>{% if source.write_json %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
+      <tr title="Delete media that is no longer on this playlist?">
+        <td class="hide-on-small-only">Delete removed media</td>
+        <td><span class="hide-on-med-and-up">Delete removed media<br></span><strong>{% if source.delete_removed_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
       {% if source.delete_old_media and source.days_to_keep > 0 %}
       <tr title="Days after which your media from this source will be locally deleted">
         <td class="hide-on-small-only">Delete old media</td>

tubesync/sync/tests.py

@@ -175,6 +175,7 @@ class FrontEndTestCase(TestCase):
             'directory': 'testdirectory',
             'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
             'download_cap': 0,
+            'filter_text':'.*',
             'index_schedule': 3600,
             'delete_old_media': False,
             'days_to_keep': 14,
@@ -217,6 +218,7 @@ class FrontEndTestCase(TestCase):
             'directory': 'testdirectory',
             'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
             'download_cap': 0,
+            'filter_text':'.*',
             'index_schedule': Source.IndexSchedule.EVERY_HOUR,
             'delete_old_media': False,
             'days_to_keep': 14,
@@ -247,6 +249,7 @@ class FrontEndTestCase(TestCase):
             'directory': 'testdirectory',
             'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
             'download_cap': 0,
+            'filter_text':'.*',
             'index_schedule': Source.IndexSchedule.EVERY_2_HOURS,  # changed
             'delete_old_media': False,
             'days_to_keep': 14,
@@ -1468,6 +1471,29 @@ class FormatMatchingTestCase(TestCase):
             self.media.get_best_video_format()
             self.media.get_best_audio_format()
 
+    def test_is_regex_match(self):
+        
+        self.media.metadata = all_test_metadata['boring']
+        expected_matches = {
+            ('.*'): (True),
+            ('no fancy stuff'): (True),
+            ('No fancy stuff'): (False),
+            ('(?i)No fancy stuff'): (True), #set case insensitive flag
+            ('no'): (True),
+            ('Foo'): (False),
+            ('^(?!.*fancy).*$'): (False),
+            ('^(?!.*funny).*$'): (True),
+            ('(?=.*f.*)(?=.{0,2}|.{4,})'): (True),
+            ('f{4,}'): (False),
+            ('^[^A-Z]*$'): (True),
+            ('^[^a-z]*$'): (False),
+            ('^[^\\s]*$'): (False)
+        }
+
+        for params, expected in expected_matches.items():
+            self.source.filter_text = params
+            expected_match_result = expected
+            self.assertEqual(self.source.is_regex_match(self.media.title), expected_match_result)
 
 class TasksTestCase(TestCase):
     def setUp(self):

tubesync/sync/views.py

@@ -294,12 +294,13 @@ class ValidateSourceView(FormView):
 
 class EditSourceMixin:
     model = Source
-    fields = ('source_type', 'key', 'name', 'directory', 'media_format',
+    fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format',
               'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
-              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
-              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 
-              'write_json', 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock',
-              'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs')
+              'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
+              'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails',
+              'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
+              'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
+              'auto_subtitles', 'sub_langs')
     errors = {
         'invalid_media_format': _('Invalid media format, the media format contains '
                                   'errors or is empty. Check the table at the end of '

tubesync/tubesync/settings.py

@@ -6,7 +6,7 @@ CONFIG_BASE_DIR = BASE_DIR
 DOWNLOADS_BASE_DIR = BASE_DIR
 
 
-VERSION = '0.13.0'
+VERSION = '0.13.3'
 SECRET_KEY = ''
 DEBUG = False
 ALLOWED_HOSTS = []