bump to 0.13.0

Subtitles

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -131,4 +132,5 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-Pipfile.lock
+Pipfile.lock
+.vscode/launch.json

Dockerfile

@@ -1,9 +1,9 @@
-FROM debian:bullseye-slim
+FROM debian:bookworm-slim
 
 ARG TARGETPLATFORM
-ARG S6_VERSION="3.1.2.1"
-ARG FFMPEG_DATE="autobuild-2023-01-03-12-55"
-ARG FFMPEG_VERSION="109474-gc94988a781"
+ARG S6_VERSION="3.1.5.0"
+ARG FFMPEG_DATE="autobuild-2023-09-24-14-11"
+ARG FFMPEG_VERSION="112171-g13a3e2a9b4"
 
 ENV DEBIAN_FRONTEND="noninteractive" \
   HOME="/root" \
@@ -19,22 +19,22 @@ RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
   "linux/arm64")   echo "aarch64" ;; \
   *)               echo ""        ;; esac) && \
   export S6_ARCH_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
-  "linux/amd64")   echo "6019b6b06cfdbb1d1cd572d46b9b158a4904fd19ca59d374de4ddaaa6a3727d5" ;; \
-  "linux/arm64")   echo "e73f9a021b64f88278830742149c14ef8a52331102881ba025bf32a66a0e7c78" ;; \
+  "linux/amd64")   echo "65d0d0f353d2ff9d0af202b268b4bf53a9948a5007650854855c729289085739" ;; \
+  "linux/arm64")   echo "3fbd14201473710a592b2189e81f00f3c8998e96d34f16bd2429c35d1bc36d00" ;; \
   *)               echo ""        ;; esac) && \
   export S6_DOWNLOAD_ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
   "linux/amd64")   echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-x86_64.tar.xz"   ;; \
   "linux/arm64")   echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-aarch64.tar.xz" ;; \
   *)               echo ""        ;; esac) && \
   export FFMPEG_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
-  "linux/amd64")   echo "ed9059668e4a6dac9bde122a775f52ad08cbb90df3658f8c1e328477c13c242e" ;; \
-  "linux/arm64")   echo "dd1375bd351d38ea1cc3efd68a998699366e28bd9b90df65d11af2b9121746b7" ;; \
+  "linux/amd64")   echo "71cd08ed38c33ff2625dcca68d05efda090bdae455625d3bb1e4be4a53bf7c11" ;; \
+  "linux/arm64")   echo "b6765d97f20cecef0121559ee26a2f0dfbac6aef49c48c71eb703271cb3f527b" ;; \
   *)               echo ""        ;; esac) && \
   export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \
   "linux/amd64")   echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linux64-gpl.tar.xz"   ;; \
   "linux/arm64")   echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linuxarm64-gpl.tar.xz" ;; \
   *)               echo ""        ;; esac) && \
-  export S6_NOARCH_EXPECTED_SHA256="cee89d3eeabdfe15239b2c5c3581d9352d2197d4fd23bba3f1e64bf916ccf496" && \
+  export S6_NOARCH_EXPECTED_SHA256="fd80c231e8ae1a0667b7ae2078b9ad0e1269c4d117bf447a4506815a700dbff3" && \
   export S6_DOWNLOAD_NOARCH="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-noarch.tar.xz" && \
   echo "Building for arch: ${ARCH}|${ARCH44}, downloading S6 from: ${S6_DOWNLOAD}}, expecting S6 SHA256: ${S6_EXPECTED_SHA256}" && \
   set -x && \
@@ -83,30 +83,30 @@ RUN set -x && \
   apt-get -y install nginx-light && \
   apt-get -y --no-install-recommends install \
   python3 \
-  python3-setuptools \
-  python3-pip \
   python3-dev \
+  python3-pip \
+  python3-wheel \
+  pipenv \
   gcc \
   g++ \
   make \
+  pkgconf \
   default-libmysqlclient-dev \
   libmariadb3 \
   postgresql-common \
   libpq-dev \
   libpq5 \
   libjpeg62-turbo \
-  libwebp6 \
+  libwebp7 \
   libjpeg-dev \
   zlib1g-dev \
   libwebp-dev \
   redis-server && \
-  # Install pipenv
-  pip3 --disable-pip-version-check install wheel pipenv && \
   # Create a 'app' user which the application will run as
   groupadd app && \
   useradd -M -d /app -s /bin/false -g app app && \
   # Install non-distro packages
-  pipenv install --system --skip-lock && \
+  PIPENV_VERBOSITY=64 pipenv install --system --skip-lock && \
   # Make absolutely sure we didn't accidentally bundle a SQLite dev database
   rm -rf /app/db.sqlite3 && \
   # Run any required app commands
@@ -120,7 +120,6 @@ RUN set -x && \
   # Clean up
   rm /app/Pipfile && \
   pipenv --clear && \
-  pip3 --disable-pip-version-check uninstall -y pipenv wheel virtualenv && \
   apt-get -y autoremove --purge \
   python3-pip \
   python3-dev \

Makefile

@@ -31,3 +31,7 @@ runcontainer:
 
 test: build
 	cd tubesync && $(python) manage.py test --verbosity=2 && cd ..
+
+
+shell:
+	cd tubesync && $(python) manage.py shell

Pipfile

@@ -4,6 +4,7 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
+autopep8 = "*"
 
 [packages]
 django = "~=3.2"
@@ -15,10 +16,10 @@ gunicorn = "*"
 django-compressor = "*"
 httptools = "*"
 django-background-tasks = "*"
-requests = "*"
 django-basicauth = "*"
 psycopg2-binary = "*"
 mysqlclient = "*"
 yt-dlp = "*"
 redis = "*"
 hiredis = "*"
+requests = {extras = ["socks"], version = "*"}

README.md

@@ -241,6 +241,7 @@ and less common features:
  * [Reset tasks from the command line](https://github.com/meeb/tubesync/blob/main/docs/reset-tasks.md)
  * [Using PostgreSQL, MySQL or MariaDB as database backends](https://github.com/meeb/tubesync/blob/main/docs/other-database-backends.md)
  * [Using cookies](https://github.com/meeb/tubesync/blob/main/docs/using-cookies.md)
+ * [Reset metadata](https://github.com/meeb/tubesync/blob/main/docs/reset-metadata.md)
 
 
 # Warnings
@@ -361,19 +362,20 @@ There are a number of other environment variables you can set. These are, mostly
 **NOT** required to be set in the default container installation, they are really only
 useful if you are manually installing TubeSync in some other environment. These are:
 
-| Name                     | What                                                         | Example                              |
-| ------------------------ | ------------------------------------------------------------ | ------------------------------------ |
-| DJANGO_SECRET_KEY        | Django's SECRET_KEY                                          | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l     |
-| DJANGO_URL_PREFIX        | Run TubeSync in a sub-URL on the web server                  | /somepath/                           |
-| TUBESYNC_DEBUG           | Enable debugging                                             | True                                 |
-| TUBESYNC_WORKERS         | Number of background workers, default is 2, max allowed is 8 | 2                                    |
-| TUBESYNC_HOSTS           | Django's ALLOWED_HOSTS, defaults to `*`                      | tubesync.example.com,otherhost.com   |
-| GUNICORN_WORKERS         | Number of gunicorn workers to spawn                          | 3                                    |
-| LISTEN_HOST              | IP address for gunicorn to listen on                         | 127.0.0.1                            |
-| LISTEN_PORT              | Port number for gunicorn to listen on                        | 8080                                 |
-| HTTP_USER                | Sets the username for HTTP basic authentication              | some-username                        |
-| HTTP_PASS                | Sets the password for HTTP basic authentication              | some-secure-password                 |
-| DATABASE_CONNECTION      | Optional external database connection details                | mysql://user:pass@host:port/database |
+| Name                        | What                                                         | Example                              |
+| --------------------------- | ------------------------------------------------------------ | ------------------------------------ |
+| DJANGO_SECRET_KEY           | Django's SECRET_KEY                                          | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l     |
+| DJANGO_URL_PREFIX           | Run TubeSync in a sub-URL on the web server                  | /somepath/                           |
+| TUBESYNC_DEBUG              | Enable debugging                                             | True                                 |
+| TUBESYNC_WORKERS            | Number of background workers, default is 2, max allowed is 8 | 2                                    |
+| TUBESYNC_HOSTS              | Django's ALLOWED_HOSTS, defaults to `*`                      | tubesync.example.com,otherhost.com   |
+| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True  | True
+| GUNICORN_WORKERS            | Number of gunicorn workers to spawn                          | 3                                    |
+| LISTEN_HOST                 | IP address for gunicorn to listen on                         | 127.0.0.1                            |
+| LISTEN_PORT                 | Port number for gunicorn to listen on                        | 8080                                 |
+| HTTP_USER                   | Sets the username for HTTP basic authentication              | some-username                        |
+| HTTP_PASS                   | Sets the password for HTTP basic authentication              | some-secure-password                 |
+| DATABASE_CONNECTION         | Optional external database connection details                | mysql://user:pass@host:port/database |
 
 
 # Manual, non-containerised, installation

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run

@@ -11,8 +11,6 @@ chown -R app:app /run/app
 chmod -R 0700 /run/app
 chown -R app:app /config
 chmod -R 0755 /config
-chown -R app:app /downloads
-chmod -R 0755 /downloads
 chown -R root:app /app
 chmod -R 0750 /app
 chown -R app:app /app/common/static
@@ -22,6 +20,15 @@ chmod -R 0750 /app/static
 find /app -type f ! -iname healthcheck.py -exec chmod 640 {} \;
 chmod 0755 /app/healthcheck.py
 
+# Optionally reset the download dir permissions
+TUBESYNC_RESET_DOWNLOAD_DIR="${TUBESYNC_RESET_DOWNLOAD_DIR:-True}"
+if [ "$TUBESYNC_RESET_DOWNLOAD_DIR" == "True" ]
+then
+    echo "TUBESYNC_RESET_DOWNLOAD_DIR=True, Resetting /downloads directory permissions"
+    chown -R app:app /downloads
+    chmod -R 0755 /downloads
+fi
+
 # Run migrations
 exec s6-setuidgid app \
     /usr/bin/python3 /app/manage.py migrate

docs/reset-metadata.md

@@ -0,0 +1,30 @@
+# TubeSync
+
+## Advanced usage guide - reset media metadata from the command line
+
+This command allows you to reset all media item metadata. You might want to use
+this if you have a lot of media items with invalid metadata and you want to
+wipe it which triggers the metadata to be redownloaded.
+
+
+## Requirements
+
+You have added some sources and media
+
+## Steps
+
+### 1. Run the reset tasks command
+
+Execute the following Django command:
+
+`./manage.py reset-metadata`
+
+When deploying TubeSync inside a container, you can execute this with:
+
+`docker exec -ti tubesync python3 /app/manage.py reset-metadata`
+
+This command will log what its doing to the terminal when you run it.
+
+When this is run, new tasks will be immediately created so all your media
+items will start downloading updated metadata straight away, any missing information
+such as thumbnails will be redownloaded, etc.

tubesync/common/static/styles/_fonts.scss

@@ -1,19 +1,19 @@
 @font-face {
-    font-family: 'roboto-light';
+    font-family: 'roboto';
     src: url('../fonts/roboto/roboto-light.woff') format('woff');
-    font-weight: normal;
+    font-weight: lighter;
     font-style: normal;
 }
 
 @font-face {
-    font-family: 'roboto-regular';
+    font-family: 'roboto';
     src: url('../fonts/roboto/roboto-regular.woff') format('woff');
     font-weight: normal;
     font-style: normal;
 }
 
 @font-face {
-    font-family: 'roboto-bold';
+    font-family: 'roboto';
     src: url('../fonts/roboto/roboto-bold.woff') format('woff');
     font-weight: bold;
     font-style: normal;

tubesync/common/static/styles/_variables.scss

@@ -1,2 +1,2 @@
-$font-family: 'roboto-regular', Arial, Helvetica, sans-serif;
+$font-family: 'roboto', Arial, Helvetica, sans-serif;
 $font-size: 1.05rem;

tubesync/sync/fields.py

@@ -0,0 +1,109 @@
+from django.forms import MultipleChoiceField,  CheckboxSelectMultiple, Field, TypedMultipleChoiceField
+from django.db import models
+from typing import Any, Optional, Dict
+from django.utils.translation import gettext_lazy as _
+
+# this is a form field!
+class CustomCheckboxSelectMultiple(CheckboxSelectMultiple):
+    template_name = 'widgets/checkbox_select.html'
+    option_template_name = 'widgets/checkbox_option.html'
+
+    def get_context(self, name: str, value: Any, attrs) -> Dict[str, Any]:
+        ctx = super().get_context(name, value, attrs)['widget']
+        ctx["multipleChoiceProperties"] = []
+        for _group, options, _index in ctx["optgroups"]:
+            for option in options:
+                if not isinstance(value,str) and not isinstance(value,list) and ( option["value"] in value.selected_choices or ( value.allow_all and value.all_choice in value.selected_choices ) ):
+                    checked = True
+                else:
+                    checked = False
+
+                ctx["multipleChoiceProperties"].append({
+                    "template_name": option["template_name"],
+                    "type": option["type"],
+                    "value": option["value"],
+                    "label": option["label"],
+                    "name": option["name"],
+                    "checked": checked})
+
+        return { 'widget': ctx }
+
+# this is a database field!
+class CommaSepChoiceField(models.Field):
+    "Implements comma-separated storage of lists"
+
+    def __init__(self, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, *args, **kwargs):
+        self.separator = separator
+        self.possible_choices = possible_choices
+        self.selected_choices = []
+        self.allow_all = allow_all
+        self.all_label = all_label
+        self.all_choice = all_choice
+        super().__init__(*args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        if self.separator != ",":
+            kwargs['separator'] = self.separator
+        kwargs['possible_choices'] = self.possible_choices
+        return name, path, args, kwargs
+
+    def db_type(self, connection):
+        return 'text'
+
+    def get_my_choices(self):
+        choiceArray = []
+        if self.possible_choices is None:
+            return choiceArray
+        if self.allow_all:
+            choiceArray.append((self.all_choice, _(self.all_label)))
+
+        for t in self.possible_choices:
+            choiceArray.append(t)
+        
+        return choiceArray
+
+    def formfield(self, **kwargs):
+        # This is a fairly standard way to set up some defaults
+        # while letting the caller override them.
+        defaults = {'form_class': MultipleChoiceField, 
+                    'choices': self.get_my_choices,
+                    'widget': CustomCheckboxSelectMultiple,
+                    'label': '',
+                    'required': False}
+        defaults.update(kwargs)
+        #del defaults.required
+        return super().formfield(**defaults)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        # Only include kwarg if it's not the default
+        if self.separator != ",":
+            kwargs['separator'] = self.separator
+        return name, path, args, kwargs
+
+    def from_db_value(self, value, expr, conn):
+        if value is None:
+            self.selected_choices = []
+        else:
+            self.selected_choices = value.split(",")
+
+        return self
+
+    def get_prep_value(self, value):
+        if value is None:
+            return ""
+        if not isinstance(value,list):
+            return ""
+
+        if self.all_choice not in value:
+            return ",".join(value)
+        else:
+            return self.all_choice
+
+    def get_text_for_value(self, val):
+        fval = [i for i in self.possible_choices if i[0] == val]
+        if len(fval) <= 0:
+            return []
+        else:
+            return fval[0][1]

tubesync/sync/management/commands/reset-metadata.py

@@ -0,0 +1,19 @@
+from django.core.management.base import BaseCommand
+from sync.models import Media
+
+
+from common.logger import log
+
+
+class Command(BaseCommand):
+
+    help = 'Resets all media item metadata'
+
+    def handle(self, *args, **options):
+        log.info('Resettings all media metadata...')
+        # Delete all metadata
+        Media.objects.update(metadata=None)
+        # Trigger the save signal on each media item
+        for item in Media.objects.all():
+            item.save()
+        log.info('Done')

tubesync/sync/matching.py

@@ -53,6 +53,8 @@ def get_best_audio_format(media):
         # If the format has a video stream, skip it
         if fmt['vcodec'] is not None:
             continue
+        if not fmt['acodec']:
+            continue
         audio_formats.append(fmt)
     audio_formats = list(reversed(sorted(audio_formats, key=lambda k: k['abr'])))
     if not audio_formats:
@@ -88,6 +90,8 @@ def get_best_video_format(media):
         # If the format has an audio stream, skip it
         if fmt['acodec'] is not None:
             continue
+        if not fmt['vcodec']:
+            continue
         if media.source.source_resolution.strip().upper() == fmt['format']:
             video_formats.append(fmt)
     # Check we matched some streams

tubesync/sync/mediaservers.py

@@ -44,7 +44,9 @@ class PlexMediaServer(MediaServer):
              '<p>The <strong>libraries</strong> is a comma-separated list of Plex '
              'library or section IDs, you can find out how to get your library or '
              'section IDs <a href="https://support.plex.tv/articles/201242707-plex-'
-             'media-scanner-via-command-line/#toc-1" target="_blank">here</a>.</p>')
+             'media-scanner-via-command-line/#toc-1" target="_blank">here</a> or '
+             '<a href="https://www.plexopedia.com/plex-media-server/api/server/libraries/" '
+             'target="_blank">here</a></p>.')
 
     def make_request(self, uri='/', params={}):
         headers = {'User-Agent': 'TubeSync'}

tubesync/sync/migrations/0016_auto_20230214_2052.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.2.18 on 2023-02-14 20:52
+
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0015_auto_20230213_0603'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='embed_metadata',
+            field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='embed_thumbnail',
+            field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='enable_sponsorblock',
+            field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='sponsorblock_categories',
+            field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))),
+        ),
+    ]

tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.18 on 2023-02-20 02:23
+
+from django.db import migrations
+import sync.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0016_auto_20230214_2052'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='sponsorblock_categories',
+            field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', separator=''),
+        ),
+    ]

tubesync/sync/migrations/0018_source_subtitles.py

@@ -0,0 +1,27 @@
+# Generated by pac
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0017_alter_source_sponsorblock_categories'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='write_subtitles',
+            field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='auto_subtitles',
+            field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto subtitles'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='sub_langs',
+            field=models.CharField(default='en', help_text='List of subtitles langs to download comma-separated. Example: en,fr',max_length=30),
+        ),
+    ]

tubesync/sync/models.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from django.conf import settings
 from django.db import models
 from django.core.files.storage import FileSystemStorage
+from django.core.validators import RegexValidator
 from django.utils.text import slugify
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
@@ -19,11 +20,10 @@ from .utils import seconds_to_timestr, parse_media_format
 from .matching import (get_best_combined_format, get_best_audio_format, 
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
-
+from .fields import CommaSepChoiceField
 
 media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/')
 
-
 class Source(models.Model):
     '''
         A Source is a source of media. Currently, this is either a YouTube channel
@@ -106,6 +106,47 @@ class Source(models.Model):
     EXTENSION_MKV = 'mkv'
     EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
 
+
+    # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
+    SPONSORBLOCK_CATEGORIES_CHOICES = (
+        ('sponsor', 'Sponsor'),
+        ('intro', 'Intermission/Intro Animation'),
+        ('outro', 'Endcards/Credits'),
+        ('selfpromo', 'Unpaid/Self Promotion'),
+        ('preview', 'Preview/Recap'),
+        ('filler', 'Filler Tangent'),
+        ('interaction', 'Interaction Reminder'),
+        ('music_offtopic', 'Non-Music Section'),
+    )
+    
+    sponsorblock_categories = CommaSepChoiceField(
+            _(''),
+            possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
+            all_choice="all",
+            allow_all=True,
+            all_label="(all options)",
+            default="all",
+            help_text=_("Select the sponsorblocks you want to enforce")
+        )
+
+    embed_metadata = models.BooleanField(
+        _('embed metadata'),
+        default=False,
+        help_text=_('Embed metadata from source into file')
+    )
+    embed_thumbnail = models.BooleanField(
+        _('embed thumbnail'),
+        default=False,
+        help_text=_('Embed thumbnail into the file')
+    )
+
+    enable_sponsorblock = models.BooleanField(
+        _('enable sponsorblock'),
+        default=True,
+        help_text=_('Use SponsorBlock?')
+    )
+
+
     # Fontawesome icons used for the source on the front end
     ICONS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
@@ -309,6 +350,30 @@ class Source(models.Model):
         help_text=_('Source has failed to index media')
     )
 
+    write_subtitles = models.BooleanField(
+        _('write subtitles'),
+        default=False,
+        help_text=_('Download video subtitles')
+    )
+
+    auto_subtitles = models.BooleanField(
+        _('accept auto-generated subs'),
+        default=False,
+        help_text=_('Accept auto-generated subtitles')
+    )
+    sub_langs = models.CharField(
+        _('subs langs'),
+        max_length=30,
+        default='en',
+        help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'),
+        validators=[
+            RegexValidator(
+                regex=r"^(\-?[\_\.a-zA-Z]+,)*(\-?[\_\.a-zA-Z]+){1}$",
+                message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat')
+            )
+        ]
+    )
+
     def __str__(self):
         return self.name
 
@@ -1019,7 +1084,7 @@ class Media(models.Model):
         duration = self.loaded_metadata.get(field, 0)
         try:
             duration = int(duration)
-        except ValueError:
+        except (TypeError, ValueError):
             duration = 0
         return duration
 
@@ -1291,7 +1356,10 @@ class Media(models.Model):
                                     f'no valid format available')
         # Download the media with youtube-dl
         download_youtube_media(self.url, format_str, self.source.extension,
-                               str(self.filepath), self.source.write_json)
+                               str(self.filepath), self.source.write_json, 
+                               self.source.sponsorblock_categories, self.source.embed_thumbnail,
+                               self.source.embed_metadata, self.source.enable_sponsorblock,
+                              self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs )
         # Return the download paramaters
         return format_str, self.source.extension
 

tubesync/sync/tasks.py

@@ -132,16 +132,14 @@ def cleanup_completed_tasks():
 
 
 def cleanup_old_media():
-    for media in Media.objects.filter(download_date__isnull=False):
-        if media.source.delete_old_media and media.source.days_to_keep > 0:
-            delta = timezone.now() - timedelta(days=media.source.days_to_keep)
-            if media.downloaded and media.download_date < delta:
-                # Media was downloaded after the cutoff date, delete it
-                log.info(f'Deleting expired media: {media.source} / {media} '
-                         f'(now older than {media.source.days_to_keep} days / '
-                         f'download_date before {delta})')
-                # .delete() also triggers a pre_delete signal that removes the files
-                media.delete()
+    for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0):
+        delta = timezone.now() - timedelta(days=source.days_to_keep)
+        for media in source.media_source.filter(downloaded=True, download_date__lt=delta):
+            log.info(f'Deleting expired media: {source} / {media} '
+                     f'(now older than {source.days_to_keep} days / '
+                     f'download_date before {delta})')
+            # .delete() also triggers a pre_delete signal that removes the files
+            media.delete()
 
 
 @background(schedule=0)

tubesync/sync/templates/sync/source.html

@@ -130,6 +130,55 @@
         <td class="hide-on-small-only">UUID</td>
         <td><span class="hide-on-med-and-up">UUID<br></span><strong>{{ source.uuid }}</strong></td>
       </tr>
+      <tr title="{{ _('Embedding thumbnail?') }}">
+        <td class="hide-on-small-only">{{ _("Embed thumbnail?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Embed thumbnail?") }}<br></span><strong><i class="fas {% if source.embed_thumbnail %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+      <tr title="{{ _('Embedding metadata?') }}">
+        <td class="hide-on-small-only">{{ _("Embed metadata?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Embed metadata?") }}<br></span><strong><i class="fas {% if source.embed_metadata %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+
+      <tr title="{{ _('Is sponsorblock enabled?') }}">
+        <td class="hide-on-small-only">{{ _("SponsorBlock?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Sponsorblock enabled?") }}<br></span><strong><i class="fas {% if source.enable_sponsorblock %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+
+      {% if source.enable_sponsorblock %}
+      <tr title="{{ _('SponsorBlock: What to block?') }}">
+        <td class="hide-on-small-only">{{ _("What blocked?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("What blocked?") }}<br></span><strong>
+          {% if source.sponsorblock_categories.all_choice in source.sponsorblock_categories.selected_choices %}
+            {% for k,v in source.sponsorblock_categories.possible_choices %}
+              {{ v }}: <i class="fas fa-check"></i><BR>
+            {% endfor %}
+          {% else %}
+            {% for c in source.sponsorblock_categories.selected_choices %} 
+              {% for k,v in source.sponsorblock_categories.possible_choices %}
+                {% if k == c %} {{ v }}: <i class="fas fa-check"></i><BR>{% endif %}
+              {% endfor %}
+            {% endfor %}
+          {% endif %}
+        </strong></td>
+      </tr>
+      {% endif %}
+
+      <tr title="{{ _('Are Subtitles downloaded?') }}">
+        <td class="hide-on-small-only">{{ _("Download subtitles?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Download subtitles?") }}:</span><strong><i class="fas {% if source.write_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+
+      {% if source.write_subtitles %}
+      <tr title="{{ _('Are auto subs accepted?') }}">
+        <td class="hide-on-small-only">{{ _("Auto-generated subtitles?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Auto-generated subtitles?") }}:</span><strong><i class="fas {% if source.auto_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+      <tr title="{{ _('Subs langs?') }}">
+        <td class="hide-on-small-only">{{ _("Subs langs?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
+      </tr>
+      {% endif %}
+      
     </table>
   </div>
 </div>

tubesync/sync/templates/widgets/checkbox_option.html

@@ -0,0 +1,7 @@
+<!--<input type="{{ option.type }}" name="{{ option.name }}" value="{{ option.value }}" id="{{ option.value }}"><BR>
+    <label for="{{ option.value }}">{{option.label}}</label>-->
+
+<label>
+    <input type="{{ option.type }}" name="{{ option.name }}" value="{{ option.value }}" id="{{ option.value }}" {% if option.checked %}checked{% endif %}>
+    <span>{{option.label}}</span>
+</label>

tubesync/sync/templates/widgets/checkbox_select.html

@@ -0,0 +1,5 @@
+</label>
+{% for option in widget.multipleChoiceProperties %}
+    {% include option.template_name with option=option %}
+{% endfor %}
+<label>

tubesync/sync/tests.py

@@ -6,7 +6,7 @@
 
 
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
 from urllib.parse import urlsplit
 from xml.etree import ElementTree
 from django.conf import settings
@@ -14,6 +14,7 @@ from django.test import TestCase, Client
 from django.utils import timezone
 from background_task.models import Task
 from .models import Source, Media
+from .tasks import cleanup_old_media
 
 
 class FrontEndTestCase(TestCase):
@@ -182,7 +183,8 @@ class FrontEndTestCase(TestCase):
             'source_acodec': 'OPUS',
             'prefer_60fps': False,
             'prefer_hdr': False,
-            'fallback': 'f'
+            'fallback': 'f',
+            'sub_langs': 'en',
         }
         response = c.post('/source-add', data)
         self.assertEqual(response.status_code, 302)
@@ -223,7 +225,8 @@ class FrontEndTestCase(TestCase):
             'source_acodec': Source.SOURCE_ACODEC_OPUS,
             'prefer_60fps': False,
             'prefer_hdr': False,
-            'fallback': Source.FALLBACK_FAIL
+            'fallback': Source.FALLBACK_FAIL,
+            'sub_langs': 'en',
         }
         response = c.post(f'/source-update/{source_uuid}', data)
         self.assertEqual(response.status_code, 302)
@@ -252,7 +255,8 @@ class FrontEndTestCase(TestCase):
             'source_acodec': Source.SOURCE_ACODEC_OPUS,
             'prefer_60fps': False,
             'prefer_hdr': False,
-            'fallback': Source.FALLBACK_FAIL
+            'fallback': Source.FALLBACK_FAIL,
+            'sub_langs': 'en',
         }
         response = c.post(f'/source-update/{source_uuid}', data)
         self.assertEqual(response.status_code, 302)
@@ -464,11 +468,14 @@ metadata_60fps_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_60
 metadata_60fps = open(metadata_60fps_filepath, 'rt').read()
 metadata_60fps_hdr_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_60fps_hdr.json'
 metadata_60fps_hdr = open(metadata_60fps_hdr_filepath, 'rt').read()
+metadata_20230629_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_2023-06-29.json'
+metadata_20230629 = open(metadata_20230629_filepath, 'rt').read()
 all_test_metadata = {
     'boring': metadata,
     'hdr': metadata_hdr,
     '60fps': metadata_60fps,
     '60fps+hdr': metadata_60fps_hdr,
+    '20230629': metadata_20230629,
 }
 
 
@@ -1397,3 +1404,95 @@ class FormatMatchingTestCase(TestCase):
             match_type, format_code = self.media.get_best_video_format()
             self.assertEqual(format_code, expected_format_code)
             self.assertEqual(match_type, expeceted_match_type)
+
+    def test_metadata_20230629(self):
+        self.media.metadata = all_test_metadata['20230629']
+        expected_matches = {
+            # (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
+            ('360p', 'AVC1', False, True): (False, '134'),             # Fallback match, no hdr
+            ('360p', 'AVC1', True, False): (False, '134'),             # Fallback match, no 60fps
+            ('360p', 'AVC1', True, True): (False, '332'),              # Fallback match, 60fps+hdr, switched to VP9
+            ('360p', 'VP9', False, False): (True, '243'),              # Exact match
+            ('360p', 'VP9', False, True): (True, '332'),               # Exact match, hdr
+            ('360p', 'VP9', True, False): (False, '332'),              # Fallback match, 60fps, extra hdr
+            ('360p', 'VP9', True, True): (True, '332'),                # Exact match, 60fps+hdr
+            ('480p', 'AVC1', False, False): (True, '135'),             # Exact match
+            ('480p', 'AVC1', False, True): (False, '135'),             # Fallback match, no hdr
+            ('480p', 'AVC1', True, False): (False, '135'),             # Fallback match, no 60fps
+            ('480p', 'AVC1', True, True): (False, '333'),              # Fallback match, 60fps+hdr, switched to VP9
+            ('480p', 'VP9', False, False): (True, '244'),              # Exact match
+            ('480p', 'VP9', False, True): (True, '333'),               # Exact match, hdr
+            ('480p', 'VP9', True, False): (False, '333'),              # Fallback match, 60fps, extra hdr
+            ('480p', 'VP9', True, True): (True, '333'),                # Exact match, 60fps+hdr
+            ('720p', 'AVC1', False, False): (True, '136'),             # Exact match
+            ('720p', 'AVC1', False, True): (False, '136'),             # Fallback match, no hdr
+            ('720p', 'AVC1', True, False): (True, '298'),              # Exact match, 60fps
+            ('720p', 'AVC1', True, True): (False, '334'),              # Fallback match, 60fps+hdr, switched to VP9
+            ('720p', 'VP9', False, False): (True, '247'),              # Exact match
+            ('720p', 'VP9', False, True): (True, '334'),               # Exact match, hdr
+            ('720p', 'VP9', True, False): (True, '302'),               # Exact match, 60fps
+            ('720p', 'VP9', True, True): (True, '334'),                # Exact match, 60fps+hdr
+            ('1440p', 'AVC1', False, False): (False, '308'),           # Fallback match, 60fps, switched to VP9 (no 1440p AVC1)
+            ('1440p', 'AVC1', False, True): (False, '336'),            # Fallback match, 60fps+hdr, switched to VP9 (no 1440p AVC1)
+            ('1440p', 'AVC1', True, False): (False, '308'),            # Fallback match, 60fps, switched to VP9 (no 1440p AVC1)
+            ('1440p', 'AVC1', True, True): (False, '336'),             # Fallback match, 60fps+hdr, switched to VP9 (no 1440p AVC1)
+            ('1440p', 'VP9', False, False): (False, '308'),            # Fallback, 60fps
+            ('1440p', 'VP9', False, True): (True, '336'),              # Exact match, hdr
+            ('1440p', 'VP9', True, False): (True, '308'),              # Exact match, 60fps
+            ('1440p', 'VP9', True, True): (True, '336'),               # Exact match, 60fps+hdr
+            ('2160p', 'AVC1', False, False): (False, '315'),           # Fallback, 60fps, switched to VP9 (no 2160p AVC1)
+            ('2160p', 'AVC1', False, True): (False, '337'),            # Fallback match, 60fps+hdr, switched to VP9 (no 2160p AVC1)
+            ('2160p', 'AVC1', True, False): (False, '315'),            # Fallback, switched to VP9 (no 2160p AVC1)
+            ('2160p', 'AVC1', True, True): (False, '337'),             # Fallback match, 60fps+hdr, switched to VP9 (no 2160p AVC1)
+            ('2160p', 'VP9', False, False): (False, '315'),            # Fallback, 60fps
+            ('2160p', 'VP9', False, True): (True, '337'),              # Exact match, hdr
+            ('2160p', 'VP9', True, False): (True, '315'),              # Exact match, 60fps
+            ('2160p', 'VP9', True, True): (True, '337'),               # Exact match, 60fps+hdr
+            ('4320p', 'AVC1', False, False): (False, '272'),           # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'AVC1', False, True): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'AVC1', True, False): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'AVC1', True, True): (False, '272'),             # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'VP9', False, False): (False, '272'),            # Fallback, 60fps (no other 8k streams)
+            ('4320p', 'VP9', False, True): (False, '272'),             # Fallback, 60fps (no other 8k streams)
+            ('4320p', 'VP9', True, False): (True, '272'),              # Exact match, 60fps
+            ('4320p', 'VP9', True, True): (False, '272'),              # Fallback, 60fps (no other 8k streams)
+        }
+        for params, expected in expected_matches.items():
+            resolution, vcodec, prefer_60fps, prefer_hdr = params
+            expeceted_match_type, expected_format_code = expected
+            self.source.source_resolution = resolution
+            self.source.source_vcodec = vcodec
+            self.source.prefer_60fps = prefer_60fps
+            self.source.prefer_hdr = prefer_hdr
+            # The aim here is to execute the matching code to find error paths, specific testing isn't required
+            self.media.get_best_video_format()
+            self.media.get_best_audio_format()
+
+
+class TasksTestCase(TestCase):
+    def setUp(self):
+        # Disable general logging for test case
+        logging.disable(logging.CRITICAL)
+
+    def test_delete_old_media(self):
+        src1 = Source.objects.create(key='aaa', name='aaa', directory='/tmp/a', delete_old_media=False, days_to_keep=14)
+        src2 = Source.objects.create(key='bbb', name='bbb', directory='/tmp/b', delete_old_media=True, days_to_keep=14)
+
+        now = timezone.now()
+
+        m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5))
+        m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25))
+        m13 = Media.objects.create(source=src1, downloaded=False, key='a13')
+
+        m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5))
+        m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25))
+        m23 = Media.objects.create(source=src2, downloaded=False, key='a23')
+
+        self.assertEquals(src1.media_source.all().count(), 3)
+        self.assertEquals(src2.media_source.all().count(), 3)
+
+        cleanup_old_media()
+
+        self.assertEquals(src1.media_source.all().count(), 3)
+        self.assertEquals(src2.media_source.all().count(), 2)
+        self.assertEquals(Media.objects.filter(pk=m22.pk).exists(), False)

tubesync/sync/utils.py

@@ -78,7 +78,7 @@ def resize_image_to_height(image, width, height):
     if scaled_width < width:
         # Width too small, stretch it
         scaled_width = width
-    image = image.resize((scaled_width, height), Image.ANTIALIAS)
+    image = image.resize((scaled_width, height), Image.LANCZOS)
     if scaled_width > width:
         # Width too large, crop it
         delta = scaled_width - width

tubesync/sync/views.py

@@ -70,7 +70,7 @@ class DashboardView(TemplateView):
             data['average_bytes_per_media'] = 0
         # Latest downloads
         data['latest_downloads'] = Media.objects.filter(
-            downloaded=True
+            downloaded=True, downloaded_filesize__isnull=False
         ).order_by('-download_date')[:10]
         # Largest downloads
         data['largest_downloads'] = Media.objects.filter(
@@ -297,7 +297,9 @@ class EditSourceMixin:
     fields = ('source_type', 'key', 'name', 'directory', 'media_format',
               'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
               'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
-              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 'write_json')
+              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 
+              'write_json', 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock',
+              'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs')
     errors = {
         'invalid_media_format': _('Invalid media format, the media format contains '
                                   'errors or is empty. Check the table at the end of '

tubesync/sync/youtube.py

@@ -64,13 +64,20 @@ def get_media_info(url):
     return response
 
 
-def download_media(url, media_format, extension, output_file, info_json, sponsor_categories="all"):
+def download_media(url, media_format, extension, output_file, info_json, 
+                   sponsor_categories="all", 
+                   embed_thumbnail=False, embed_metadata=False, skip_sponsors=True, 
+                   write_subtitles=False, auto_subtitles=False, sub_langs='en'):
     '''
         Downloads a YouTube URL to a file on disk.
     '''
 
     def hook(event):
         filename = os.path.basename(event['filename'])
+    
+        if event.get('downloaded_bytes') is None or event.get('total_bytes') is None:
+            return None
+
         if event['status'] == 'error':
             log.error(f'[youtube-dl] error occured downloading: {filename}')
         elif event['status'] == 'downloading':
@@ -100,27 +107,42 @@ def download_media(url, media_format, extension, output_file, info_json, sponsor
         else:
             log.warn(f'[youtube-dl] unknown event: {str(event)}')
     hook.download_progress = 0
-    postprocessors = []
-    postprocessors.append({
-        'key': 'FFmpegMetadata',
-        'add_chapters': True,
-        'add_metadata': True
-    })
-    # Pending configuration options from PR #338
-    #postprocessors.append({
-    #    'key': 'SponsorBlock',
-    #    'categories': [sponsor_categories]
-    #})
-    opts = get_yt_opts()
-    opts.update({
+
+    ytopts = {
         'format': media_format,
         'merge_output_format': extension,
         'outtmpl': output_file,
         'quiet': True,
         'progress_hooks': [hook],
         'writeinfojson': info_json,
-        'postprocessors': postprocessors,
-    })
+        'postprocessors': [],
+        'writesubtitles': write_subtitles,
+        'writeautomaticsub': auto_subtitles,
+        'subtitleslangs': sub_langs.split(','),
+    }
+    
+    sbopt = {
+        'key': 'SponsorBlock',
+        'categories': [sponsor_categories]
+    }
+    ffmdopt = {
+        'key': 'FFmpegMetadata',
+        'add_chapters': True,
+        'add_metadata': True
+    }
+
+    opts = get_yt_opts()
+    if embed_thumbnail:
+        ytopts['postprocessors'].append({'key': 'EmbedThumbnail'})
+    if embed_metadata:
+        ffmdopt["add_metadata"] = True
+    if skip_sponsors:
+        ytopts['postprocessors'].append(sbopt)
+    
+    ytopts['postprocessors'].append(ffmdopt)
+        
+    opts.update(ytopts)
+        
     with yt_dlp.YoutubeDL(opts) as y:
         try:
             return y.download([url])

tubesync/tubesync/local_settings.py.container

@@ -67,6 +67,12 @@ YOUTUBE_DL_CACHEDIR = CONFIG_BASE_DIR / 'cache'
 COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt'
 
 
+HEALTHCHECK_FIREWALL_STR = str(os.getenv('TUBESYNC_HEALTHCHECK_FIREWAL', 'True')).strip().lower()
+HEALTHCHECK_FIREWALL = True if HEALTHCHECK_FIREWALL_STR == 'true' else False
+HEALTHCHECK_ALLOWED_IPS_STR = str(os.getenv('TUBESYNC_HEALTHCHECK_ALLOWED_IPS', '127.0.0.1'))
+HEALTHCHECK_ALLOWED_IPS = HEALTHCHECK_ALLOWED_IPS_STR.split(',')
+
+
 BASICAUTH_USERNAME = os.getenv('HTTP_USER', '').strip()
 BASICAUTH_PASSWORD = os.getenv('HTTP_PASS', '').strip()
 if BASICAUTH_USERNAME and BASICAUTH_PASSWORD:

tubesync/tubesync/settings.py

@@ -6,7 +6,7 @@ CONFIG_BASE_DIR = BASE_DIR
 DOWNLOADS_BASE_DIR = BASE_DIR
 
 
-VERSION = '0.12.1'
+VERSION = '0.13.0'
 SECRET_KEY = ''
 DEBUG = False
 ALLOWED_HOSTS = []