bump ffmpeg to 2023-11-14 and yt-dlp to 2023.11.16

Adds workflow to delete local media that no longer exists in the source

.github/workflows/ci.yaml

@@ -7,9 +7,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    branches:
-      - main
 
 jobs:
   test:

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -130,3 +131,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+Pipfile.lock
+.vscode/launch.json

Dockerfile

@@ -1,16 +1,17 @@
-FROM debian:bullseye-slim
+FROM debian:bookworm-slim
 
 ARG TARGETPLATFORM
-ARG S6_VERSION="3.1.2.1"
-ARG FFMPEG_DATE="autobuild-2023-01-03-12-55"
-ARG FFMPEG_VERSION="109474-gc94988a781"
+ARG S6_VERSION="3.1.5.0"
+ARG FFMPEG_DATE="autobuild-2023-11-14-14-18"
+ARG FFMPEG_VERSION="112750-g6d60cc7baf"
 
 ENV DEBIAN_FRONTEND="noninteractive" \
   HOME="/root" \
   LANGUAGE="en_US.UTF-8" \
   LANG="en_US.UTF-8" \
   LC_ALL="en_US.UTF-8" \
-  TERM="xterm"
+  TERM="xterm" \
+  S6_CMD_WAIT_FOR_SERVICES_MAXTIME="0"
 
 # Install third party software
 RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
@@ -18,22 +19,22 @@ RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
   "linux/arm64")   echo "aarch64" ;; \
   *)               echo ""        ;; esac) && \
   export S6_ARCH_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
-  "linux/amd64")   echo "6019b6b06cfdbb1d1cd572d46b9b158a4904fd19ca59d374de4ddaaa6a3727d5" ;; \
-  "linux/arm64")   echo "e73f9a021b64f88278830742149c14ef8a52331102881ba025bf32a66a0e7c78" ;; \
+  "linux/amd64")   echo "65d0d0f353d2ff9d0af202b268b4bf53a9948a5007650854855c729289085739" ;; \
+  "linux/arm64")   echo "3fbd14201473710a592b2189e81f00f3c8998e96d34f16bd2429c35d1bc36d00" ;; \
   *)               echo ""        ;; esac) && \
   export S6_DOWNLOAD_ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
   "linux/amd64")   echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-x86_64.tar.xz"   ;; \
   "linux/arm64")   echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-aarch64.tar.xz" ;; \
   *)               echo ""        ;; esac) && \
   export FFMPEG_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
-  "linux/amd64")   echo "ed9059668e4a6dac9bde122a775f52ad08cbb90df3658f8c1e328477c13c242e" ;; \
-  "linux/arm64")   echo "dd1375bd351d38ea1cc3efd68a998699366e28bd9b90df65d11af2b9121746b7" ;; \
+  "linux/amd64")   echo "d905684195f16412d8ee4a61a5a32d4bea530b4f93260e800b5a74904f6a1528" ;; \
+  "linux/arm64")   echo "5fdbf8d83d05b39d3e1cd666d485340115bc31cfc686993dcb77f99d1b35751e" ;; \
   *)               echo ""        ;; esac) && \
   export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \
   "linux/amd64")   echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linux64-gpl.tar.xz"   ;; \
   "linux/arm64")   echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linuxarm64-gpl.tar.xz" ;; \
   *)               echo ""        ;; esac) && \
-  export S6_NOARCH_EXPECTED_SHA256="cee89d3eeabdfe15239b2c5c3581d9352d2197d4fd23bba3f1e64bf916ccf496" && \
+  export S6_NOARCH_EXPECTED_SHA256="fd80c231e8ae1a0667b7ae2078b9ad0e1269c4d117bf447a4506815a700dbff3" && \
   export S6_DOWNLOAD_NOARCH="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-noarch.tar.xz" && \
   echo "Building for arch: ${ARCH}|${ARCH44}, downloading S6 from: ${S6_DOWNLOAD}}, expecting S6 SHA256: ${S6_EXPECTED_SHA256}" && \
   set -x && \
@@ -82,30 +83,30 @@ RUN set -x && \
   apt-get -y install nginx-light && \
   apt-get -y --no-install-recommends install \
   python3 \
-  python3-setuptools \
-  python3-pip \
   python3-dev \
+  python3-pip \
+  python3-wheel \
+  pipenv \
   gcc \
   g++ \
   make \
+  pkgconf \
   default-libmysqlclient-dev \
   libmariadb3 \
   postgresql-common \
   libpq-dev \
   libpq5 \
   libjpeg62-turbo \
-  libwebp6 \
+  libwebp7 \
   libjpeg-dev \
   zlib1g-dev \
   libwebp-dev \
   redis-server && \
-  # Install pipenv
-  pip3 --disable-pip-version-check install wheel pipenv && \
   # Create a 'app' user which the application will run as
   groupadd app && \
   useradd -M -d /app -s /bin/false -g app app && \
   # Install non-distro packages
-  pipenv install --system --skip-lock && \
+  PIPENV_VERBOSITY=64 pipenv install --system --skip-lock && \
   # Make absolutely sure we didn't accidentally bundle a SQLite dev database
   rm -rf /app/db.sqlite3 && \
   # Run any required app commands
@@ -119,7 +120,6 @@ RUN set -x && \
   # Clean up
   rm /app/Pipfile && \
   pipenv --clear && \
-  pip3 --disable-pip-version-check uninstall -y pipenv wheel virtualenv && \
   apt-get -y autoremove --purge \
   python3-pip \
   python3-dev \

Makefile

@@ -31,3 +31,7 @@ runcontainer:
 
 test: build
 	cd tubesync && $(python) manage.py test --verbosity=2 && cd ..
+
+
+shell:
+	cd tubesync && $(python) manage.py shell

Pipfile

@@ -4,6 +4,7 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
+autopep8 = "*"
 
 [packages]
 django = "~=3.2"
@@ -15,10 +16,10 @@ gunicorn = "*"
 django-compressor = "*"
 httptools = "*"
 django-background-tasks = "*"
-requests = "*"
 django-basicauth = "*"
 psycopg2-binary = "*"
 mysqlclient = "*"
 yt-dlp = "*"
 redis = "*"
 hiredis = "*"
+requests = {extras = ["socks"], version = "*"}

README.md

@@ -241,6 +241,7 @@ and less common features:
  * [Reset tasks from the command line](https://github.com/meeb/tubesync/blob/main/docs/reset-tasks.md)
  * [Using PostgreSQL, MySQL or MariaDB as database backends](https://github.com/meeb/tubesync/blob/main/docs/other-database-backends.md)
  * [Using cookies](https://github.com/meeb/tubesync/blob/main/docs/using-cookies.md)
+ * [Reset metadata](https://github.com/meeb/tubesync/blob/main/docs/reset-metadata.md)
 
 
 # Warnings
@@ -350,6 +351,10 @@ etc.). Configuration of this is beyond the scope of this README.
 
 Just `amd64` for the moment. Others may be made available if there is demand.
 
+### The pipenv install fails with "Locking failed"!
+
+Make sure that you have `mysql_config` or `mariadb_config` available, as required by the python module `mysqlclient`. On Debian-based systems this is usually found in the package `libmysqlclient-dev`
+
 
 # Advanced configuration
 
@@ -358,12 +363,13 @@ There are a number of other environment variables you can set. These are, mostly
 useful if you are manually installing TubeSync in some other environment. These are:
 
 | Name                        | What                                                         | Example                              |
-| ------------------------ | ------------------------------------------------------------ | ------------------------------------ |
+| --------------------------- | ------------------------------------------------------------ | ------------------------------------ |
 | DJANGO_SECRET_KEY           | Django's SECRET_KEY                                          | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l     |
 | DJANGO_URL_PREFIX           | Run TubeSync in a sub-URL on the web server                  | /somepath/                           |
 | TUBESYNC_DEBUG              | Enable debugging                                             | True                                 |
 | TUBESYNC_WORKERS            | Number of background workers, default is 2, max allowed is 8 | 2                                    |
 | TUBESYNC_HOSTS              | Django's ALLOWED_HOSTS, defaults to `*`                      | tubesync.example.com,otherhost.com   |
+| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True  | True
 | GUNICORN_WORKERS            | Number of gunicorn workers to spawn                          | 3                                    |
 | LISTEN_HOST                 | IP address for gunicorn to listen on                         | 127.0.0.1                            |
 | LISTEN_PORT                 | Port number for gunicorn to listen on                        | 8080                                 |

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run

@@ -11,8 +11,6 @@ chown -R app:app /run/app
 chmod -R 0700 /run/app
 chown -R app:app /config
 chmod -R 0755 /config
-chown -R app:app /downloads
-chmod -R 0755 /downloads
 chown -R root:app /app
 chmod -R 0750 /app
 chown -R app:app /app/common/static
@@ -22,6 +20,15 @@ chmod -R 0750 /app/static
 find /app -type f ! -iname healthcheck.py -exec chmod 640 {} \;
 chmod 0755 /app/healthcheck.py
 
+# Optionally reset the download dir permissions
+TUBESYNC_RESET_DOWNLOAD_DIR="${TUBESYNC_RESET_DOWNLOAD_DIR:-True}"
+if [ "$TUBESYNC_RESET_DOWNLOAD_DIR" == "True" ]
+then
+    echo "TUBESYNC_RESET_DOWNLOAD_DIR=True, Resetting /downloads directory permissions"
+    chown -R app:app /downloads
+    chmod -R 0755 /downloads
+fi
+
 # Run migrations
 exec s6-setuidgid app \
     /usr/bin/python3 /app/manage.py migrate

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/timeout-up

@@ -1 +0,0 @@
-60000

docs/other-database-backends.md

@@ -24,7 +24,7 @@ $ docker exec -i tubesync python3 /app/manage.py dumpdata > some-file.json
 Then change you database backend over, then use
 
 ```bash
-$ cat some-file.json | docker exec -i tubesync python3 /app/manage.py loaddata --format=json -
+$ cat some-file.json | docker exec -i tubesync python3 /app/manage.py loaddata - --format=json
 ```
 
 As detailed in the Django documentation:

docs/reset-metadata.md

@@ -0,0 +1,30 @@
+# TubeSync
+
+## Advanced usage guide - reset media metadata from the command line
+
+This command allows you to reset all media item metadata. You might want to use
+this if you have a lot of media items with invalid metadata and you want to
+wipe it which triggers the metadata to be redownloaded.
+
+
+## Requirements
+
+You have added some sources and media
+
+## Steps
+
+### 1. Run the reset tasks command
+
+Execute the following Django command:
+
+`./manage.py reset-metadata`
+
+When deploying TubeSync inside a container, you can execute this with:
+
+`docker exec -ti tubesync python3 /app/manage.py reset-metadata`
+
+This command will log what its doing to the terminal when you run it.
+
+When this is run, new tasks will be immediately created so all your media
+items will start downloading updated metadata straight away, any missing information
+such as thumbnails will be redownloaded, etc.

tubesync/common/static/styles/_fonts.scss

@@ -1,19 +1,19 @@
 @font-face {
-    font-family: 'roboto-light';
+    font-family: 'roboto';
     src: url('../fonts/roboto/roboto-light.woff') format('woff');
-    font-weight: normal;
+    font-weight: lighter;
     font-style: normal;
 }
 
 @font-face {
-    font-family: 'roboto-regular';
+    font-family: 'roboto';
     src: url('../fonts/roboto/roboto-regular.woff') format('woff');
     font-weight: normal;
     font-style: normal;
 }
 
 @font-face {
-    font-family: 'roboto-bold';
+    font-family: 'roboto';
     src: url('../fonts/roboto/roboto-bold.woff') format('woff');
     font-weight: bold;
     font-style: normal;

tubesync/common/static/styles/_variables.scss

@@ -1,2 +1,2 @@
-$font-family: 'roboto-regular', Arial, Helvetica, sans-serif;
+$font-family: 'roboto', Arial, Helvetica, sans-serif;
 $font-size: 1.05rem;

tubesync/common/static/styles/fontawesome/_icons.scss

@@ -65,6 +65,7 @@ readers do not read off random characters that represent icons */
 .#{$fa-css-prefix}-arrows-alt-h:before { content: fa-content($fa-var-arrows-alt-h); }
 .#{$fa-css-prefix}-arrows-alt-v:before { content: fa-content($fa-var-arrows-alt-v); }
 .#{$fa-css-prefix}-artstation:before { content: fa-content($fa-var-artstation); }
+.#{$fa-css-prefix}-arrow-rotate-right:before { content: fa-content($fa-var-arrow-rotate-right); }
 .#{$fa-css-prefix}-assistive-listening-systems:before { content: fa-content($fa-var-assistive-listening-systems); }
 .#{$fa-css-prefix}-asterisk:before { content: fa-content($fa-var-asterisk); }
 .#{$fa-css-prefix}-asymmetrik:before { content: fa-content($fa-var-asymmetrik); }

tubesync/common/static/styles/fontawesome/_variables.scss

@@ -80,6 +80,7 @@ $fa-var-arrow-right: \f061;
 $fa-var-arrow-up: \f062;
 $fa-var-arrows-alt: \f0b2;
 $fa-var-arrows-alt-h: \f337;
+$fa-var-arrow-rotate-right: \f01e;
 $fa-var-arrows-alt-v: \f338;
 $fa-var-artstation: \f77a;
 $fa-var-assistive-listening-systems: \f2a2;

tubesync/common/static/styles/materializecss/components/forms/_checkboxes.scss

@@ -14,7 +14,7 @@
   // Text Label Style
   + span:not(.lever) {
     position: relative;
-    padding-left: 35px;
+    padding-left: 27px;
     cursor: pointer;
     display: inline-block;
     height: 25px;

tubesync/common/static/styles/tubesync.scss

@@ -17,3 +17,16 @@ html {
     visibility: visible;
     opacity: 1;
 }
+
+.flex-collection-container {
+    display: flex !important;
+    align-items: center;
+}
+
+.flex-grow {
+    flex-grow: 1;
+}
+
+.help-text > i {
+    padding-right: 6px;
+}

tubesync/sync/fields.py

@@ -0,0 +1,109 @@
+from django.forms import MultipleChoiceField,  CheckboxSelectMultiple, Field, TypedMultipleChoiceField
+from django.db import models
+from typing import Any, Optional, Dict
+from django.utils.translation import gettext_lazy as _
+
+# this is a form field!
+class CustomCheckboxSelectMultiple(CheckboxSelectMultiple):
+    template_name = 'widgets/checkbox_select.html'
+    option_template_name = 'widgets/checkbox_option.html'
+
+    def get_context(self, name: str, value: Any, attrs) -> Dict[str, Any]:
+        ctx = super().get_context(name, value, attrs)['widget']
+        ctx["multipleChoiceProperties"] = []
+        for _group, options, _index in ctx["optgroups"]:
+            for option in options:
+                if not isinstance(value,str) and not isinstance(value,list) and ( option["value"] in value.selected_choices or ( value.allow_all and value.all_choice in value.selected_choices ) ):
+                    checked = True
+                else:
+                    checked = False
+
+                ctx["multipleChoiceProperties"].append({
+                    "template_name": option["template_name"],
+                    "type": option["type"],
+                    "value": option["value"],
+                    "label": option["label"],
+                    "name": option["name"],
+                    "checked": checked})
+
+        return { 'widget': ctx }
+
+# this is a database field!
+class CommaSepChoiceField(models.Field):
+    "Implements comma-separated storage of lists"
+
+    def __init__(self, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, *args, **kwargs):
+        self.separator = separator
+        self.possible_choices = possible_choices
+        self.selected_choices = []
+        self.allow_all = allow_all
+        self.all_label = all_label
+        self.all_choice = all_choice
+        super().__init__(*args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        if self.separator != ",":
+            kwargs['separator'] = self.separator
+        kwargs['possible_choices'] = self.possible_choices
+        return name, path, args, kwargs
+
+    def db_type(self, connection):
+        return 'text'
+
+    def get_my_choices(self):
+        choiceArray = []
+        if self.possible_choices is None:
+            return choiceArray
+        if self.allow_all:
+            choiceArray.append((self.all_choice, _(self.all_label)))
+
+        for t in self.possible_choices:
+            choiceArray.append(t)
+        
+        return choiceArray
+
+    def formfield(self, **kwargs):
+        # This is a fairly standard way to set up some defaults
+        # while letting the caller override them.
+        defaults = {'form_class': MultipleChoiceField, 
+                    'choices': self.get_my_choices,
+                    'widget': CustomCheckboxSelectMultiple,
+                    'label': '',
+                    'required': False}
+        defaults.update(kwargs)
+        #del defaults.required
+        return super().formfield(**defaults)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        # Only include kwarg if it's not the default
+        if self.separator != ",":
+            kwargs['separator'] = self.separator
+        return name, path, args, kwargs
+
+    def from_db_value(self, value, expr, conn):
+        if value is None:
+            self.selected_choices = []
+        else:
+            self.selected_choices = value.split(",")
+
+        return self
+
+    def get_prep_value(self, value):
+        if value is None:
+            return ""
+        if not isinstance(value,list):
+            return ""
+
+        if self.all_choice not in value:
+            return ",".join(value)
+        else:
+            return self.all_choice
+
+    def get_text_for_value(self, val):
+        fval = [i for i in self.possible_choices if i[0] == val]
+        if len(fval) <= 0:
+            return []
+        else:
+            return fval[0][1]

tubesync/sync/management/commands/reset-metadata.py

@@ -0,0 +1,19 @@
+from django.core.management.base import BaseCommand
+from sync.models import Media
+
+
+from common.logger import log
+
+
+class Command(BaseCommand):
+
+    help = 'Resets all media item metadata'
+
+    def handle(self, *args, **options):
+        log.info('Resettings all media metadata...')
+        # Delete all metadata
+        Media.objects.update(metadata=None)
+        # Trigger the save signal on each media item
+        for item in Media.objects.all():
+            item.save()
+        log.info('Done')

tubesync/sync/matching.py

@@ -53,6 +53,8 @@ def get_best_audio_format(media):
         # If the format has a video stream, skip it
         if fmt['vcodec'] is not None:
             continue
+        if not fmt['acodec']:
+            continue
         audio_formats.append(fmt)
     audio_formats = list(reversed(sorted(audio_formats, key=lambda k: k['abr'])))
     if not audio_formats:
@@ -88,6 +90,8 @@ def get_best_video_format(media):
         # If the format has an audio stream, skip it
         if fmt['acodec'] is not None:
             continue
+        if not fmt['vcodec']:
+            continue
         if media.source.source_resolution.strip().upper() == fmt['format']:
             video_formats.append(fmt)
     # Check we matched some streams

tubesync/sync/mediaservers.py

@@ -44,7 +44,9 @@ class PlexMediaServer(MediaServer):
              '<p>The <strong>libraries</strong> is a comma-separated list of Plex '
              'library or section IDs, you can find out how to get your library or '
              'section IDs <a href="https://support.plex.tv/articles/201242707-plex-'
-             'media-scanner-via-command-line/#toc-1" target="_blank">here</a>.</p>')
+             'media-scanner-via-command-line/#toc-1" target="_blank">here</a> or '
+             '<a href="https://www.plexopedia.com/plex-media-server/api/server/libraries/" '
+             'target="_blank">here</a></p>.')
 
     def make_request(self, uri='/', params={}):
         headers = {'User-Agent': 'TubeSync'}

tubesync/sync/migrations/0015_auto_20230213_0603.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.17 on 2023-02-13 06:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0014_alter_media_media_file'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='media',
+            name='manual_skip',
+            field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\' be downloaded', verbose_name='manual_skip'),
+        ),
+        migrations.AlterField(
+            model_name='media',
+            name='skip',
+            field=models.BooleanField(db_index=True, default=False, help_text='INTERNAL FLAG - Media will be skipped and not downloaded', verbose_name='skip'),
+        ),
+    ]

tubesync/sync/migrations/0016_auto_20230214_2052.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.2.18 on 2023-02-14 20:52
+
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0015_auto_20230213_0603'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='embed_metadata',
+            field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='embed_thumbnail',
+            field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='enable_sponsorblock',
+            field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='sponsorblock_categories',
+            field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))),
+        ),
+    ]

tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.18 on 2023-02-20 02:23
+
+from django.db import migrations
+import sync.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0016_auto_20230214_2052'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='sponsorblock_categories',
+            field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', separator=''),
+        ),
+    ]

tubesync/sync/migrations/0018_source_subtitles.py

@@ -0,0 +1,27 @@
+# Generated by pac
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0017_alter_source_sponsorblock_categories'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='write_subtitles',
+            field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='auto_subtitles',
+            field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto subtitles'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='sub_langs',
+            field=models.CharField(default='en', help_text='List of subtitles langs to download comma-separated. Example: en,fr',max_length=30),
+        ),
+    ]

tubesync/sync/migrations/0019_add_delete_removed_media.py

@@ -0,0 +1,17 @@
+# Generated by pac
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0018_source_subtitles'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='delete_removed_media',
+            field=models.BooleanField(default=False, help_text='Delete media that is no longer on this playlist', verbose_name='delete removed media'),
+        ),
+    ]

tubesync/sync/models.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from django.conf import settings
 from django.db import models
 from django.core.files.storage import FileSystemStorage
+from django.core.validators import RegexValidator
 from django.utils.text import slugify
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
@@ -19,11 +20,10 @@ from .utils import seconds_to_timestr, parse_media_format
 from .matching import (get_best_combined_format, get_best_audio_format, 
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
-
+from .fields import CommaSepChoiceField
 
 media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/')
 
-
 class Source(models.Model):
     '''
         A Source is a source of media. Currently, this is either a YouTube channel
@@ -106,6 +106,47 @@ class Source(models.Model):
     EXTENSION_MKV = 'mkv'
     EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
 
+
+    # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
+    SPONSORBLOCK_CATEGORIES_CHOICES = (
+        ('sponsor', 'Sponsor'),
+        ('intro', 'Intermission/Intro Animation'),
+        ('outro', 'Endcards/Credits'),
+        ('selfpromo', 'Unpaid/Self Promotion'),
+        ('preview', 'Preview/Recap'),
+        ('filler', 'Filler Tangent'),
+        ('interaction', 'Interaction Reminder'),
+        ('music_offtopic', 'Non-Music Section'),
+    )
+    
+    sponsorblock_categories = CommaSepChoiceField(
+            _(''),
+            possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
+            all_choice="all",
+            allow_all=True,
+            all_label="(all options)",
+            default="all",
+            help_text=_("Select the sponsorblocks you want to enforce")
+        )
+
+    embed_metadata = models.BooleanField(
+        _('embed metadata'),
+        default=False,
+        help_text=_('Embed metadata from source into file')
+    )
+    embed_thumbnail = models.BooleanField(
+        _('embed thumbnail'),
+        default=False,
+        help_text=_('Embed thumbnail into the file')
+    )
+
+    enable_sponsorblock = models.BooleanField(
+        _('enable sponsorblock'),
+        default=True,
+        help_text=_('Use SponsorBlock?')
+    )
+
+
     # Fontawesome icons used for the source on the front end
     ICONS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
@@ -246,6 +287,11 @@ class Source(models.Model):
         help_text=_('If "delete old media" is ticked, the number of days after which '
                     'to automatically delete media')
     )
+    delete_removed_media = models.BooleanField(
+        _('delete removed media'),
+        default=False,
+        help_text=_('Delete media that is no longer on this playlist')
+    )
     source_resolution = models.CharField(
         _('source resolution'),
         max_length=8,
@@ -309,6 +355,30 @@ class Source(models.Model):
         help_text=_('Source has failed to index media')
     )
 
+    write_subtitles = models.BooleanField(
+        _('write subtitles'),
+        default=False,
+        help_text=_('Download video subtitles')
+    )
+
+    auto_subtitles = models.BooleanField(
+        _('accept auto-generated subs'),
+        default=False,
+        help_text=_('Accept auto-generated subtitles')
+    )
+    sub_langs = models.CharField(
+        _('subs langs'),
+        max_length=30,
+        default='en',
+        help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'),
+        validators=[
+            RegexValidator(
+                regex=r"^(\-?[\_\.a-zA-Z]+,)*(\-?[\_\.a-zA-Z]+){1}$",
+                message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat')
+            )
+        ]
+    )
+
     def __str__(self):
         return self.name
 
@@ -446,6 +516,7 @@ class Source(models.Model):
             'dd': now.strftime('%d'),
             'source': self.slugname,
             'source_full': self.name,
+            'uploader': 'Some Channel Name',
             'title': 'some-media-title-name',
             'title_full': 'Some Media Title Name',
             'key': 'SoMeUnIqUiD',
@@ -679,7 +750,13 @@ class Media(models.Model):
         _('skip'),
         db_index=True,
         default=False,
-        help_text=_('Media will be skipped and not downloaded')
+        help_text=_('INTERNAL FLAG - Media will be skipped and not downloaded')
+    )
+    manual_skip = models.BooleanField(
+        _('manual_skip'),
+        db_index=True,
+        default=False,
+        help_text=_('Media marked as "skipped", won\' be downloaded')
     )
     downloaded = models.BooleanField(
         _('downloaded'),
@@ -949,6 +1026,7 @@ class Media(models.Model):
             'acodec': display_format['acodec'],
             'fps': display_format['fps'],
             'hdr': display_format['hdr'],
+            'uploader': self.uploader,
         }
 
     @property
@@ -1013,7 +1091,7 @@ class Media(models.Model):
         duration = self.loaded_metadata.get(field, 0)
         try:
             duration = int(duration)
-        except ValueError:
+        except (TypeError, ValueError):
             duration = 0
         return duration
 
@@ -1136,6 +1214,31 @@ class Media(models.Model):
             return False
         return os.path.exists(self.media_file.path)
 
+    @property
+    def content_type(self):
+        if not self.downloaded:
+            return 'video/mp4'
+        vcodec = self.downloaded_video_codec
+        if vcodec is None:
+            acodec = self.downloaded_audio_codec
+            if acodec is None:
+                raise TypeError() # nothing here.
+            
+            acodec = acodec.lower()
+            if acodec == "mp4a":
+                return "audio/mp4"
+            elif acodec == "opus":
+                return "audio/opus"
+            else:
+                # fall-fall-back.
+                return 'audio/ogg'
+
+        vcodec = vcodec.lower()
+        if vcodec == 'vp9':
+            return 'video/webm'
+        else:
+            return 'video/mp4'
+
     @property
     def nfoxml(self):
         '''
@@ -1260,7 +1363,10 @@ class Media(models.Model):
                                     f'no valid format available')
         # Download the media with youtube-dl
         download_youtube_media(self.url, format_str, self.source.extension,
-                               str(self.filepath), self.source.write_json)
+                               str(self.filepath), self.source.write_json, 
+                               self.source.sponsorblock_categories, self.source.embed_thumbnail,
+                               self.source.embed_metadata, self.source.enable_sponsorblock,
+                              self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs )
         # Return the download paramaters
         return format_str, self.source.extension
 

tubesync/sync/signals.py

@@ -93,6 +93,10 @@ def task_task_failed(sender, task_id, completed_task, **kwargs):
 
 @receiver(post_save, sender=Media)
 def media_post_save(sender, instance, created, **kwargs):
+    # If the media is skipped manually, bail.
+    if instance.manual_skip:
+        return
+
     # Triggered after media is saved
     cap_changed = False
     can_download_changed = False
@@ -152,7 +156,7 @@ def media_post_save(sender, instance, created, **kwargs):
         verbose_name = _('Downloading metadata for "{}"')
         download_media_metadata(
             str(instance.pk),
-            priority=10,
+            priority=5,
             verbose_name=verbose_name.format(instance.pk),
             remove_existing_tasks=True
         )

tubesync/sync/tasks.py

@@ -132,18 +132,25 @@ def cleanup_completed_tasks():
 
 
 def cleanup_old_media():
-    for media in Media.objects.filter(download_date__isnull=False):
-        if media.source.delete_old_media and media.source.days_to_keep > 0:
-            delta = timezone.now() - timedelta(days=media.source.days_to_keep)
-            if media.downloaded and media.download_date < delta:
-                # Media was downloaded after the cutoff date, delete it
-                log.info(f'Deleting expired media: {media.source} / {media} '
-                         f'(now older than {media.source.days_to_keep} days / '
+    for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0):
+        delta = timezone.now() - timedelta(days=source.days_to_keep)
+        for media in source.media_source.filter(downloaded=True, download_date__lt=delta):
+            log.info(f'Deleting expired media: {source} / {media} '
+                     f'(now older than {source.days_to_keep} days / '
                      f'download_date before {delta})')
             # .delete() also triggers a pre_delete signal that removes the files
             media.delete()
 
 
+def cleanup_removed_media(source, videos):
+    media_objects = Media.objects.filter(source=source, downloaded=True)
+    for item in media_objects:
+        matching_source_item = [video['id'] for video in videos if video['id'] == item.key]
+        if not matching_source_item:
+            log.info(f'{item.title} is no longer in source, removing')
+            item.delete()
+
+
 @background(schedule=0)
 def index_source_task(source_id):
     '''
@@ -153,7 +160,6 @@ def index_source_task(source_id):
         source = Source.objects.get(pk=source_id)
     except Source.DoesNotExist:
         # Task triggered but the Source has been deleted, delete the task
-        delete_index_source_task(source_id)
         return
     # Reset any errors
     source.has_failed = False
@@ -189,6 +195,9 @@ def index_source_task(source_id):
     cleanup_completed_tasks()
     # Tack on a cleanup of old media
     cleanup_old_media()
+    if source.delete_removed_media:
+        log.info(f'Cleaning up media no longer in source {source}')
+        cleanup_removed_media(source, videos)
 
 
 @background(schedule=0)
@@ -202,7 +211,6 @@ def check_source_directory_exists(source_id):
         source = Source.objects.get(pk=source_id)
     except Source.DoesNotExist:
         # Task triggered but the Source has been deleted, delete the task
-        delete_index_source_task(source_id)
         return
     # Check the source output directory exists
     if not source.directory_exists():
@@ -223,6 +231,11 @@ def download_media_metadata(media_id):
         log.error(f'Task download_media_metadata(pk={media_id}) called but no '
                   f'media exists with ID: {media_id}')
         return
+    
+    if media.manual_skip:
+        log.info(f'Task for ID: {media_id} skipped, due to task being manually skipped.')
+        return
+
     source = media.source
     metadata = media.index_metadata()
     media.metadata = json.dumps(metadata, default=json_serial)

tubesync/sync/templates/sync/_mediaformatvars.html

@@ -43,6 +43,11 @@
       <td>Full source name</td>
       <td>My Source</td>
     </tr>
+    <tr>
+      <td>{uploader}</td>
+      <td>Uploader name</td>
+      <td>Some Channel Name</td>
+    </tr>
     <tr>
       <td>{title}</td>
       <td>Lower case media title, max 80 chars</td>

tubesync/sync/templates/sync/media-item.html

@@ -10,15 +10,23 @@
     <p class="truncate"><strong><a href="{{ media.url }}" target="_blank"><i class="fas fa-link"></i> {{ media.url }}</a></strong></p>
     <p class="truncate">Downloading to: <strong>{{ media.source.directory_path }}</strong></p>
     {% if download_state == 'downloaded' %}
+    {% if media.source.is_audio %}
+    <audio controls src="{% url 'sync:media-content' pk=media.pk %}"></audio>
+    {% else %}
     <video controls style="width: 100%">
       <source src="{% url 'sync:media-content' pk=media.pk %}">
     </video>
-    <p class="truncate"><a href="{% url 'sync:media-content' pk=media.pk %}" download="{{ media.filename }}">Download</a></p>
+    {% endif %}
+
+    <p class="truncate"><a href="{% url 'sync:media-content' pk=media.pk %}" download="{{ media.filename }}"><strong><i class="fas fa-download"></i> Download</strong></a></p>
     {% endif %}
   </div>
 </div>
+{% if media.manual_skip %}{% include 'errorbox.html' with message='Media is marked to be skipped and will not be downloaded.' %}
+{% else %}
   {% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
-{% if media.skip %}{% include 'errorbox.html' with message='Media is marked to be skipped and will not be downloaded.' %}{% endif %}
+  {% if media.skip %}{% include 'errorbox.html' with message='This media may be skipped due to error(s).' %}{% endif %}
+{% endif %}
 {% include 'infobox.html' with message=message %}
 <div class="row">
   <div class="col s12 m7">
@@ -162,10 +170,10 @@
 {% else %}
 <div class="row">
   <div class="col s12">
-    {% if media.skip %}
-      <a href="{% url 'sync:enable-media' pk=media.pk %}" class="btn">Enable (unskip) media <i class="fas fa-cloud-download-alt"></i></a>
+    {% if media.manual_skip %}
+      <a href="{% url 'sync:enable-media' pk=media.pk %}" class="btn">Unskip media (manually) <i class="fas fa-cloud-download-alt"></i></a>
     {% else %}
-      <a href="{% url 'sync:skip-media' pk=media.pk %}" class="btn delete-button">Skip media <i class="fas fa-times-circle"></i></a>
+      <a href="{% url 'sync:skip-media' pk=media.pk %}" class="btn delete-button">Manually mark media to be skipped <i class="fas fa-times-circle"></i></a>
     {% endif %}
   </div>
 </div>

tubesync/sync/templates/sync/media.html

@@ -36,8 +36,10 @@
             {% if m.downloaded %}
               <i class="fas fa-check-circle" title="Downloaded"></i> {{ m.download_date|date:'Y-m-d' }}
             {% else %}
-              {% if m.skip %}
-              <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped</span>
+              {% if m.manual_skip %}
+              <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Manually skipped</span>
+              {% elif m.skip %}
+              <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped by system</span>
               {% elif not m.source.download_media %}
               <span class="error-text"><i class="fas fa-times" title="Not downloading media for this source"></i> Disabled at source</span>
               {% elif not m.has_metadata %}

tubesync/sync/templates/sync/source.html

@@ -115,6 +115,10 @@
         <td class="hide-on-small-only">Write JSON?</td>
         <td><span class="hide-on-med-and-up">Write JSON?<br></span><strong>{% if source.write_json %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
+      <tr title="Delete media that is no longer on this playlist?">
+        <td class="hide-on-small-only">Delete removed media</td>
+        <td><span class="hide-on-med-and-up">Delete removed media<br></span><strong>{% if source.delete_removed_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
       {% if source.delete_old_media and source.days_to_keep > 0 %}
       <tr title="Days after which your media from this source will be locally deleted">
         <td class="hide-on-small-only">Delete old media</td>
@@ -130,6 +134,55 @@
         <td class="hide-on-small-only">UUID</td>
         <td><span class="hide-on-med-and-up">UUID<br></span><strong>{{ source.uuid }}</strong></td>
       </tr>
+      <tr title="{{ _('Embedding thumbnail?') }}">
+        <td class="hide-on-small-only">{{ _("Embed thumbnail?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Embed thumbnail?") }}<br></span><strong><i class="fas {% if source.embed_thumbnail %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+      <tr title="{{ _('Embedding metadata?') }}">
+        <td class="hide-on-small-only">{{ _("Embed metadata?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Embed metadata?") }}<br></span><strong><i class="fas {% if source.embed_metadata %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+
+      <tr title="{{ _('Is sponsorblock enabled?') }}">
+        <td class="hide-on-small-only">{{ _("SponsorBlock?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Sponsorblock enabled?") }}<br></span><strong><i class="fas {% if source.enable_sponsorblock %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+
+      {% if source.enable_sponsorblock %}
+      <tr title="{{ _('SponsorBlock: What to block?') }}">
+        <td class="hide-on-small-only">{{ _("What blocked?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("What blocked?") }}<br></span><strong>
+          {% if source.sponsorblock_categories.all_choice in source.sponsorblock_categories.selected_choices %}
+            {% for k,v in source.sponsorblock_categories.possible_choices %}
+              {{ v }}: <i class="fas fa-check"></i><BR>
+            {% endfor %}
+          {% else %}
+            {% for c in source.sponsorblock_categories.selected_choices %} 
+              {% for k,v in source.sponsorblock_categories.possible_choices %}
+                {% if k == c %} {{ v }}: <i class="fas fa-check"></i><BR>{% endif %}
+              {% endfor %}
+            {% endfor %}
+          {% endif %}
+        </strong></td>
+      </tr>
+      {% endif %}
+
+      <tr title="{{ _('Are Subtitles downloaded?') }}">
+        <td class="hide-on-small-only">{{ _("Download subtitles?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Download subtitles?") }}:</span><strong><i class="fas {% if source.write_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+
+      {% if source.write_subtitles %}
+      <tr title="{{ _('Are auto subs accepted?') }}">
+        <td class="hide-on-small-only">{{ _("Auto-generated subtitles?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Auto-generated subtitles?") }}:</span><strong><i class="fas {% if source.auto_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+      <tr title="{{ _('Subs langs?') }}">
+        <td class="hide-on-small-only">{{ _("Subs langs?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
+      </tr>
+      {% endif %}
+      
     </table>
   </div>
 </div>

tubesync/sync/templates/sync/sources.html

@@ -24,7 +24,8 @@
   <div class="col s12">
     <div class="collection">
     {% for source in sources %}
-      <a href="{% url 'sync:source' pk=source.pk %}" class="collection-item">
+      <span class="collection-item flex-collection-container">
+        <a href="{% url 'sync:source' pk=source.pk %}" class="flex-grow">
           {{ source.icon|safe }} <strong>{{ source.name }}</strong> ({{ source.get_source_type_display }} &quot;{{ source.key }}&quot;)<br>
           {{ source.format_summary }}<br>
           {% if source.has_failed %}
@@ -33,6 +34,8 @@
           <strong>{{ source.media_count }}</strong> media items, <strong>{{ source.downloaded_count }}</strong> downloaded{% if source.delete_old_media and source.days_to_keep > 0 %}, keeping {{ source.days_to_keep }} days of media{% endif %}
           {% endif %}
         </a>
+        <a href="{% url 'sync:source-sync-now' pk=source.pk %}" class="collection-item"><i class="fas fa-arrow-rotate-right"></i></a>
+      </span>
     {% empty %}
       <span class="collection-item no-items"><i class="fas fa-info-circle"></i> You haven't added any sources.</span>
     {% endfor %}

tubesync/sync/templates/sync/tasks.html

@@ -66,7 +66,7 @@
       {% for task in scheduled %}
         <a href="{% url task.url pk=task.instance.pk %}" class="collection-item">
           <i class="fas fa-stopwatch"></i> <strong>{{ task }}</strong><br>
-          {% if task.instance.index_schedule %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
+          {% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
           <i class="fas fa-redo"></i> Task will run {% if task.run_now %}<strong>immediately</strong>{% else %}at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>{% endif %}
         </a>
       {% empty %}

tubesync/sync/templates/widgets/checkbox_option.html

@@ -0,0 +1,7 @@
+<!--<input type="{{ option.type }}" name="{{ option.name }}" value="{{ option.value }}" id="{{ option.value }}"><BR>
+    <label for="{{ option.value }}">{{option.label}}</label>-->
+
+<label>
+    <input type="{{ option.type }}" name="{{ option.name }}" value="{{ option.value }}" id="{{ option.value }}" {% if option.checked %}checked{% endif %}>
+    <span>{{option.label}}</span>
+</label>

tubesync/sync/templates/widgets/checkbox_select.html

@@ -0,0 +1,5 @@
+</label>
+{% for option in widget.multipleChoiceProperties %}
+    {% include option.template_name with option=option %}
+{% endfor %}
+<label>

tubesync/sync/tests.py

@@ -6,7 +6,7 @@
 
 
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
 from urllib.parse import urlsplit
 from xml.etree import ElementTree
 from django.conf import settings
@@ -14,6 +14,7 @@ from django.test import TestCase, Client
 from django.utils import timezone
 from background_task.models import Task
 from .models import Source, Media
+from .tasks import cleanup_old_media
 
 
 class FrontEndTestCase(TestCase):
@@ -182,7 +183,8 @@ class FrontEndTestCase(TestCase):
             'source_acodec': 'OPUS',
             'prefer_60fps': False,
             'prefer_hdr': False,
-            'fallback': 'f'
+            'fallback': 'f',
+            'sub_langs': 'en',
         }
         response = c.post('/source-add', data)
         self.assertEqual(response.status_code, 302)
@@ -223,7 +225,8 @@ class FrontEndTestCase(TestCase):
             'source_acodec': Source.SOURCE_ACODEC_OPUS,
             'prefer_60fps': False,
             'prefer_hdr': False,
-            'fallback': Source.FALLBACK_FAIL
+            'fallback': Source.FALLBACK_FAIL,
+            'sub_langs': 'en',
         }
         response = c.post(f'/source-update/{source_uuid}', data)
         self.assertEqual(response.status_code, 302)
@@ -252,7 +255,8 @@ class FrontEndTestCase(TestCase):
             'source_acodec': Source.SOURCE_ACODEC_OPUS,
             'prefer_60fps': False,
             'prefer_hdr': False,
-            'fallback': Source.FALLBACK_FAIL
+            'fallback': Source.FALLBACK_FAIL,
+            'sub_langs': 'en',
         }
         response = c.post(f'/source-update/{source_uuid}', data)
         self.assertEqual(response.status_code, 302)
@@ -464,11 +468,14 @@ metadata_60fps_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_60
 metadata_60fps = open(metadata_60fps_filepath, 'rt').read()
 metadata_60fps_hdr_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_60fps_hdr.json'
 metadata_60fps_hdr = open(metadata_60fps_hdr_filepath, 'rt').read()
+metadata_20230629_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_2023-06-29.json'
+metadata_20230629 = open(metadata_20230629_filepath, 'rt').read()
 all_test_metadata = {
     'boring': metadata,
     'hdr': metadata_hdr,
     '60fps': metadata_60fps,
     '60fps+hdr': metadata_60fps_hdr,
+    '20230629': metadata_20230629,
 }
 
 
@@ -1397,3 +1404,95 @@ class FormatMatchingTestCase(TestCase):
             match_type, format_code = self.media.get_best_video_format()
             self.assertEqual(format_code, expected_format_code)
             self.assertEqual(match_type, expeceted_match_type)
+
+    def test_metadata_20230629(self):
+        self.media.metadata = all_test_metadata['20230629']
+        expected_matches = {
+            # (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
+            ('360p', 'AVC1', False, True): (False, '134'),             # Fallback match, no hdr
+            ('360p', 'AVC1', True, False): (False, '134'),             # Fallback match, no 60fps
+            ('360p', 'AVC1', True, True): (False, '332'),              # Fallback match, 60fps+hdr, switched to VP9
+            ('360p', 'VP9', False, False): (True, '243'),              # Exact match
+            ('360p', 'VP9', False, True): (True, '332'),               # Exact match, hdr
+            ('360p', 'VP9', True, False): (False, '332'),              # Fallback match, 60fps, extra hdr
+            ('360p', 'VP9', True, True): (True, '332'),                # Exact match, 60fps+hdr
+            ('480p', 'AVC1', False, False): (True, '135'),             # Exact match
+            ('480p', 'AVC1', False, True): (False, '135'),             # Fallback match, no hdr
+            ('480p', 'AVC1', True, False): (False, '135'),             # Fallback match, no 60fps
+            ('480p', 'AVC1', True, True): (False, '333'),              # Fallback match, 60fps+hdr, switched to VP9
+            ('480p', 'VP9', False, False): (True, '244'),              # Exact match
+            ('480p', 'VP9', False, True): (True, '333'),               # Exact match, hdr
+            ('480p', 'VP9', True, False): (False, '333'),              # Fallback match, 60fps, extra hdr
+            ('480p', 'VP9', True, True): (True, '333'),                # Exact match, 60fps+hdr
+            ('720p', 'AVC1', False, False): (True, '136'),             # Exact match
+            ('720p', 'AVC1', False, True): (False, '136'),             # Fallback match, no hdr
+            ('720p', 'AVC1', True, False): (True, '298'),              # Exact match, 60fps
+            ('720p', 'AVC1', True, True): (False, '334'),              # Fallback match, 60fps+hdr, switched to VP9
+            ('720p', 'VP9', False, False): (True, '247'),              # Exact match
+            ('720p', 'VP9', False, True): (True, '334'),               # Exact match, hdr
+            ('720p', 'VP9', True, False): (True, '302'),               # Exact match, 60fps
+            ('720p', 'VP9', True, True): (True, '334'),                # Exact match, 60fps+hdr
+            ('1440p', 'AVC1', False, False): (False, '308'),           # Fallback match, 60fps, switched to VP9 (no 1440p AVC1)
+            ('1440p', 'AVC1', False, True): (False, '336'),            # Fallback match, 60fps+hdr, switched to VP9 (no 1440p AVC1)
+            ('1440p', 'AVC1', True, False): (False, '308'),            # Fallback match, 60fps, switched to VP9 (no 1440p AVC1)
+            ('1440p', 'AVC1', True, True): (False, '336'),             # Fallback match, 60fps+hdr, switched to VP9 (no 1440p AVC1)
+            ('1440p', 'VP9', False, False): (False, '308'),            # Fallback, 60fps
+            ('1440p', 'VP9', False, True): (True, '336'),              # Exact match, hdr
+            ('1440p', 'VP9', True, False): (True, '308'),              # Exact match, 60fps
+            ('1440p', 'VP9', True, True): (True, '336'),               # Exact match, 60fps+hdr
+            ('2160p', 'AVC1', False, False): (False, '315'),           # Fallback, 60fps, switched to VP9 (no 2160p AVC1)
+            ('2160p', 'AVC1', False, True): (False, '337'),            # Fallback match, 60fps+hdr, switched to VP9 (no 2160p AVC1)
+            ('2160p', 'AVC1', True, False): (False, '315'),            # Fallback, switched to VP9 (no 2160p AVC1)
+            ('2160p', 'AVC1', True, True): (False, '337'),             # Fallback match, 60fps+hdr, switched to VP9 (no 2160p AVC1)
+            ('2160p', 'VP9', False, False): (False, '315'),            # Fallback, 60fps
+            ('2160p', 'VP9', False, True): (True, '337'),              # Exact match, hdr
+            ('2160p', 'VP9', True, False): (True, '315'),              # Exact match, 60fps
+            ('2160p', 'VP9', True, True): (True, '337'),               # Exact match, 60fps+hdr
+            ('4320p', 'AVC1', False, False): (False, '272'),           # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'AVC1', False, True): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'AVC1', True, False): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'AVC1', True, True): (False, '272'),             # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'VP9', False, False): (False, '272'),            # Fallback, 60fps (no other 8k streams)
+            ('4320p', 'VP9', False, True): (False, '272'),             # Fallback, 60fps (no other 8k streams)
+            ('4320p', 'VP9', True, False): (True, '272'),              # Exact match, 60fps
+            ('4320p', 'VP9', True, True): (False, '272'),              # Fallback, 60fps (no other 8k streams)
+        }
+        for params, expected in expected_matches.items():
+            resolution, vcodec, prefer_60fps, prefer_hdr = params
+            expeceted_match_type, expected_format_code = expected
+            self.source.source_resolution = resolution
+            self.source.source_vcodec = vcodec
+            self.source.prefer_60fps = prefer_60fps
+            self.source.prefer_hdr = prefer_hdr
+            # The aim here is to execute the matching code to find error paths, specific testing isn't required
+            self.media.get_best_video_format()
+            self.media.get_best_audio_format()
+
+
+class TasksTestCase(TestCase):
+    def setUp(self):
+        # Disable general logging for test case
+        logging.disable(logging.CRITICAL)
+
+    def test_delete_old_media(self):
+        src1 = Source.objects.create(key='aaa', name='aaa', directory='/tmp/a', delete_old_media=False, days_to_keep=14)
+        src2 = Source.objects.create(key='bbb', name='bbb', directory='/tmp/b', delete_old_media=True, days_to_keep=14)
+
+        now = timezone.now()
+
+        m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5))
+        m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25))
+        m13 = Media.objects.create(source=src1, downloaded=False, key='a13')
+
+        m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5))
+        m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25))
+        m23 = Media.objects.create(source=src2, downloaded=False, key='a23')
+
+        self.assertEquals(src1.media_source.all().count(), 3)
+        self.assertEquals(src2.media_source.all().count(), 3)
+
+        cleanup_old_media()
+
+        self.assertEquals(src1.media_source.all().count(), 3)
+        self.assertEquals(src2.media_source.all().count(), 2)
+        self.assertEquals(Media.objects.filter(pk=m22.pk).exists(), False)

tubesync/sync/urls.py

@@ -28,6 +28,10 @@ urlpatterns = [
          ValidateSourceView.as_view(),
          name='validate-source'),
 
+    path('source-sync-now/<uuid:pk>',
+         SourcesView.as_view(),
+         name='source-sync-now'),
+
     path('source-add',
          AddSourceView.as_view(),
          name='add-source'),

tubesync/sync/utils.py

@@ -78,7 +78,7 @@ def resize_image_to_height(image, width, height):
     if scaled_width < width:
         # Width too small, stretch it
         scaled_width = width
-    image = image.resize((scaled_width, height), Image.ANTIALIAS)
+    image = image.resize((scaled_width, height), Image.LANCZOS)
     if scaled_width > width:
         # Width too large, crop it
         delta = scaled_width - width

tubesync/sync/views.py

@@ -1,18 +1,22 @@
 import os
 import json
 from base64 import b64decode
+import pathlib
+import sys
 from django.conf import settings
-from django.http import Http404
+from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect
 from django.views.generic import TemplateView, ListView, DetailView
 from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateView,
                                        DeleteView)
 from django.views.generic.detail import SingleObjectMixin
+from django.core.exceptions import SuspiciousFileOperation
 from django.http import HttpResponse
 from django.urls import reverse_lazy
 from django.db import IntegrityError
 from django.db.models import Q, Count, Sum, When, Case
-from django.forms import ValidationError
+from django.forms import Form, ValidationError
 from django.utils.text import slugify
+from django.utils._os import safe_join
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from common.utils import append_uri_params
@@ -66,7 +70,7 @@ class DashboardView(TemplateView):
             data['average_bytes_per_media'] = 0
         # Latest downloads
         data['latest_downloads'] = Media.objects.filter(
-            downloaded=True
+            downloaded=True, downloaded_filesize__isnull=False
         ).order_by('-download_date')[:10]
         # Largest downloads
         data['largest_downloads'] = Media.objects.filter(
@@ -92,8 +96,27 @@ class SourcesView(ListView):
     paginate_by = settings.SOURCES_PER_PAGE
     messages = {
         'source-deleted': _('Your selected source has been deleted.'),
+        'source-refreshed': _('The source has been scheduled to be synced now.')
     }
 
+    def get(self, *args, **kwargs):
+        if args[0].path.startswith("/source-sync-now/"):
+            sobj = Source.objects.get(pk=kwargs["pk"])
+            if sobj is None:
+                return HttpResponseNotFound()
+            
+            verbose_name = _('Index media from source "{}" once')
+            index_source_task(
+                str(sobj.pk),
+                queue=str(sobj.pk),
+                repeat=0,
+                verbose_name=verbose_name.format(sobj.name))
+            url = reverse_lazy('sync:sources')
+            url = append_uri_params(url, {'message': 'source-refreshed'})
+            return HttpResponseRedirect(url)
+        else:
+            return super().get(self, *args, **kwargs)    
+
     def __init__(self, *args, **kwargs):
         self.message = None
         super().__init__(*args, **kwargs)
@@ -273,26 +296,44 @@ class EditSourceMixin:
     model = Source
     fields = ('source_type', 'key', 'name', 'directory', 'media_format',
               'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
-              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
-              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 'write_json')
+              'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
+              'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails',
+              'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
+              'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
+              'auto_subtitles', 'sub_langs')
     errors = {
         'invalid_media_format': _('Invalid media format, the media format contains '
                                   'errors or is empty. Check the table at the end of '
                                   'this page for valid media name variables'),
+        'dir_outside_dlroot': _('You cannot specify a directory outside of the '
+                                'base directory (%BASEDIR%)')
     }
 
-    def form_valid(self, form):
+    def form_valid(self, form: Form):
         # Perform extra validation to make sure the media_format is valid
         obj = form.save(commit=False)
         source_type = form.cleaned_data['media_format']
         example_media_file = obj.get_example_media_format()
+        
         if example_media_file == '':
             form.add_error(
                 'media_format',
                 ValidationError(self.errors['invalid_media_format'])
             )
+
+        # Check for suspicious file path(s)
+        try:
+            targetCheck = form.cleaned_data['directory']+"/.virt"
+            newdir = safe_join(settings.DOWNLOAD_ROOT,targetCheck)
+        except SuspiciousFileOperation:
+            form.add_error(
+                'directory',
+                ValidationError(self.errors['dir_outside_dlroot'].replace("%BASEDIR%",str(settings.DOWNLOAD_ROOT)))
+            )
+
         if form.errors:
             return super().form_invalid(form)
+
         return super().form_valid(form)
 
 
@@ -448,16 +489,16 @@ class MediaView(ListView):
             if self.show_skipped:
                 q = Media.objects.filter(source=self.filter_source)
             elif self.only_skipped:
-                q = Media.objects.filter(source=self.filter_source, skip=True)
+                q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=True) | Q(manual_skip=True)))
             else:
-                q = Media.objects.filter(source=self.filter_source, skip=False)
+                q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=False) & Q(manual_skip=False)))
         else:
             if self.show_skipped:
                 q = Media.objects.all()
             elif self.only_skipped:
-                q = Media.objects.filter(skip=True)
+                q = Media.objects.filter(Q(skip=True)|Q(manual_skip=True))
             else:
-                q = Media.objects.filter(skip=False)
+                q = Media.objects.filter(Q(skip=False)&Q(manual_skip=False))
         return q.order_by('-published', '-created')
 
     def get_context_data(self, *args, **kwargs):
@@ -629,6 +670,7 @@ class MediaSkipView(FormView, SingleObjectMixin):
         self.object.downloaded_filesize = None
         # Mark it to be skipped
         self.object.skip = True
+        self.object.manual_skip = True
         self.object.save()
         return super().form_valid(form)
 
@@ -657,6 +699,7 @@ class MediaEnableView(FormView, SingleObjectMixin):
     def form_valid(self, form):
         # Mark it as not skipped
         self.object.skip = False
+        self.object.manual_skip = False
         self.object.save()
         return super().form_valid(form)
 
@@ -677,8 +720,35 @@ class MediaContent(DetailView):
 
     def dispatch(self, request, *args, **kwargs):
         self.object = self.get_object()
+        # development direct file stream - DO NOT USE PRODUCTIVLY
+        if settings.DEBUG and 'runserver' in sys.argv:
+            # get media URL
+            pth = self.object.media_file.url
+            # remove "/media-data/"
+            pth = pth.split("/media-data/",1)[1]
+            # remove "/" (incase of absolute path)
+            pth = pth.split(str(settings.DOWNLOAD_ROOT).lstrip("/"),1)
 
+            # if we do not have a "/" at the beginning, it is not a absolute path...
+            if len(pth) > 1:
+                pth = pth[1]
+            else:
+                pth = pth[0]
+            
+            
+            # build final path
+            filepth = pathlib.Path(str(settings.DOWNLOAD_ROOT) + pth)
+            
+            if filepth.exists():
+                # return file
+                response = FileResponse(open(filepth,'rb'))
+                return response
+            else:
+                return HttpResponseNotFound()
+            
+        else:
             headers = {
+                'Content-Type': self.object.content_type,
                 'X-Accel-Redirect': self.object.media_file.url,
             }
             return HttpResponse(headers=headers)

tubesync/sync/youtube.py

@@ -64,13 +64,20 @@ def get_media_info(url):
     return response
 
 
-def download_media(url, media_format, extension, output_file, info_json):
+def download_media(url, media_format, extension, output_file, info_json, 
+                   sponsor_categories="all", 
+                   embed_thumbnail=False, embed_metadata=False, skip_sponsors=True, 
+                   write_subtitles=False, auto_subtitles=False, sub_langs='en'):
     '''
         Downloads a YouTube URL to a file on disk.
     '''
 
     def hook(event):
         filename = os.path.basename(event['filename'])
+    
+        if event.get('downloaded_bytes') is None or event.get('total_bytes') is None:
+            return None
+
         if event['status'] == 'error':
             log.error(f'[youtube-dl] error occured downloading: {filename}')
         elif event['status'] == 'downloading':
@@ -101,15 +108,41 @@ def download_media(url, media_format, extension, output_file, info_json):
             log.warn(f'[youtube-dl] unknown event: {str(event)}')
     hook.download_progress = 0
 
-    opts = get_yt_opts()
-    opts.update({
+    ytopts = {
         'format': media_format,
         'merge_output_format': extension,
         'outtmpl': output_file,
         'quiet': True,
         'progress_hooks': [hook],
-        'writeinfojson': info_json
-    })
+        'writeinfojson': info_json,
+        'postprocessors': [],
+        'writesubtitles': write_subtitles,
+        'writeautomaticsub': auto_subtitles,
+        'subtitleslangs': sub_langs.split(','),
+    }
+    
+    sbopt = {
+        'key': 'SponsorBlock',
+        'categories': [sponsor_categories]
+    }
+    ffmdopt = {
+        'key': 'FFmpegMetadata',
+        'add_chapters': True,
+        'add_metadata': True
+    }
+
+    opts = get_yt_opts()
+    if embed_thumbnail:
+        ytopts['postprocessors'].append({'key': 'EmbedThumbnail'})
+    if embed_metadata:
+        ffmdopt["add_metadata"] = True
+    if skip_sponsors:
+        ytopts['postprocessors'].append(sbopt)
+    
+    ytopts['postprocessors'].append(ffmdopt)
+        
+    opts.update(ytopts)
+        
     with yt_dlp.YoutubeDL(opts) as y:
         try:
             return y.download([url])

tubesync/tubesync/local_settings.py.container

@@ -67,6 +67,12 @@ YOUTUBE_DL_CACHEDIR = CONFIG_BASE_DIR / 'cache'
 COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt'
 
 
+HEALTHCHECK_FIREWALL_STR = str(os.getenv('TUBESYNC_HEALTHCHECK_FIREWAL', 'True')).strip().lower()
+HEALTHCHECK_FIREWALL = True if HEALTHCHECK_FIREWALL_STR == 'true' else False
+HEALTHCHECK_ALLOWED_IPS_STR = str(os.getenv('TUBESYNC_HEALTHCHECK_ALLOWED_IPS', '127.0.0.1'))
+HEALTHCHECK_ALLOWED_IPS = HEALTHCHECK_ALLOWED_IPS_STR.split(',')
+
+
 BASICAUTH_USERNAME = os.getenv('HTTP_USER', '').strip()
 BASICAUTH_PASSWORD = os.getenv('HTTP_PASS', '').strip()
 if BASICAUTH_USERNAME and BASICAUTH_PASSWORD:

tubesync/tubesync/settings.py

@@ -6,7 +6,7 @@ CONFIG_BASE_DIR = BASE_DIR
 DOWNLOADS_BASE_DIR = BASE_DIR
 
 
-VERSION = '0.12.0'
+VERSION = '0.13.2'
 SECRET_KEY = ''
 DEBUG = False
 ALLOWED_HOSTS = []