Merge pull request #338 from kuhnchris/embed-thumbnail

Configurations in Sources

.gitignore

@@ -132,3 +132,4 @@ dmypy.json
 .pyre/
 
 Pipfile.lock
+.vscode/launch.json

Pipfile

@@ -4,6 +4,7 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
+autopep8 = "*"
 
 [packages]
 django = "~=3.2"

tubesync/sync/fields.py

@@ -0,0 +1,109 @@
+from django.forms import MultipleChoiceField,  CheckboxSelectMultiple, Field, TypedMultipleChoiceField
+from django.db import models
+from typing import Any, Optional, Dict
+from django.utils.translation import gettext_lazy as _
+
+# this is a form field!
+class CustomCheckboxSelectMultiple(CheckboxSelectMultiple):
+    template_name = 'widgets/checkbox_select.html'
+    option_template_name = 'widgets/checkbox_option.html'
+
+    def get_context(self, name: str, value: Any, attrs) -> Dict[str, Any]:
+        ctx = super().get_context(name, value, attrs)['widget']
+        ctx["multipleChoiceProperties"] = []
+        for _group, options, _index in ctx["optgroups"]:
+            for option in options:
+                if not isinstance(value,str) and not isinstance(value,list) and ( option["value"] in value.selected_choices or ( value.allow_all and value.all_choice in value.selected_choices ) ):
+                    checked = True
+                else:
+                    checked = False
+
+                ctx["multipleChoiceProperties"].append({
+                    "template_name": option["template_name"],
+                    "type": option["type"],
+                    "value": option["value"],
+                    "label": option["label"],
+                    "name": option["name"],
+                    "checked": checked})
+
+        return { 'widget': ctx }
+
+# this is a database field!
+class CommaSepChoiceField(models.Field):
+    "Implements comma-separated storage of lists"
+
+    def __init__(self, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, *args, **kwargs):
+        self.separator = separator
+        self.possible_choices = possible_choices
+        self.selected_choices = []
+        self.allow_all = allow_all
+        self.all_label = all_label
+        self.all_choice = all_choice
+        super().__init__(*args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        if self.separator != ",":
+            kwargs['separator'] = self.separator
+        kwargs['possible_choices'] = self.possible_choices
+        return name, path, args, kwargs
+
+    def db_type(self, _connection):
+        return 'char(1024)'
+
+    def get_my_choices(self):
+        choiceArray = []
+        if self.possible_choices is None:
+            return choiceArray
+        if self.allow_all:
+            choiceArray.append((self.all_choice, _(self.all_label)))
+
+        for t in self.possible_choices:
+            choiceArray.append(t)
+        
+        return choiceArray
+
+    def formfield(self, **kwargs):
+        # This is a fairly standard way to set up some defaults
+        # while letting the caller override them.
+        defaults = {'form_class': MultipleChoiceField, 
+                    'choices': self.get_my_choices,
+                    'widget': CustomCheckboxSelectMultiple,
+                    'label': '',
+                    'required': False}
+        defaults.update(kwargs)
+        #del defaults.required
+        return super().formfield(**defaults)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        # Only include kwarg if it's not the default
+        if self.separator != ",":
+            kwargs['separator'] = self.separator
+        return name, path, args, kwargs
+
+    def from_db_value(self, value, expr, conn):
+        if value is None:
+            self.selected_choices = []
+        else:
+            self.selected_choices = value.split(",")
+
+        return self
+
+    def get_prep_value(self, value):
+        if value is None:
+            return ""
+        if not isinstance(value,list):
+            return ""
+
+        if self.all_choice not in value:
+            return ",".join(value)
+        else:
+            return self.all_choice
+
+    def get_text_for_value(self, val):
+        fval = [i for i in self.possible_choices if i[0] == val]
+        if len(fval) <= 0:
+            return []
+        else:
+            return fval[0][1]

tubesync/sync/migrations/0016_auto_20230214_2052.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.2.18 on 2023-02-14 20:52
+
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0015_auto_20230213_0603'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='embed_metadata',
+            field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='embed_thumbnail',
+            field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='enable_sponsorblock',
+            field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='sponsorblock_categories',
+            field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))),
+        ),
+    ]

tubesync/sync/models.py

@@ -19,11 +19,10 @@ from .utils import seconds_to_timestr, parse_media_format
 from .matching import (get_best_combined_format, get_best_audio_format, 
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
-
+from .fields import CommaSepChoiceField
 
 media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/')
 
-
 class Source(models.Model):
     '''
         A Source is a source of media. Currently, this is either a YouTube channel
@@ -106,6 +105,47 @@ class Source(models.Model):
     EXTENSION_MKV = 'mkv'
     EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
 
+
+    # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
+    SPONSORBLOCK_CATEGORIES_CHOICES = (
+        ('sponsor', 'Sponsor'),
+        ('intro', 'Intermission/Intro Animation'),
+        ('outro', 'Endcards/Credits'),
+        ('selfpromo', 'Unpaid/Self Promotion'),
+        ('preview', 'Preview/Recap'),
+        ('filler', 'Filler Tangent'),
+        ('interaction', 'Interaction Reminder'),
+        ('music_offtopic', 'Non-Music Section'),
+    )
+    
+    sponsorblock_categories = CommaSepChoiceField(
+            _(''),
+            possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
+            all_choice="all",
+            allow_all=True,
+            all_label="(all options)",
+            default="all",
+            help_text=_("Select the sponsorblocks you want to enforce")
+        )
+
+    embed_metadata = models.BooleanField(
+        _('embed metadata'),
+        default=False,
+        help_text=_('Embed metadata from source into file')
+    )
+    embed_thumbnail = models.BooleanField(
+        _('embed thumbnail'),
+        default=False,
+        help_text=_('Embed thumbnail into the file')
+    )
+
+    enable_sponsorblock = models.BooleanField(
+        _('enable sponsorblock'),
+        default=True,
+        help_text=_('Use SponsorBlock?')
+    )
+
+
     # Fontawesome icons used for the source on the front end
     ICONS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
@@ -1291,7 +1331,9 @@ class Media(models.Model):
                                     f'no valid format available')
         # Download the media with youtube-dl
         download_youtube_media(self.url, format_str, self.source.extension,
-                               str(self.filepath), self.source.write_json)
+                               str(self.filepath), self.source.write_json, 
+                               self.source.sponsorblock_categories, self.source.embed_thumbnail,
+                               self.source.embed_metadata, self.source.enable_sponsorblock)
         # Return the download paramaters
         return format_str, self.source.extension
 

tubesync/sync/templates/sync/source.html

@@ -130,6 +130,39 @@
         <td class="hide-on-small-only">UUID</td>
         <td><span class="hide-on-med-and-up">UUID<br></span><strong>{{ source.uuid }}</strong></td>
       </tr>
+      <tr title="{{ _('Embedding thumbnail?') }}">
+        <td class="hide-on-small-only">{{ _("Embed thumbnail?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Embed thumbnail?") }}<br></span><strong><i class="fas {% if source.embed_thumbnail %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+      <tr title="{{ _('Embedding metadata?') }}">
+        <td class="hide-on-small-only">{{ _("Embed metadata?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Embed metadata?") }}<br></span><strong><i class="fas {% if source.embed_metadata %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+
+      <tr title="{{ _('Is sponsorblock enabled?') }}">
+        <td class="hide-on-small-only">{{ _("SponsorBlock?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Sponsorblock enabled?") }}<br></span><strong><i class="fas {% if source.enable_sponsorblock %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+
+      {% if source.enable_sponsorblock %}
+      <tr title="{{ _('SponsorBlock: What to block?') }}">
+        <td class="hide-on-small-only">{{ _("What blocked?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("What blocked?") }}<br></span><strong>
+          {% if source.sponsorblock_categories.all_choice in source.sponsorblock_categories.selected_choices %}
+            {% for k,v in source.sponsorblock_categories.possible_choices %}
+              {{ v }}: <i class="fas fa-check"></i><BR>
+            {% endfor %}
+          {% else %}
+            {% for c in source.sponsorblock_categories.selected_choices %} 
+              {% for k,v in source.sponsorblock_categories.possible_choices %}
+                {% if k == c %} {{ v }}: <i class="fas fa-check"></i><BR>{% endif %}
+              {% endfor %}
+            {% endfor %}
+          {% endif %}
+        </strong></td>
+      </tr>
+      {% endif %}
+      
     </table>
   </div>
 </div>

tubesync/sync/templates/widgets/checkbox_option.html

@@ -0,0 +1,7 @@
+<!--<input type="{{ option.type }}" name="{{ option.name }}" value="{{ option.value }}" id="{{ option.value }}"><BR>
+    <label for="{{ option.value }}">{{option.label}}</label>-->
+
+<label>
+    <input type="{{ option.type }}" name="{{ option.name }}" value="{{ option.value }}" id="{{ option.value }}" {% if option.checked %}checked{% endif %}>
+    <span>{{option.label}}</span>
+</label>

tubesync/sync/templates/widgets/checkbox_select.html

@@ -0,0 +1,5 @@
+</label>
+{% for option in widget.multipleChoiceProperties %}
+    {% include option.template_name with option=option %}
+{% endfor %}
+<label>

tubesync/sync/views.py

@@ -297,7 +297,9 @@ class EditSourceMixin:
     fields = ('source_type', 'key', 'name', 'directory', 'media_format',
               'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
               'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
-              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 'write_json')
+              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 
+              'write_json', 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock',
+              'sponsorblock_categories')
     errors = {
         'invalid_media_format': _('Invalid media format, the media format contains '
                                   'errors or is empty. Check the table at the end of '

tubesync/sync/youtube.py

@@ -64,7 +64,9 @@ def get_media_info(url):
     return response
 
 
-def download_media(url, media_format, extension, output_file, info_json, sponsor_categories="all"):
+def download_media(url, media_format, extension, output_file, info_json, 
+                   sponsor_categories="all", 
+                   embed_thumbnail=False, embed_metadata=False, skip_sponsors=True):
     '''
         Downloads a YouTube URL to a file on disk.
     '''
@@ -100,27 +102,38 @@ def download_media(url, media_format, extension, output_file, info_json, sponsor
         else:
             log.warn(f'[youtube-dl] unknown event: {str(event)}')
     hook.download_progress = 0
-    postprocessors = []
+
-    postprocessors.append({
+    ytopts = {
-        'key': 'FFmpegMetadata',
-        'add_chapters': True,
-        'add_metadata': True
-    })
-    # Pending configuration options from PR #338
-    #postprocessors.append({
-    #    'key': 'SponsorBlock',
-    #    'categories': [sponsor_categories]
-    #})
-    opts = get_yt_opts()
-    opts.update({
         'format': media_format,
         'merge_output_format': extension,
         'outtmpl': output_file,
         'quiet': True,
         'progress_hooks': [hook],
         'writeinfojson': info_json,
-        'postprocessors': postprocessors,
+        'postprocessors': []
-    })
+    }
+    sbopt = {
+        'key': 'SponsorBlock',
+        'categories': [sponsor_categories]
+    }
+    ffmdopt = {
+        'key': 'FFmpegMetadata',
+        'add_chapters': True,
+        'add_metadata': True
+    }
+
+    opts = get_yt_opts()
+    if embed_thumbnail:
+        ytopts['postprocessors'].append({'key': 'EmbedThumbnail'})
+    if embed_metadata:
+        ffmdopt["add_metadata"] = True
+    if skip_sponsors:
+        ytopts['postprocessors'].append(sbopt)
+    
+    ytopts['postprocessors'].append(ffmdopt)
+        
+    opts.update(ytopts)
+        
     with yt_dlp.YoutubeDL(opts) as y:
         try:
             return y.download([url])