From 24a49d2f14036ca6c69713025a7a9e9ca5deece0 Mon Sep 17 00:00:00 2001 From: KuhnChris Date: Tue, 14 Feb 2023 21:52:50 +0100 Subject: [PATCH 1/7] Phase 1 - extend model for new fields --- .gitignore | 3 +- Pipfile | 1 + .../migrations/0015_auto_20230214_2052.py | 34 +++++++ tubesync/sync/models.py | 95 +++++++++++++++++++ .../templates/widgets/checkbox_option.html | 7 ++ .../templates/widgets/checkbox_select.html | 5 + tubesync/sync/views.py | 4 +- tubesync/sync/youtube.py | 30 ++++-- 8 files changed, 169 insertions(+), 10 deletions(-) create mode 100644 tubesync/sync/migrations/0015_auto_20230214_2052.py create mode 100644 tubesync/sync/templates/widgets/checkbox_option.html create mode 100644 tubesync/sync/templates/widgets/checkbox_select.html diff --git a/.gitignore b/.gitignore index e6e892e..eec472f 100644 --- a/.gitignore +++ b/.gitignore @@ -131,4 +131,5 @@ dmypy.json # Pyre type checker .pyre/ -Pipfile.lock \ No newline at end of file +Pipfile.lock +.vscode/launch.json diff --git a/Pipfile b/Pipfile index 243f0f3..21e4e49 100644 --- a/Pipfile +++ b/Pipfile @@ -4,6 +4,7 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] +autopep8 = "*" [packages] django = "~=3.2" diff --git a/tubesync/sync/migrations/0015_auto_20230214_2052.py b/tubesync/sync/migrations/0015_auto_20230214_2052.py new file mode 100644 index 0000000..aab006f --- /dev/null +++ b/tubesync/sync/migrations/0015_auto_20230214_2052.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.18 on 2023-02-14 20:52 + +from django.db import migrations, models +import sync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0014_alter_media_media_file'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='embed_metadata', + field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'), + ), + migrations.AddField( + model_name='source', + name='embed_thumbnail', + field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'), + ), + migrations.AddField( + model_name='source', + name='enable_sponsorblock', + field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'), + ), + migrations.AddField( + model_name='source', + name='sponsorblock_categories', + field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))), + ), + ] diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 16415c7..9279c1f 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -7,6 +7,7 @@ from datetime import datetime, timedelta from pathlib import Path from django.conf import settings from django.db import models +from django.forms import MultipleChoiceField, CheckboxSelectMultiple from django.core.files.storage import FileSystemStorage from django.utils.text import slugify from django.utils import timezone @@ -23,6 +24,63 @@ from .mediaservers import PlexMediaServer media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') +class CommaSepField(models.Field): + "Implements comma-separated storage of lists" + + def __init__(self, separator=",", *args, **kwargs): + self.separator = separator + super().__init__(*args, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + # Only include kwarg if it's not the default + if self.separator != ",": + kwargs['separator'] = self.separator + return name, path, args, kwargs + + +class CustomCheckboxSelectMultiple(CheckboxSelectMultiple): + template_name = 'widgets/checkbox_select.html' + option_template_name = 'widgets/checkbox_option.html' + +class CommaSepChoiceField(CommaSepField): + "Implements comma-separated storage of lists" + + def __init__(self, separator=",", possible_choices=(("","")), *args, **kwargs): + print(">",separator, possible_choices, args, kwargs) + self.possible_choices = possible_choices + super().__init__(separator=separator, *args, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + print("<",name,path,args,kwargs) + # Only include kwarg if it's not the default + if self.separator != ",": + kwargs['separator'] = self.separator + kwargs['possible_choices'] = self.possible_choices + return name, path, args, kwargs + + def db_type(self, _connection): + return 'char(1024)' + + def get_choices(self): + choiceArray = [] + if self.possible_choices is None: + return choiceArray + for t in self.possible_choices: + choiceArray.append(t) + return choiceArray + + def formfield(self, **kwargs): + # This is a fairly standard way to set up some defaults + # while letting the caller override them. + print(self.choices) + defaults = {'form_class': MultipleChoiceField, + 'choices': self.get_choices, + 'widget': CustomCheckboxSelectMultiple} + defaults.update(kwargs) + #del defaults.required + return super().formfield(**defaults) class Source(models.Model): ''' @@ -106,6 +164,43 @@ class Source(models.Model): EXTENSION_MKV = 'mkv' EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV) + + # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py + SPONSORBLOCK_CATEGORIES_CHOICES = ( + ('all', 'All'), + ('sponsor', 'Sponsor'), + ('intro', 'Intermission/Intro Animation'), + ('outro', 'Endcards/Credits'), + ('selfpromo', 'Unpaid/Self Promotion'), + ('preview', 'Preview/Recap'), + ('filler', 'Filler Tangent'), + ('interaction', 'Interaction Reminder'), + ('music_offtopic', 'Non-Music Section'), + ) + + sponsorblock_categories = CommaSepChoiceField( + possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES, + default="all" + ) + + embed_metadata = models.BooleanField( + _('embed metadata'), + default=False, + help_text=_('Embed metadata from source into file') + ) + embed_thumbnail = models.BooleanField( + _('embed thumbnail'), + default=False, + help_text=_('Embed thumbnail into the file') + ) + + enable_sponsorblock = models.BooleanField( + _('enable sponsorblock'), + default=True, + help_text=_('Use SponsorBlock?') + ) + + # Fontawesome icons used for the source on the front end ICONS = { SOURCE_TYPE_YOUTUBE_CHANNEL: '', diff --git a/tubesync/sync/templates/widgets/checkbox_option.html b/tubesync/sync/templates/widgets/checkbox_option.html new file mode 100644 index 0000000..8578ecc --- /dev/null +++ b/tubesync/sync/templates/widgets/checkbox_option.html @@ -0,0 +1,7 @@ + + + \ No newline at end of file diff --git a/tubesync/sync/templates/widgets/checkbox_select.html b/tubesync/sync/templates/widgets/checkbox_select.html new file mode 100644 index 0000000..f621908 --- /dev/null +++ b/tubesync/sync/templates/widgets/checkbox_select.html @@ -0,0 +1,5 @@ +{% for group, options, index in widget.optgroups %} + {% for option in options %} + {% include option.template_name with option=option %} + {% endfor%} +{% endfor %} \ No newline at end of file diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 912dd22..5739ee0 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -297,7 +297,9 @@ class EditSourceMixin: fields = ('source_type', 'key', 'name', 'directory', 'media_format', 'index_schedule', 'download_media', 'download_cap', 'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', - 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 'write_json') + 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', + 'write_json', 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock', + 'sponsorblock_categories') errors = { 'invalid_media_format': _('Invalid media format, the media format contains ' 'errors or is empty. Check the table at the end of ' diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 08b60fe..9bbfbb7 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -64,7 +64,7 @@ def get_media_info(url): return response -def download_media(url, media_format, extension, output_file, info_json, sponsor_categories="all"): +def download_media(url, media_format, extension, output_file, info_json, sponsor_categories="all", embed_thumbnail=False, embed_metadata=False, skip_sponsors=True): ''' Downloads a YouTube URL to a file on disk. ''' @@ -101,23 +101,37 @@ def download_media(url, media_format, extension, output_file, info_json, sponsor log.warn(f'[youtube-dl] unknown event: {str(event)}') hook.download_progress = 0 - opts = get_yt_opts() - opts.update({ + ytopts = { 'format': media_format, 'merge_output_format': extension, 'outtmpl': output_file, 'quiet': True, 'progress_hooks': [hook], 'writeinfojson': info_json, - 'postprocessors': [{ + 'postprocessors': [] + } + sbopt = { 'key': 'SponsorBlock', 'categories': [sponsor_categories] - },{ - 'key': 'FFmpegMetadata', + } + ffmdopt = { + 'key': 'FFmpegMetadata', 'add_chapters': True, 'add_metadata': True - }] - }) + } + + opts = get_yt_opts() + if embed_thumbnail: + ytopts['postprocessors'].push({'key': 'EmbedThumbnail'}) + if embed_metadata: + ffmdopt["embed-metadata"] = True + if skip_sponsors: + ytopts['postprocessors'].push(sbopt) + + ytopts['postprocessors'].push(ffmdopt) + + opts.update(ytopts) + with yt_dlp.YoutubeDL(opts) as y: try: return y.download([url]) From 2772e85d9ffbb95767d61256141d210bc848468a Mon Sep 17 00:00:00 2001 From: KuhnChris Date: Wed, 15 Feb 2023 00:01:44 +0100 Subject: [PATCH 2/7] ffmpeg embed thumbnails, configuration --- tubesync/sync/models.py | 101 +++++++++++++----- .../templates/widgets/checkbox_option.html | 2 +- .../templates/widgets/checkbox_select.html | 10 +- tubesync/sync/youtube.py | 12 ++- 4 files changed, 85 insertions(+), 40 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 9279c1f..fc1d9f8 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1,4 +1,5 @@ import os +from typing import Any, Optional, Dict import uuid import json from xml.etree import ElementTree @@ -7,7 +8,7 @@ from datetime import datetime, timedelta from pathlib import Path from django.conf import settings from django.db import models -from django.forms import MultipleChoiceField, CheckboxSelectMultiple +from django.forms import MultipleChoiceField, CheckboxSelectMultiple, Field, TypedMultipleChoiceField from django.core.files.storage import FileSystemStorage from django.utils.text import slugify from django.utils import timezone @@ -24,37 +25,44 @@ from .mediaservers import PlexMediaServer media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') -class CommaSepField(models.Field): - "Implements comma-separated storage of lists" - - def __init__(self, separator=",", *args, **kwargs): - self.separator = separator - super().__init__(*args, **kwargs) - - def deconstruct(self): - name, path, args, kwargs = super().deconstruct() - # Only include kwarg if it's not the default - if self.separator != ",": - kwargs['separator'] = self.separator - return name, path, args, kwargs - - class CustomCheckboxSelectMultiple(CheckboxSelectMultiple): template_name = 'widgets/checkbox_select.html' option_template_name = 'widgets/checkbox_option.html' -class CommaSepChoiceField(CommaSepField): + def get_context(self, name: str, value: Any, attrs) -> Dict[str, Any]: + ctx = super().get_context(name, value, attrs)['widget'] + ctx["multipleChoiceProperties"] = [] + for _group, options, _index in ctx["optgroups"]: + for option in options: + if not isinstance(value,list) and ( option["value"] in value.selected_choices or ( value.allow_all and value.all_choice in value.selected_choices ) ): + checked = True + else: + checked = False + + ctx["multipleChoiceProperties"].append({ + "template_name": option["template_name"], + "type": option["type"], + "value": option["value"], + "label": option["label"], + "name": option["name"], + "checked": checked}) + + return { 'widget': ctx } + +class CommaSepChoiceField(models.Field): "Implements comma-separated storage of lists" - def __init__(self, separator=",", possible_choices=(("","")), *args, **kwargs): - print(">",separator, possible_choices, args, kwargs) + def __init__(self, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, *args, **kwargs): + self.separator = separator self.possible_choices = possible_choices - super().__init__(separator=separator, *args, **kwargs) + self.selected_choices = [] + self.allow_all = allow_all + self.all_label = all_label + self.all_choice = all_choice + super().__init__(*args, **kwargs) def deconstruct(self): name, path, args, kwargs = super().deconstruct() - print("<",name,path,args,kwargs) - # Only include kwarg if it's not the default if self.separator != ",": kwargs['separator'] = self.separator kwargs['possible_choices'] = self.possible_choices @@ -63,25 +71,54 @@ class CommaSepChoiceField(CommaSepField): def db_type(self, _connection): return 'char(1024)' - def get_choices(self): + def get_my_choices(self): choiceArray = [] if self.possible_choices is None: return choiceArray + if self.allow_all: + choiceArray.append((self.all_choice, _(self.all_label))) + for t in self.possible_choices: choiceArray.append(t) + return choiceArray def formfield(self, **kwargs): # This is a fairly standard way to set up some defaults # while letting the caller override them. - print(self.choices) defaults = {'form_class': MultipleChoiceField, - 'choices': self.get_choices, - 'widget': CustomCheckboxSelectMultiple} + 'choices': self.get_my_choices, + 'widget': CustomCheckboxSelectMultiple, + 'label': '', + 'required': False} defaults.update(kwargs) #del defaults.required return super().formfield(**defaults) + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + # Only include kwarg if it's not the default + if self.separator != ",": + kwargs['separator'] = self.separator + return name, path, args, kwargs + + def from_db_value(self, value, expr, conn): + if value is None: + self.selected_choices = [] + else: + self.selected_choices = value.split(",") + + return self + + def get_prep_value(self, value): + if value is None: + return "" + if not isinstance(value,list): + print("?! CommaSepChoiceField -> ",value) + return "" + + return ",".join(value) + class Source(models.Model): ''' A Source is a source of media. Currently, this is either a YouTube channel @@ -167,7 +204,6 @@ class Source(models.Model): # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py SPONSORBLOCK_CATEGORIES_CHOICES = ( - ('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), @@ -179,8 +215,13 @@ class Source(models.Model): ) sponsorblock_categories = CommaSepChoiceField( + _(''), possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES, - default="all" + all_choice="all", + allow_all=True, + all_label="(all options)", + default="all", + help_text=_("Select the sponsorblocks you want to enforce") ) embed_metadata = models.BooleanField( @@ -1380,7 +1421,9 @@ class Media(models.Model): f'no valid format available') # Download the media with youtube-dl download_youtube_media(self.url, format_str, self.source.extension, - str(self.filepath), self.source.write_json) + str(self.filepath), self.source.write_json, + self.source.sponsorblock_categories, self.source.embed_thumbnail, + self.source.embed_metadata, self.source.enable_sponsorblock) # Return the download paramaters return format_str, self.source.extension diff --git a/tubesync/sync/templates/widgets/checkbox_option.html b/tubesync/sync/templates/widgets/checkbox_option.html index 8578ecc..06a6723 100644 --- a/tubesync/sync/templates/widgets/checkbox_option.html +++ b/tubesync/sync/templates/widgets/checkbox_option.html @@ -2,6 +2,6 @@ --> \ No newline at end of file diff --git a/tubesync/sync/templates/widgets/checkbox_select.html b/tubesync/sync/templates/widgets/checkbox_select.html index f621908..ded69d0 100644 --- a/tubesync/sync/templates/widgets/checkbox_select.html +++ b/tubesync/sync/templates/widgets/checkbox_select.html @@ -1,5 +1,5 @@ -{% for group, options, index in widget.optgroups %} - {% for option in options %} - {% include option.template_name with option=option %} - {% endfor%} -{% endfor %} \ No newline at end of file + +{% for option in widget.multipleChoiceProperties %} + {% include option.template_name with option=option %} +{% endfor %} + +{% for option in widget.multipleChoiceProperties %} + {% include option.template_name with option=option %} +{% endfor %} +