From e9f03cb6bfd1ae1b768df70bc7fb9e81bd7c1eb6 Mon Sep 17 00:00:00 2001 From: pacoccino Date: Thu, 31 Aug 2023 22:40:29 +0200 Subject: [PATCH 1/4] download subtitles draft --- .gitignore | 1 + tubesync/sync/models.py | 5 +++-- tubesync/sync/youtube.py | 4 ++++ tubesync/tubesync/settings.py | 3 +++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index eec472f..cb14cfd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.DS_Store # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 35fcda1..fb4d950 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -103,7 +103,8 @@ class Source(models.Model): EXTENSION_M4A = 'm4a' EXTENSION_OGG = 'ogg' EXTENSION_MKV = 'mkv' - EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV) + EXTENSION_MP4 = 'mp4' + EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV, EXTENSION_MP4) # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py @@ -398,7 +399,7 @@ class Source(models.Model): else: raise ValueError('Unable to choose audio extension, uknown acodec') else: - return self.EXTENSION_MKV + return self.EXTENSION_MP4 @classmethod def create_url(obj, source_type, key): diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 00ae958..a55e92a 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -73,6 +73,10 @@ def download_media(url, media_format, extension, output_file, info_json, def hook(event): filename = os.path.basename(event['filename']) + + if event.get('downloaded_bytes') is None or event.get('total_bytes') is None: + return None + if event['status'] == 'error': log.error(f'[youtube-dl] error occured downloading: {filename}') elif event['status'] == 'downloading': diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index de7a1bc..8762d4c 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -160,6 +160,9 @@ YOUTUBE_DEFAULTS = { 'ignoreerrors': True, # Skip on errors (such as unavailable videos in playlists) 'cachedir': False, # Disable on-disk caching 'addmetadata': True, # Embed metadata during postprocessing where available + 'writesubtitles': True, + 'writeautomaticsub': True, + 'subtitleslangs': ['fr', 'en'], } COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt' From be7454f72a0ba70acc484e5688fdf69b5b2dbe44 Mon Sep 17 00:00:00 2001 From: pacoccino Date: Sat, 2 Sep 2023 14:30:23 +0200 Subject: [PATCH 2/4] Add subtitles config into sources model --- .../sync/migrations/0018_source_subtitles.py | 27 +++++++++++++++++++ tubesync/sync/models.py | 21 ++++++++++++++- tubesync/sync/templates/sync/source.html | 16 +++++++++++ tubesync/sync/views.py | 2 +- tubesync/sync/youtube.py | 9 +++++-- tubesync/tubesync/settings.py | 3 --- 6 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 tubesync/sync/migrations/0018_source_subtitles.py diff --git a/tubesync/sync/migrations/0018_source_subtitles.py b/tubesync/sync/migrations/0018_source_subtitles.py new file mode 100644 index 0000000..c526b99 --- /dev/null +++ b/tubesync/sync/migrations/0018_source_subtitles.py @@ -0,0 +1,27 @@ +# Generated by pac + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0017_alter_source_sponsorblock_categories'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='write_subtitles', + field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'), + ), + migrations.AddField( + model_name='source', + name='auto_subtitles', + field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto subtitles'), + ), + migrations.AddField( + model_name='source', + name='sub_langs', + field=models.CharField(default='en', help_text='List of subtitles langs to download comma-separated. Example: en,fr',max_length=30), + ), + ] diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index fb4d950..f7f949e 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -350,6 +350,24 @@ class Source(models.Model): help_text=_('Source has failed to index media') ) + write_subtitles = models.BooleanField( + _('write subtitles'), + default=False, + help_text=_('Download video subtitles') + ) + + auto_subtitles = models.BooleanField( + _('accept auto-generated subs'), + default=False, + help_text=_('Accept auto-generated subtitles') + ) + sub_langs = models.CharField( + _('subs langs'), + max_length=30, + default='en', + help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr') + ) + def __str__(self): return self.name @@ -1334,7 +1352,8 @@ class Media(models.Model): download_youtube_media(self.url, format_str, self.source.extension, str(self.filepath), self.source.write_json, self.source.sponsorblock_categories, self.source.embed_thumbnail, - self.source.embed_metadata, self.source.enable_sponsorblock) + self.source.embed_metadata, self.source.enable_sponsorblock, + self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs ) # Return the download paramaters return format_str, self.source.extension diff --git a/tubesync/sync/templates/sync/source.html b/tubesync/sync/templates/sync/source.html index 1b789ed..65cf242 100644 --- a/tubesync/sync/templates/sync/source.html +++ b/tubesync/sync/templates/sync/source.html @@ -162,6 +162,22 @@ {% endif %} + + + {{ _("Download subtitles?") }}: + {{ _("Download subtitles?") }}: + + + {% if source.write_subtitles %} + + {{ _("Auto-generated subtitles?") }}: + {{ _("Auto-generated subtitles?") }}: + + + {{ _("Subs langs?") }}: + {{ _("Subs langs?") }}:{{source.sub_langs}} + + {% endif %} diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index b1a2860..6fb36d1 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -299,7 +299,7 @@ class EditSourceMixin: 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock', - 'sponsorblock_categories') + 'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs') errors = { 'invalid_media_format': _('Invalid media format, the media format contains ' 'errors or is empty. Check the table at the end of ' diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index a55e92a..4ac6e83 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -66,7 +66,8 @@ def get_media_info(url): def download_media(url, media_format, extension, output_file, info_json, sponsor_categories="all", - embed_thumbnail=False, embed_metadata=False, skip_sponsors=True): + embed_thumbnail=False, embed_metadata=False, skip_sponsors=True, + write_subtitles=False, auto_subtitles=False, sub_langs='en'): ''' Downloads a YouTube URL to a file on disk. ''' @@ -114,8 +115,12 @@ def download_media(url, media_format, extension, output_file, info_json, 'quiet': True, 'progress_hooks': [hook], 'writeinfojson': info_json, - 'postprocessors': [] + 'postprocessors': [], + 'writesubtitles': write_subtitles, + 'writeautomaticsub': auto_subtitles, + 'subtitleslangs': sub_langs.split(','), } + sbopt = { 'key': 'SponsorBlock', 'categories': [sponsor_categories] diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 8762d4c..de7a1bc 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -160,9 +160,6 @@ YOUTUBE_DEFAULTS = { 'ignoreerrors': True, # Skip on errors (such as unavailable videos in playlists) 'cachedir': False, # Disable on-disk caching 'addmetadata': True, # Embed metadata during postprocessing where available - 'writesubtitles': True, - 'writeautomaticsub': True, - 'subtitleslangs': ['fr', 'en'], } COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt' From 86744c05107630518dece519ccdcbe2a2f3c9b04 Mon Sep 17 00:00:00 2001 From: pacoccino Date: Sat, 2 Sep 2023 14:37:07 +0200 Subject: [PATCH 3/4] Remove extension edits --- tubesync/sync/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index f7f949e..97d76cb 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -103,8 +103,7 @@ class Source(models.Model): EXTENSION_M4A = 'm4a' EXTENSION_OGG = 'ogg' EXTENSION_MKV = 'mkv' - EXTENSION_MP4 = 'mp4' - EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV, EXTENSION_MP4) + EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV) # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py @@ -417,7 +416,7 @@ class Source(models.Model): else: raise ValueError('Unable to choose audio extension, uknown acodec') else: - return self.EXTENSION_MP4 + return self.EXTENSION_MKV @classmethod def create_url(obj, source_type, key): From 3dd445bf96aa410ce7accf017ea4f76108812c78 Mon Sep 17 00:00:00 2001 From: pacoccino Date: Mon, 4 Sep 2023 14:58:57 +0200 Subject: [PATCH 4/4] Add a validator for sub_lang --- tubesync/sync/models.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 97d76cb..874524c 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -8,6 +8,7 @@ from pathlib import Path from django.conf import settings from django.db import models from django.core.files.storage import FileSystemStorage +from django.core.validators import RegexValidator from django.utils.text import slugify from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -364,7 +365,13 @@ class Source(models.Model): _('subs langs'), max_length=30, default='en', - help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr') + help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'), + validators=[ + RegexValidator( + regex=r"^(\-?[\_\.a-zA-Z]+,)*(\-?[\_\.a-zA-Z]+){1}$", + message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat') + ) + ] ) def __str__(self):