From a40e3e715e55764085ab3dbe8746a57d79a61f0d Mon Sep 17 00:00:00 2001 From: Mike Weyandt Date: Wed, 3 Jan 2024 02:53:45 -0500 Subject: [PATCH] Add support for indexing/downloading previously streamed content for channel and channel_id sources --- .../migrations/0021_auto_20240103_0646.py | 23 ++++++++ tubesync/sync/models.py | 56 ++++++++++++++----- tubesync/sync/templates/sync/source.html | 8 +++ tubesync/sync/views.py | 12 ++-- 4 files changed, 78 insertions(+), 21 deletions(-) create mode 100644 tubesync/sync/migrations/0021_auto_20240103_0646.py diff --git a/tubesync/sync/migrations/0021_auto_20240103_0646.py b/tubesync/sync/migrations/0021_auto_20240103_0646.py new file mode 100644 index 0000000..6fe969a --- /dev/null +++ b/tubesync/sync/migrations/0021_auto_20240103_0646.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.23 on 2024-01-03 06:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0020_auto_20231024_1825'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='download_streams', + field=models.BooleanField(default=False, help_text='Download live stream media from this source', verbose_name='download streams'), + ), + migrations.AddField( + model_name='source', + name='index_streams', + field=models.BooleanField(default=False, help_text='Index live stream media from this source', verbose_name='index streams'), + ), + ] diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 356e779..9c965a6 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -158,8 +158,8 @@ class Source(models.Model): } # Format used to create indexable URLs INDEX_URLS = { - SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos', - SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos', + SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/{type}', + SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/{type}', SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}', } # Callback functions to get a list of media from the source @@ -267,6 +267,16 @@ class Source(models.Model): default=True, help_text=_('Download media from this source, if not selected the source will only be indexed') ) + index_streams = models.BooleanField( + _('index streams'), + default=False, + help_text=_('Index live stream media from this source') + ) + download_streams = models.BooleanField( + _('download streams'), + default=False, + help_text=_('Download live stream media from this source') + ) download_cap = models.IntegerField( _('download cap'), choices=CapChoices.choices, @@ -440,17 +450,16 @@ class Source(models.Model): return url.format(key=key) @classmethod - def create_index_url(obj, source_type, key): + def create_index_url(obj, source_type, key, type): url = obj.INDEX_URLS.get(source_type) - return url.format(key=key) + return url.format(key=key, type=type) @property def url(self): return Source.create_url(self.source_type, self.key) - @property - def index_url(self): - return Source.create_index_url(self.source_type, self.key) + def get_index_url(self, type='videos'): + return Source.create_index_url(self.source_type, self.key, type) @property def format_summary(self): @@ -547,18 +556,35 @@ class Source(models.Model): return True return bool(re.search(self.filter_text, media_item_title)) + def index_media_videos(self): + indexer = self.INDEXERS.get(self.source_type, None) + if not callable(indexer): + raise Exception(f'Source type f"{self.source_type}" has no indexer') + response = indexer(self.get_index_url(type='videos')) + if not isinstance(response, dict): + return [] + entries = response.get('entries', []) + return entries + + def index_media_streams(self): + indexer = self.INDEXERS.get(self.source_type, None) + if not callable(indexer): + raise Exception(f'Source type f"{self.source_type}" has no indexer') + response = indexer(self.get_index_url(type='streams')) + if not isinstance(response, dict): + return [] + entries = response.get('entries', []) + return entries + def index_media(self): ''' Index the media source returning a list of media metadata as dicts. ''' - indexer = self.INDEXERS.get(self.source_type, None) - if not callable(indexer): - raise Exception(f'Source type f"{self.source_type}" has no indexer') - response = indexer(self.index_url) - if not isinstance(response, dict): - return [] - entries = response.get('entries', []) - + entries = self.index_media_videos() + # Playlists do something different that I have yet to figure out + if self.source_type != Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: + if self.index_streams: + entries += self.index_media_streams() if settings.MAX_ENTRIES_PROCESSING: entries = entries[:settings.MAX_ENTRIES_PROCESSING] return entries diff --git a/tubesync/sync/templates/sync/source.html b/tubesync/sync/templates/sync/source.html index c5812b2..e260f66 100644 --- a/tubesync/sync/templates/sync/source.html +++ b/tubesync/sync/templates/sync/source.html @@ -69,6 +69,14 @@ Download media? Download media?
{% if source.download_media %}{% else %}{% endif %} + + Index streams? + Index streams?
{% if source.index_streams %}{% else %}{% endif %} + + + Download streams? + Download streams?
{% if source.download_streams %}{% else %}{% endif %} + Created Created
{{ source.created|date:'Y-m-d H:i:s' }} diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 0b808eb..e6f2553 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -295,12 +295,12 @@ class ValidateSourceView(FormView): class EditSourceMixin: model = Source fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format', - 'index_schedule', 'download_media', 'download_cap', 'delete_old_media', - 'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec', - 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', - 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', - 'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', - 'auto_subtitles', 'sub_langs') + 'index_schedule', 'download_media', 'index_streams', 'download_streams', + 'download_cap', 'delete_old_media', 'delete_removed_media', 'days_to_keep', + 'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps', + 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 'write_json', + 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock', + 'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs') errors = { 'invalid_media_format': _('Invalid media format, the media format contains ' 'errors or is empty. Check the table at the end of '