From a40e3e715e55764085ab3dbe8746a57d79a61f0d Mon Sep 17 00:00:00 2001 From: Mike Weyandt Date: Wed, 3 Jan 2024 02:53:45 -0500 Subject: [PATCH 1/3] Add support for indexing/downloading previously streamed content for channel and channel_id sources --- .../migrations/0021_auto_20240103_0646.py | 23 ++++++++ tubesync/sync/models.py | 56 ++++++++++++++----- tubesync/sync/templates/sync/source.html | 8 +++ tubesync/sync/views.py | 12 ++-- 4 files changed, 78 insertions(+), 21 deletions(-) create mode 100644 tubesync/sync/migrations/0021_auto_20240103_0646.py diff --git a/tubesync/sync/migrations/0021_auto_20240103_0646.py b/tubesync/sync/migrations/0021_auto_20240103_0646.py new file mode 100644 index 0000000..6fe969a --- /dev/null +++ b/tubesync/sync/migrations/0021_auto_20240103_0646.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.23 on 2024-01-03 06:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0020_auto_20231024_1825'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='download_streams', + field=models.BooleanField(default=False, help_text='Download live stream media from this source', verbose_name='download streams'), + ), + migrations.AddField( + model_name='source', + name='index_streams', + field=models.BooleanField(default=False, help_text='Index live stream media from this source', verbose_name='index streams'), + ), + ] diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 356e779..9c965a6 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -158,8 +158,8 @@ class Source(models.Model): } # Format used to create indexable URLs INDEX_URLS = { - SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos', - SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos', + SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/{type}', + SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/{type}', SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}', } # Callback functions to get a list of media from the source @@ -267,6 +267,16 @@ class Source(models.Model): default=True, help_text=_('Download media from this source, if not selected the source will only be indexed') ) + index_streams = models.BooleanField( + _('index streams'), + default=False, + help_text=_('Index live stream media from this source') + ) + download_streams = models.BooleanField( + _('download streams'), + default=False, + help_text=_('Download live stream media from this source') + ) download_cap = models.IntegerField( _('download cap'), choices=CapChoices.choices, @@ -440,17 +450,16 @@ class Source(models.Model): return url.format(key=key) @classmethod - def create_index_url(obj, source_type, key): + def create_index_url(obj, source_type, key, type): url = obj.INDEX_URLS.get(source_type) - return url.format(key=key) + return url.format(key=key, type=type) @property def url(self): return Source.create_url(self.source_type, self.key) - @property - def index_url(self): - return Source.create_index_url(self.source_type, self.key) + def get_index_url(self, type='videos'): + return Source.create_index_url(self.source_type, self.key, type) @property def format_summary(self): @@ -547,18 +556,35 @@ class Source(models.Model): return True return bool(re.search(self.filter_text, media_item_title)) + def index_media_videos(self): + indexer = self.INDEXERS.get(self.source_type, None) + if not callable(indexer): + raise Exception(f'Source type f"{self.source_type}" has no indexer') + response = indexer(self.get_index_url(type='videos')) + if not isinstance(response, dict): + return [] + entries = response.get('entries', []) + return entries + + def index_media_streams(self): + indexer = self.INDEXERS.get(self.source_type, None) + if not callable(indexer): + raise Exception(f'Source type f"{self.source_type}" has no indexer') + response = indexer(self.get_index_url(type='streams')) + if not isinstance(response, dict): + return [] + entries = response.get('entries', []) + return entries + def index_media(self): ''' Index the media source returning a list of media metadata as dicts. ''' - indexer = self.INDEXERS.get(self.source_type, None) - if not callable(indexer): - raise Exception(f'Source type f"{self.source_type}" has no indexer') - response = indexer(self.index_url) - if not isinstance(response, dict): - return [] - entries = response.get('entries', []) - + entries = self.index_media_videos() + # Playlists do something different that I have yet to figure out + if self.source_type != Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: + if self.index_streams: + entries += self.index_media_streams() if settings.MAX_ENTRIES_PROCESSING: entries = entries[:settings.MAX_ENTRIES_PROCESSING] return entries diff --git a/tubesync/sync/templates/sync/source.html b/tubesync/sync/templates/sync/source.html index c5812b2..e260f66 100644 --- a/tubesync/sync/templates/sync/source.html +++ b/tubesync/sync/templates/sync/source.html @@ -69,6 +69,14 @@ Download media? Download media?
{% if source.download_media %}{% else %}{% endif %} + + Index streams? + Index streams?
{% if source.index_streams %}{% else %}{% endif %} + + + Download streams? + Download streams?
{% if source.download_streams %}{% else %}{% endif %} + Created Created
{{ source.created|date:'Y-m-d H:i:s' }} diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 0b808eb..e6f2553 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -295,12 +295,12 @@ class ValidateSourceView(FormView): class EditSourceMixin: model = Source fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format', - 'index_schedule', 'download_media', 'download_cap', 'delete_old_media', - 'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec', - 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', - 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', - 'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', - 'auto_subtitles', 'sub_langs') + 'index_schedule', 'download_media', 'index_streams', 'download_streams', + 'download_cap', 'delete_old_media', 'delete_removed_media', 'days_to_keep', + 'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps', + 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 'write_json', + 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock', + 'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs') errors = { 'invalid_media_format': _('Invalid media format, the media format contains ' 'errors or is empty. Check the table at the end of ' From 17bec04cef5efc4614459a3145a2a51124fa1b12 Mon Sep 17 00:00:00 2001 From: Mike Weyandt Date: Wed, 3 Jan 2024 16:48:53 -0500 Subject: [PATCH 2/3] Update ffmpeg version --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index d0037e2..2f721e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,8 @@ FROM debian:bookworm-slim ARG TARGETPLATFORM ARG S6_VERSION="3.1.5.0" -ARG FFMPEG_DATE="autobuild-2023-11-29-14-19" -ARG FFMPEG_VERSION="112875-g47e214245b" +ARG FFMPEG_DATE="autobuild-2023-11-30-15-12" +ARG FFMPEG_VERSION="112876-ga30adf9f96" ENV DEBIAN_FRONTEND="noninteractive" \ HOME="/root" \ @@ -27,8 +27,8 @@ RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \ "linux/arm64") echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-aarch64.tar.xz" ;; \ *) echo "" ;; esac) && \ export FFMPEG_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \ - "linux/amd64") echo "36bac8c527bf390603416f749ab0dd860142b0a66f0865b67366062a9c286c8b" ;; \ - "linux/arm64") echo "8f36e45d99d2367a5c0c220ee3164fa48f4f0cec35f78204ccced8dc303bfbdc" ;; \ + "linux/amd64") echo "c935cca5fa6fc8f3d99d7de1647ebad32c113b51ec83e669c5499a2bf7b289ea" ;; \ + "linux/arm64") echo "02e5d6193946ca9f757788b3cee7220c4d840d88b132d17f274e5ca53b8e701b" ;; \ *) echo "" ;; esac) && \ export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \ "linux/amd64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linux64-gpl.tar.xz" ;; \ From f9aeb6cf48161936ff5f4e787f53d6835275e3d9 Mon Sep 17 00:00:00 2001 From: Mike Weyandt Date: Wed, 3 Jan 2024 19:02:07 -0500 Subject: [PATCH 3/3] Update s6 version --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2f721e5..096c8e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM debian:bookworm-slim ARG TARGETPLATFORM -ARG S6_VERSION="3.1.5.0" +ARG S6_VERSION="3.1.6.2" ARG FFMPEG_DATE="autobuild-2023-11-30-15-12" ARG FFMPEG_VERSION="112876-ga30adf9f96" @@ -19,8 +19,8 @@ RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \ "linux/arm64") echo "aarch64" ;; \ *) echo "" ;; esac) && \ export S6_ARCH_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \ - "linux/amd64") echo "65d0d0f353d2ff9d0af202b268b4bf53a9948a5007650854855c729289085739" ;; \ - "linux/arm64") echo "3fbd14201473710a592b2189e81f00f3c8998e96d34f16bd2429c35d1bc36d00" ;; \ + "linux/amd64") echo "95081f11c56e5a351e9ccab4e70c2b1c3d7d056d82b72502b942762112c03d1c" ;; \ + "linux/arm64") echo "3fc0bae418a0e3811b3deeadfca9cc2f0869fb2f4787ab8a53f6944067d140ee" ;; \ *) echo "" ;; esac) && \ export S6_DOWNLOAD_ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \ "linux/amd64") echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-x86_64.tar.xz" ;; \ @@ -34,7 +34,7 @@ RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \ "linux/amd64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linux64-gpl.tar.xz" ;; \ "linux/arm64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linuxarm64-gpl.tar.xz" ;; \ *) echo "" ;; esac) && \ - export S6_NOARCH_EXPECTED_SHA256="fd80c231e8ae1a0667b7ae2078b9ad0e1269c4d117bf447a4506815a700dbff3" && \ + export S6_NOARCH_EXPECTED_SHA256="05af2536ec4fb23f087a43ce305f8962512890d7c71572ed88852ab91d1434e3" && \ export S6_DOWNLOAD_NOARCH="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-noarch.tar.xz" && \ echo "Building for arch: ${ARCH}|${ARCH44}, downloading S6 from: ${S6_DOWNLOAD}}, expecting S6 SHA256: ${S6_EXPECTED_SHA256}" && \ set -x && \