Add support for indexing/downloading previously streamed content for channel and channel_id sources

tubesync/sync/migrations/0021_auto_20240103_0646.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.23 on 2024-01-03 06:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0020_auto_20231024_1825'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='download_streams',
+            field=models.BooleanField(default=False, help_text='Download live stream media from this source', verbose_name='download streams'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='index_streams',
+            field=models.BooleanField(default=False, help_text='Index live stream media from this source', verbose_name='index streams'),
+        ),
+    ]

tubesync/sync/models.py

@@ -158,8 +158,8 @@ class Source(models.Model):
     }
     # Format used to create indexable URLs
     INDEX_URLS = {
-        SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos',
+        SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/{type}',
-        SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos',
+        SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/{type}',
         SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
     }
     # Callback functions to get a list of media from the source
@@ -267,6 +267,16 @@ class Source(models.Model):
         default=True,
         help_text=_('Download media from this source, if not selected the source will only be indexed')
     )
+    index_streams = models.BooleanField(
+        _('index streams'),
+        default=False,
+        help_text=_('Index live stream media from this source')
+    )
+    download_streams = models.BooleanField(
+        _('download streams'),
+        default=False,
+        help_text=_('Download live stream media from this source')
+    )
     download_cap = models.IntegerField(
         _('download cap'),
         choices=CapChoices.choices,
@@ -440,17 +450,16 @@ class Source(models.Model):
         return url.format(key=key)
 
     @classmethod
-    def create_index_url(obj, source_type, key):
+    def create_index_url(obj, source_type, key, type):
         url = obj.INDEX_URLS.get(source_type)
-        return url.format(key=key)
+        return url.format(key=key, type=type)
 
     @property
     def url(self):
         return Source.create_url(self.source_type, self.key)
 
-    @property
+    def get_index_url(self, type='videos'):
-    def index_url(self):
+        return Source.create_index_url(self.source_type, self.key, type)
-        return Source.create_index_url(self.source_type, self.key)
 
     @property
     def format_summary(self):
@@ -547,18 +556,35 @@ class Source(models.Model):
             return True
         return bool(re.search(self.filter_text, media_item_title))
     
+    def index_media_videos(self):
+        indexer = self.INDEXERS.get(self.source_type, None)
+        if not callable(indexer):
+            raise Exception(f'Source type f"{self.source_type}" has no indexer')
+        response = indexer(self.get_index_url(type='videos'))
+        if not isinstance(response, dict):
+            return []
+        entries = response.get('entries', [])
+        return entries
+
+    def index_media_streams(self):
+        indexer = self.INDEXERS.get(self.source_type, None)
+        if not callable(indexer):
+            raise Exception(f'Source type f"{self.source_type}" has no indexer')
+        response = indexer(self.get_index_url(type='streams'))
+        if not isinstance(response, dict):
+            return []
+        entries = response.get('entries', [])
+        return entries
+
     def index_media(self):
         '''
             Index the media source returning a list of media metadata as dicts.
         '''
-        indexer = self.INDEXERS.get(self.source_type, None)
+        entries = self.index_media_videos()
-        if not callable(indexer):
+        # Playlists do something different that I have yet to figure out
-            raise Exception(f'Source type f"{self.source_type}" has no indexer')
+        if self.source_type != Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
-        response = indexer(self.index_url)
+            if self.index_streams:
-        if not isinstance(response, dict):
+                entries += self.index_media_streams()
-            return []
-        entries = response.get('entries', [])
-
         if settings.MAX_ENTRIES_PROCESSING:
             entries = entries[:settings.MAX_ENTRIES_PROCESSING]
         return entries

tubesync/sync/templates/sync/source.html

@@ -69,6 +69,14 @@
         <td class="hide-on-small-only">Download media?</td>
         <td><span class="hide-on-med-and-up">Download media?<br></span><strong>{% if source.download_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
+      <tr title="Index streams from this source">
+        <td class="hide-on-small-only">Index streams?</td>
+        <td><span class="hide-on-med-and-up">Index streams?<br></span><strong>{% if source.index_streams %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
+      <tr title="Download streams from this source">
+        <td class="hide-on-small-only">Download streams?</td>
+        <td><span class="hide-on-med-and-up">Download streams?<br></span><strong>{% if source.download_streams %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
       <tr title="When then source was created locally in TubeSync">
         <td class="hide-on-small-only">Created</td>
         <td><span class="hide-on-med-and-up">Created<br></span><strong>{{ source.created|date:'Y-m-d H:i:s' }}</strong></td>

tubesync/sync/views.py

@@ -295,12 +295,12 @@ class ValidateSourceView(FormView):
 class EditSourceMixin:
     model = Source
     fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format',
-              'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
+              'index_schedule', 'download_media', 'index_streams', 'download_streams',
-              'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
+              'download_cap', 'delete_old_media', 'delete_removed_media', 'days_to_keep',
-              'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails',
+              'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
-              'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
+              'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 'write_json',
-              'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
+              'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock',
-              'auto_subtitles', 'sub_langs')
+              'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs')
     errors = {
         'invalid_media_format': _('Invalid media format, the media format contains '
                                   'errors or is empty. Check the table at the end of '