PublicSnapshot
/
tubesync
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for indexing/downloading previously streamed content for channel and channel_id sources

This commit is contained in:
Mike Weyandt 2024-01-03 02:53:45 -05:00
parent 7384c00713
commit a40e3e715e
4 changed files with 78 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
tubesync/sync/migrations/0021_auto_20240103_0646.py Normal file
View File

@ -0,0 +1,23 @@
# Generated by Django 3.2.23 on 2024-01-03 06:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sync', '0020_auto_20231024_1825'),
]
operations = [
migrations.AddField(
model_name='source',
name='download_streams',
field=models.BooleanField(default=False, help_text='Download live stream media from this source', verbose_name='download streams'),
),
migrations.AddField(
model_name='source',
name='index_streams',
field=models.BooleanField(default=False, help_text='Index live stream media from this source', verbose_name='index streams'),
),
]

56
tubesync/sync/models.py
View File

@ -158,8 +158,8 @@ class Source(models.Model):
}
# Format used to create indexable URLs
INDEX_URLS = {
SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos',
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos',
SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/{type}',
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/{type}',
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
}
# Callback functions to get a list of media from the source
@ -267,6 +267,16 @@ class Source(models.Model):
default=True,
help_text=_('Download media from this source, if not selected the source will only be indexed')
)
index_streams = models.BooleanField(
_('index streams'),
default=False,
help_text=_('Index live stream media from this source')
)
download_streams = models.BooleanField(
_('download streams'),
default=False,
help_text=_('Download live stream media from this source')
)
download_cap = models.IntegerField(
_('download cap'),
choices=CapChoices.choices,
@ -440,17 +450,16 @@ class Source(models.Model):
return url.format(key=key)
@classmethod
def create_index_url(obj, source_type, key):
def create_index_url(obj, source_type, key, type):
url = obj.INDEX_URLS.get(source_type)
return url.format(key=key)
return url.format(key=key, type=type)
@property
def url(self):
return Source.create_url(self.source_type, self.key)
@property
def index_url(self):
return Source.create_index_url(self.source_type, self.key)
def get_index_url(self, type='videos'):
return Source.create_index_url(self.source_type, self.key, type)
@property
def format_summary(self):
@ -547,18 +556,35 @@ class Source(models.Model):
return True
return bool(re.search(self.filter_text, media_item_title))
def index_media_videos(self):
indexer = self.INDEXERS.get(self.source_type, None)
if not callable(indexer):
raise Exception(f'Source type f"{self.source_type}" has no indexer')
response = indexer(self.get_index_url(type='videos'))
if not isinstance(response, dict):
return []
entries = response.get('entries', [])
return entries
def index_media_streams(self):
indexer = self.INDEXERS.get(self.source_type, None)
if not callable(indexer):
raise Exception(f'Source type f"{self.source_type}" has no indexer')
response = indexer(self.get_index_url(type='streams'))
if not isinstance(response, dict):
return []
entries = response.get('entries', [])
return entries
def index_media(self):
'''
Index the media source returning a list of media metadata as dicts.
'''
indexer = self.INDEXERS.get(self.source_type, None)
if not callable(indexer):
raise Exception(f'Source type f"{self.source_type}" has no indexer')
response = indexer(self.index_url)
if not isinstance(response, dict):
return []
entries = response.get('entries', [])
entries = self.index_media_videos()
# Playlists do something different that I have yet to figure out
if self.source_type != Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
if self.index_streams:
entries += self.index_media_streams()
if settings.MAX_ENTRIES_PROCESSING:
entries = entries[:settings.MAX_ENTRIES_PROCESSING]
return entries

8
tubesync/sync/templates/sync/source.html
View File

@ -69,6 +69,14 @@
<td class="hide-on-small-only">Download media?</td>
<td><span class="hide-on-med-and-up">Download media?<br></span><strong>{% if source.download_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr>
<tr title="Index streams from this source">
<td class="hide-on-small-only">Index streams?</td>
<td><span class="hide-on-med-and-up">Index streams?<br></span><strong>{% if source.index_streams %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr>
<tr title="Download streams from this source">
<td class="hide-on-small-only">Download streams?</td>
<td><span class="hide-on-med-and-up">Download streams?<br></span><strong>{% if source.download_streams %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr>
<tr title="When then source was created locally in TubeSync">
<td class="hide-on-small-only">Created</td>
<td><span class="hide-on-med-and-up">Created<br></span><strong>{{ source.created|date:'Y-m-d H:i:s' }}</strong></td>

12
tubesync/sync/views.py
View File

@ -295,12 +295,12 @@ class ValidateSourceView(FormView):
class EditSourceMixin:
model = Source
fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format',
'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails',
'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
'auto_subtitles', 'sub_langs')
'index_schedule', 'download_media', 'index_streams', 'download_streams',
'download_cap', 'delete_old_media', 'delete_removed_media', 'days_to_keep',
'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 'write_json',
'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock',
'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs')
errors = {
'invalid_media_format': _('Invalid media format, the media format contains '
'errors or is empty. Check the table at the end of '