diff --git a/tubesync/sync/admin.py b/tubesync/sync/admin.py index 1e445b7..584da4c 100644 --- a/tubesync/sync/admin.py +++ b/tubesync/sync/admin.py @@ -16,10 +16,14 @@ class SourceAdmin(admin.ModelAdmin): class MediaAdmin(admin.ModelAdmin): ordering = ('-created',) - list_display = ('uuid', 'key', 'source', 'can_download', 'skip', 'downloaded') + list_display = ('uuid', 'key', 'source', 'can_download', 'skip', 'is_live', 'downloaded') readonly_fields = ('uuid', 'created') search_fields = ('uuid', 'source__key', 'key') + @admin.display(boolean=True) + def is_live(self, obj): + return obj.is_live + @admin.register(MediaServer) class MediaServerAdmin(admin.ModelAdmin): diff --git a/tubesync/sync/migrations/0018_twitch_support.py b/tubesync/sync/migrations/0018_twitch_support.py new file mode 100644 index 0000000..3fe52c7 --- /dev/null +++ b/tubesync/sync/migrations/0018_twitch_support.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.18 on 2023-03-28 15:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0017_alter_source_sponsorblock_categories'), + ] + + operations = [ + migrations.AddField( + model_name='media', + name='last_crawl', + field=models.DateTimeField(blank=True, db_index=True, help_text='Date and time the metadata of the media was last crawled', null=True, verbose_name='last crawl'), + ), + migrations.AlterField( + model_name='source', + name='source_type', + field=models.CharField(choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist'), ('t', 'Twitch channel')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type'), + ), + ] diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 35fcda1..15b7ad3 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -14,8 +14,9 @@ from django.utils.translation import gettext_lazy as _ from common.errors import NoFormatException from common.utils import clean_filename from .youtube import (get_media_info as get_youtube_media_info, - download_media as download_youtube_media) -from .utils import seconds_to_timestr, parse_media_format + download_media as download_youtube_media, + get_twitch_media_info) +from .utils import seconds_to_timestr, parse_media_format, parse_twitch_media_format from .matching import (get_best_combined_format, get_best_audio_format, get_best_video_format) from .mediaservers import PlexMediaServer @@ -32,12 +33,14 @@ class Source(models.Model): SOURCE_TYPE_YOUTUBE_CHANNEL = 'c' SOURCE_TYPE_YOUTUBE_CHANNEL_ID = 'i' SOURCE_TYPE_YOUTUBE_PLAYLIST = 'p' + SOURCE_TYPE_TWITCH_CHANNEL = 't' SOURCE_TYPES = (SOURCE_TYPE_YOUTUBE_CHANNEL, SOURCE_TYPE_YOUTUBE_CHANNEL_ID, - SOURCE_TYPE_YOUTUBE_PLAYLIST) + SOURCE_TYPE_YOUTUBE_PLAYLIST, SOURCE_TYPE_TWITCH_CHANNEL) SOURCE_TYPE_CHOICES = ( (SOURCE_TYPE_YOUTUBE_CHANNEL, _('YouTube channel')), (SOURCE_TYPE_YOUTUBE_CHANNEL_ID, _('YouTube channel by ID')), (SOURCE_TYPE_YOUTUBE_PLAYLIST, _('YouTube playlist')), + (SOURCE_TYPE_TWITCH_CHANNEL, _('Twitch channel')), ) SOURCE_RESOLUTION_360P = '360p' @@ -151,30 +154,35 @@ class Source(models.Model): SOURCE_TYPE_YOUTUBE_CHANNEL: '', SOURCE_TYPE_YOUTUBE_CHANNEL_ID: '', SOURCE_TYPE_YOUTUBE_PLAYLIST: '', + SOURCE_TYPE_TWITCH_CHANNEL: '', } # Format to use to display a URL for the source URLS = { SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}', SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}', SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}', + SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/{key}/videos', } # Format used to create indexable URLs INDEX_URLS = { SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos', SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos', SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}', + SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/{key}/videos?filter=all&sort=time', } # Callback functions to get a list of media from the source INDEXERS = { SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info, SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info, SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info, + SOURCE_TYPE_TWITCH_CHANNEL: get_twitch_media_info, } # Field names to find the media ID used as the key when storing media KEY_FIELD = { SOURCE_TYPE_YOUTUBE_CHANNEL: 'id', SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'id', SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id', + SOURCE_TYPE_TWITCH_CHANNEL: 'id', } class CapChoices(models.IntegerChoices): @@ -546,12 +554,21 @@ class Media(models.Model): Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/watch?v={key}', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/watch?v={key}', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/videos/{key}', } # Callback functions to get a list of media from the source INDEXERS = { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info, Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info, Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info, + Source.SOURCE_TYPE_TWITCH_CHANNEL: get_youtube_media_info, + } + # Callback functions to get a list of media from the source + FORMAT_PARSER = { + Source.SOURCE_TYPE_YOUTUBE_CHANNEL: parse_media_format, + Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: parse_media_format, + Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: parse_media_format, + Source.SOURCE_TYPE_TWITCH_CHANNEL: parse_twitch_media_format, } # Maps standardised names to names used in source metdata METADATA_FIELDS = { @@ -559,16 +576,19 @@ class Media(models.Model): Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'upload_date', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'upload_date', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'upload_date', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'upload_date', }, 'title': { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'title', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'title', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'title', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'title', }, 'thumbnail': { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'thumbnail', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'thumbnail', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'thumbnail', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'thumbnail', }, 'description': { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'description', @@ -579,46 +599,61 @@ class Media(models.Model): Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'duration', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'duration', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'duration', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'duration', }, 'formats': { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'formats', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'formats', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'formats', }, 'categories': { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'categories', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'categories', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'categories', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'categories', }, 'rating': { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'average_rating', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'average_rating', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'average_rating', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'average_rating', }, 'age_limit': { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'age_limit', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'age_limit', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'age_limit', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'age_limit', }, 'uploader': { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'uploader', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'uploader', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'uploader', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'uploader', }, 'upvotes': { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'like_count', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'like_count', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'like_count', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'like_count', }, 'downvotes': { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'dislike_count', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'dislike_count', }, 'playlist_title': { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'playlist_title', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'playlist_title', + }, + 'is_live': { + Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'is_live', + Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'is_live', + Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'is_live', + Source.SOURCE_TYPE_TWITCH_CHANNEL: 'is_live', }, } STATE_UNKNOWN = 'unknown' @@ -626,6 +661,7 @@ class Media(models.Model): STATE_DOWNLOADING = 'downloading' STATE_DOWNLOADED = 'downloaded' STATE_SKIPPED = 'skipped' + STATE_LIVE = 'live' STATE_DISABLED_AT_SOURCE = 'source-disabled' STATE_ERROR = 'error' STATES = (STATE_UNKNOWN, STATE_SCHEDULED, STATE_DOWNLOADING, STATE_DOWNLOADED, @@ -638,6 +674,7 @@ class Media(models.Model): STATE_SKIPPED: '', STATE_DISABLED_AT_SOURCE: '', STATE_ERROR: '', + STATE_LIVE: '', } uuid = models.UUIDField( @@ -798,6 +835,13 @@ class Media(models.Model): null=True, help_text=_('Size of the downloaded media in bytes') ) + last_crawl = models.DateTimeField( + _('last crawl'), + db_index=True, + null=True, + blank=True, + help_text=_('Date and time the metadata of the media was last crawled') + ) def __str__(self): return self.key @@ -814,8 +858,9 @@ class Media(models.Model): return fields.get(self.source.source_type, '') def iter_formats(self): + parser = self.FORMAT_PARSER.get(self.source.source_type) for fmt in self.formats: - yield parse_media_format(fmt) + yield parser(fmt) def get_best_combined_format(self): return get_best_combined_format(self) @@ -1018,13 +1063,20 @@ class Media(models.Model): @property def description(self): + if self.source.source_type == Source.SOURCE_TYPE_TWITCH_CHANNEL: + chapters = self.loaded_metadata.get('chapters', []) + descriptions = [chapter['title'] for chapter in chapters] + return ' - '.join(descriptions) + field = self.get_metadata_field('description') - return self.loaded_metadata.get(field, '').strip() + description = self.loaded_metadata.get(field) or '' + return description.strip() @property def title(self): field = self.get_metadata_field('title') - return self.loaded_metadata.get(field, '').strip() + title = self.loaded_metadata.get(field) or '' + return title.strip() @property def slugtitle(self): @@ -1034,7 +1086,8 @@ class Media(models.Model): @property def thumbnail(self): field = self.get_metadata_field('thumbnail') - return self.loaded_metadata.get(field, '').strip() + thumbnail = self.loaded_metadata.get(field) or '' + return thumbnail.strip() @property def name(self): @@ -1207,6 +1260,13 @@ class Media(models.Model): else: return 'video/mp4' + @property + def is_live(self): + field = self.get_metadata_field('is_live') + is_live = self.loaded_metadata.get(field) + is_live = True if is_live else False + return is_live + @property def nfoxml(self): ''' @@ -1318,6 +1378,8 @@ class Media(models.Model): return self.STATE_SKIPPED if not self.source.download_media: return self.STATE_DISABLED_AT_SOURCE + if self.is_live: + return self.STATE_LIVE return self.STATE_UNKNOWN def get_download_state_icon(self, task=None): diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 6800a2f..8883f92 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -1,8 +1,10 @@ +from datetime import timedelta import os from django.conf import settings from django.db.models.signals import pre_save, post_save, pre_delete, post_delete from django.dispatch import receiver from django.utils.translation import gettext_lazy as _ +from django.utils import timezone from background_task.signals import task_failed from background_task.models import Task from common.logger import log @@ -150,20 +152,28 @@ def media_post_save(sender, instance, created, **kwargs): post_save.disconnect(media_post_save, sender=Media) instance.save() post_save.connect(media_post_save, sender=Media) - # If the media is missing metadata schedule it to be downloaded - if not instance.metadata: - log.info(f'Scheduling task to download metadata for: {instance.url}') + # If the media is missing metadata schedule it to be downloaded, + # or refresh it if it was live during last run + if not instance.metadata or instance.is_live: + kwargs = {} + if instance.is_live: + kwargs['schedule'] = timedelta(hours=1) + + log.info(f'Recheduling task to download metadata for live stream in 1h: {instance.url}') + else: + log.info(f'Scheduling task to download metadata for: {instance.url}') verbose_name = _('Downloading metadata for "{}"') download_media_metadata( str(instance.pk), priority=10, verbose_name=verbose_name.format(instance.pk), - remove_existing_tasks=True + remove_existing_tasks=True, + **kwargs ) # If the media is missing a thumbnail schedule it to be downloaded if not instance.thumb_file_exists: instance.thumb = None - if not instance.thumb: + if not instance.thumb and not instance.is_live: thumbnail_url = instance.thumbnail if thumbnail_url: log.info(f'Scheduling task to download thumbnail for: {instance.name} ' @@ -177,12 +187,13 @@ def media_post_save(sender, instance, created, **kwargs): verbose_name=verbose_name.format(instance.name), remove_existing_tasks=True ) + # If the media has not yet been downloaded schedule it to be downloaded if not instance.media_file_exists: instance.downloaded = False instance.media_file = None if (not instance.downloaded and instance.can_download and not instance.skip - and instance.source.download_media): + and instance.source.download_media and not instance.is_live): delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) verbose_name = _('Downloading media for "{}"') download_media( diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index def7529..409de4e 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -229,6 +229,7 @@ def download_media_metadata(media_id): source = media.source metadata = media.index_metadata() media.metadata = json.dumps(metadata, default=json_serial) + media.last_crawl = timezone.now() upload_date = media.upload_date # Media must have a valid upload date if upload_date: diff --git a/tubesync/sync/templates/sync/sources.html b/tubesync/sync/templates/sync/sources.html index 0c9a2ee..3c87cf0 100644 --- a/tubesync/sync/templates/sync/sources.html +++ b/tubesync/sync/templates/sync/sources.html @@ -19,6 +19,9 @@
+