twitch support

This commit is contained in:
Laurent DEFERT 2022-12-28 17:16:39 +01:00 committed by Laurent Defert
parent 72c3242e70
commit f8b86f666e
9 changed files with 225 additions and 17 deletions

View File

@ -16,10 +16,14 @@ class SourceAdmin(admin.ModelAdmin):
class MediaAdmin(admin.ModelAdmin): class MediaAdmin(admin.ModelAdmin):
ordering = ('-created',) ordering = ('-created',)
list_display = ('uuid', 'key', 'source', 'can_download', 'skip', 'downloaded') list_display = ('uuid', 'key', 'source', 'can_download', 'skip', 'is_live', 'downloaded')
readonly_fields = ('uuid', 'created') readonly_fields = ('uuid', 'created')
search_fields = ('uuid', 'source__key', 'key') search_fields = ('uuid', 'source__key', 'key')
@admin.display(boolean=True)
def is_live(self, obj):
return obj.is_live
@admin.register(MediaServer) @admin.register(MediaServer)
class MediaServerAdmin(admin.ModelAdmin): class MediaServerAdmin(admin.ModelAdmin):

View File

@ -0,0 +1,23 @@
# Generated by Django 3.2.18 on 2023-03-28 15:40
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sync', '0017_alter_source_sponsorblock_categories'),
]
operations = [
migrations.AddField(
model_name='media',
name='last_crawl',
field=models.DateTimeField(blank=True, db_index=True, help_text='Date and time the metadata of the media was last crawled', null=True, verbose_name='last crawl'),
),
migrations.AlterField(
model_name='source',
name='source_type',
field=models.CharField(choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist'), ('t', 'Twitch channel')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type'),
),
]

View File

@ -14,8 +14,9 @@ from django.utils.translation import gettext_lazy as _
from common.errors import NoFormatException from common.errors import NoFormatException
from common.utils import clean_filename from common.utils import clean_filename
from .youtube import (get_media_info as get_youtube_media_info, from .youtube import (get_media_info as get_youtube_media_info,
download_media as download_youtube_media) download_media as download_youtube_media,
from .utils import seconds_to_timestr, parse_media_format get_twitch_media_info)
from .utils import seconds_to_timestr, parse_media_format, parse_twitch_media_format
from .matching import (get_best_combined_format, get_best_audio_format, from .matching import (get_best_combined_format, get_best_audio_format,
get_best_video_format) get_best_video_format)
from .mediaservers import PlexMediaServer from .mediaservers import PlexMediaServer
@ -32,12 +33,14 @@ class Source(models.Model):
SOURCE_TYPE_YOUTUBE_CHANNEL = 'c' SOURCE_TYPE_YOUTUBE_CHANNEL = 'c'
SOURCE_TYPE_YOUTUBE_CHANNEL_ID = 'i' SOURCE_TYPE_YOUTUBE_CHANNEL_ID = 'i'
SOURCE_TYPE_YOUTUBE_PLAYLIST = 'p' SOURCE_TYPE_YOUTUBE_PLAYLIST = 'p'
SOURCE_TYPE_TWITCH_CHANNEL = 't'
SOURCE_TYPES = (SOURCE_TYPE_YOUTUBE_CHANNEL, SOURCE_TYPE_YOUTUBE_CHANNEL_ID, SOURCE_TYPES = (SOURCE_TYPE_YOUTUBE_CHANNEL, SOURCE_TYPE_YOUTUBE_CHANNEL_ID,
SOURCE_TYPE_YOUTUBE_PLAYLIST) SOURCE_TYPE_YOUTUBE_PLAYLIST, SOURCE_TYPE_TWITCH_CHANNEL)
SOURCE_TYPE_CHOICES = ( SOURCE_TYPE_CHOICES = (
(SOURCE_TYPE_YOUTUBE_CHANNEL, _('YouTube channel')), (SOURCE_TYPE_YOUTUBE_CHANNEL, _('YouTube channel')),
(SOURCE_TYPE_YOUTUBE_CHANNEL_ID, _('YouTube channel by ID')), (SOURCE_TYPE_YOUTUBE_CHANNEL_ID, _('YouTube channel by ID')),
(SOURCE_TYPE_YOUTUBE_PLAYLIST, _('YouTube playlist')), (SOURCE_TYPE_YOUTUBE_PLAYLIST, _('YouTube playlist')),
(SOURCE_TYPE_TWITCH_CHANNEL, _('Twitch channel')),
) )
SOURCE_RESOLUTION_360P = '360p' SOURCE_RESOLUTION_360P = '360p'
@ -151,30 +154,35 @@ class Source(models.Model):
SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>', SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: '<i class="fab fa-youtube"></i>', SOURCE_TYPE_YOUTUBE_CHANNEL_ID: '<i class="fab fa-youtube"></i>',
SOURCE_TYPE_YOUTUBE_PLAYLIST: '<i class="fab fa-youtube"></i>', SOURCE_TYPE_YOUTUBE_PLAYLIST: '<i class="fab fa-youtube"></i>',
SOURCE_TYPE_TWITCH_CHANNEL: '<i class="fab fa-twitch"></i>',
} }
# Format to use to display a URL for the source # Format to use to display a URL for the source
URLS = { URLS = {
SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}', SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}',
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}', SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}',
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}', SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/{key}/videos',
} }
# Format used to create indexable URLs # Format used to create indexable URLs
INDEX_URLS = { INDEX_URLS = {
SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos', SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos',
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos', SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos',
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}', SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/{key}/videos?filter=all&sort=time',
} }
# Callback functions to get a list of media from the source # Callback functions to get a list of media from the source
INDEXERS = { INDEXERS = {
SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info, SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info, SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info, SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
SOURCE_TYPE_TWITCH_CHANNEL: get_twitch_media_info,
} }
# Field names to find the media ID used as the key when storing media # Field names to find the media ID used as the key when storing media
KEY_FIELD = { KEY_FIELD = {
SOURCE_TYPE_YOUTUBE_CHANNEL: 'id', SOURCE_TYPE_YOUTUBE_CHANNEL: 'id',
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'id', SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'id',
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id', SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id',
SOURCE_TYPE_TWITCH_CHANNEL: 'id',
} }
class CapChoices(models.IntegerChoices): class CapChoices(models.IntegerChoices):
@ -546,12 +554,21 @@ class Media(models.Model):
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/watch?v={key}', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/watch?v={key}',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/watch?v={key}', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/watch?v={key}',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/videos/{key}',
} }
# Callback functions to get a list of media from the source # Callback functions to get a list of media from the source
INDEXERS = { INDEXERS = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info, Source.SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info, Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info, Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
Source.SOURCE_TYPE_TWITCH_CHANNEL: get_youtube_media_info,
}
# Callback functions to get a list of media from the source
FORMAT_PARSER = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: parse_media_format,
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: parse_media_format,
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: parse_media_format,
Source.SOURCE_TYPE_TWITCH_CHANNEL: parse_twitch_media_format,
} }
# Maps standardised names to names used in source metdata # Maps standardised names to names used in source metdata
METADATA_FIELDS = { METADATA_FIELDS = {
@ -559,16 +576,19 @@ class Media(models.Model):
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'upload_date', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'upload_date',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'upload_date', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'upload_date',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'upload_date', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'upload_date',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'upload_date',
}, },
'title': { 'title': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'title', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'title',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'title', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'title',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'title', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'title',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'title',
}, },
'thumbnail': { 'thumbnail': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'thumbnail', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'thumbnail',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'thumbnail', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'thumbnail',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'thumbnail', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'thumbnail',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'thumbnail',
}, },
'description': { 'description': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'description', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'description',
@ -579,46 +599,61 @@ class Media(models.Model):
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'duration', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'duration',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'duration', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'duration',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'duration', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'duration',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'duration',
}, },
'formats': { 'formats': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'formats', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'formats',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'formats', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'formats',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'formats',
}, },
'categories': { 'categories': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'categories', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'categories',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'categories', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'categories',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'categories', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'categories',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'categories',
}, },
'rating': { 'rating': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'average_rating', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'average_rating',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'average_rating', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'average_rating',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'average_rating', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'average_rating',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'average_rating',
}, },
'age_limit': { 'age_limit': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'age_limit', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'age_limit',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'age_limit', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'age_limit',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'age_limit', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'age_limit',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'age_limit',
}, },
'uploader': { 'uploader': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'uploader', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'uploader',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'uploader', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'uploader',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'uploader', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'uploader',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'uploader',
}, },
'upvotes': { 'upvotes': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'like_count', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'like_count',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'like_count', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'like_count',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'like_count', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'like_count',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'like_count',
}, },
'downvotes': { 'downvotes': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'dislike_count', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'dislike_count',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'dislike_count',
}, },
'playlist_title': { 'playlist_title': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'playlist_title', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'playlist_title',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'playlist_title',
},
'is_live': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'is_live',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'is_live',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'is_live',
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'is_live',
}, },
} }
STATE_UNKNOWN = 'unknown' STATE_UNKNOWN = 'unknown'
@ -626,6 +661,7 @@ class Media(models.Model):
STATE_DOWNLOADING = 'downloading' STATE_DOWNLOADING = 'downloading'
STATE_DOWNLOADED = 'downloaded' STATE_DOWNLOADED = 'downloaded'
STATE_SKIPPED = 'skipped' STATE_SKIPPED = 'skipped'
STATE_LIVE = 'live'
STATE_DISABLED_AT_SOURCE = 'source-disabled' STATE_DISABLED_AT_SOURCE = 'source-disabled'
STATE_ERROR = 'error' STATE_ERROR = 'error'
STATES = (STATE_UNKNOWN, STATE_SCHEDULED, STATE_DOWNLOADING, STATE_DOWNLOADED, STATES = (STATE_UNKNOWN, STATE_SCHEDULED, STATE_DOWNLOADING, STATE_DOWNLOADED,
@ -638,6 +674,7 @@ class Media(models.Model):
STATE_SKIPPED: '<i class="fas fa-exclamation-circle" title="Skipped"></i>', STATE_SKIPPED: '<i class="fas fa-exclamation-circle" title="Skipped"></i>',
STATE_DISABLED_AT_SOURCE: '<i class="fas fa-stop-circle" title="Media downloading disabled at source"></i>', STATE_DISABLED_AT_SOURCE: '<i class="fas fa-stop-circle" title="Media downloading disabled at source"></i>',
STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>', STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>',
STATE_LIVE: '<i class="fas fa-clock" title="Live, download postponed"></i>',
} }
uuid = models.UUIDField( uuid = models.UUIDField(
@ -798,6 +835,13 @@ class Media(models.Model):
null=True, null=True,
help_text=_('Size of the downloaded media in bytes') help_text=_('Size of the downloaded media in bytes')
) )
last_crawl = models.DateTimeField(
_('last crawl'),
db_index=True,
null=True,
blank=True,
help_text=_('Date and time the metadata of the media was last crawled')
)
def __str__(self): def __str__(self):
return self.key return self.key
@ -814,8 +858,9 @@ class Media(models.Model):
return fields.get(self.source.source_type, '') return fields.get(self.source.source_type, '')
def iter_formats(self): def iter_formats(self):
parser = self.FORMAT_PARSER.get(self.source.source_type)
for fmt in self.formats: for fmt in self.formats:
yield parse_media_format(fmt) yield parser(fmt)
def get_best_combined_format(self): def get_best_combined_format(self):
return get_best_combined_format(self) return get_best_combined_format(self)
@ -1018,13 +1063,20 @@ class Media(models.Model):
@property @property
def description(self): def description(self):
if self.source.source_type == Source.SOURCE_TYPE_TWITCH_CHANNEL:
chapters = self.loaded_metadata.get('chapters', [])
descriptions = [chapter['title'] for chapter in chapters]
return ' - '.join(descriptions)
field = self.get_metadata_field('description') field = self.get_metadata_field('description')
return self.loaded_metadata.get(field, '').strip() description = self.loaded_metadata.get(field) or ''
return description.strip()
@property @property
def title(self): def title(self):
field = self.get_metadata_field('title') field = self.get_metadata_field('title')
return self.loaded_metadata.get(field, '').strip() title = self.loaded_metadata.get(field) or ''
return title.strip()
@property @property
def slugtitle(self): def slugtitle(self):
@ -1034,7 +1086,8 @@ class Media(models.Model):
@property @property
def thumbnail(self): def thumbnail(self):
field = self.get_metadata_field('thumbnail') field = self.get_metadata_field('thumbnail')
return self.loaded_metadata.get(field, '').strip() thumbnail = self.loaded_metadata.get(field) or ''
return thumbnail.strip()
@property @property
def name(self): def name(self):
@ -1207,6 +1260,13 @@ class Media(models.Model):
else: else:
return 'video/mp4' return 'video/mp4'
@property
def is_live(self):
field = self.get_metadata_field('is_live')
is_live = self.loaded_metadata.get(field)
is_live = True if is_live else False
return is_live
@property @property
def nfoxml(self): def nfoxml(self):
''' '''
@ -1318,6 +1378,8 @@ class Media(models.Model):
return self.STATE_SKIPPED return self.STATE_SKIPPED
if not self.source.download_media: if not self.source.download_media:
return self.STATE_DISABLED_AT_SOURCE return self.STATE_DISABLED_AT_SOURCE
if self.is_live:
return self.STATE_LIVE
return self.STATE_UNKNOWN return self.STATE_UNKNOWN
def get_download_state_icon(self, task=None): def get_download_state_icon(self, task=None):

View File

@ -1,8 +1,10 @@
from datetime import timedelta
import os import os
from django.conf import settings from django.conf import settings
from django.db.models.signals import pre_save, post_save, pre_delete, post_delete from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
from django.dispatch import receiver from django.dispatch import receiver
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from django.utils import timezone
from background_task.signals import task_failed from background_task.signals import task_failed
from background_task.models import Task from background_task.models import Task
from common.logger import log from common.logger import log
@ -150,20 +152,28 @@ def media_post_save(sender, instance, created, **kwargs):
post_save.disconnect(media_post_save, sender=Media) post_save.disconnect(media_post_save, sender=Media)
instance.save() instance.save()
post_save.connect(media_post_save, sender=Media) post_save.connect(media_post_save, sender=Media)
# If the media is missing metadata schedule it to be downloaded # If the media is missing metadata schedule it to be downloaded,
if not instance.metadata: # or refresh it if it was live during last run
log.info(f'Scheduling task to download metadata for: {instance.url}') if not instance.metadata or instance.is_live:
kwargs = {}
if instance.is_live:
kwargs['schedule'] = timedelta(hours=1)
log.info(f'Recheduling task to download metadata for live stream in 1h: {instance.url}')
else:
log.info(f'Scheduling task to download metadata for: {instance.url}')
verbose_name = _('Downloading metadata for "{}"') verbose_name = _('Downloading metadata for "{}"')
download_media_metadata( download_media_metadata(
str(instance.pk), str(instance.pk),
priority=10, priority=10,
verbose_name=verbose_name.format(instance.pk), verbose_name=verbose_name.format(instance.pk),
remove_existing_tasks=True remove_existing_tasks=True,
**kwargs
) )
# If the media is missing a thumbnail schedule it to be downloaded # If the media is missing a thumbnail schedule it to be downloaded
if not instance.thumb_file_exists: if not instance.thumb_file_exists:
instance.thumb = None instance.thumb = None
if not instance.thumb: if not instance.thumb and not instance.is_live:
thumbnail_url = instance.thumbnail thumbnail_url = instance.thumbnail
if thumbnail_url: if thumbnail_url:
log.info(f'Scheduling task to download thumbnail for: {instance.name} ' log.info(f'Scheduling task to download thumbnail for: {instance.name} '
@ -177,12 +187,13 @@ def media_post_save(sender, instance, created, **kwargs):
verbose_name=verbose_name.format(instance.name), verbose_name=verbose_name.format(instance.name),
remove_existing_tasks=True remove_existing_tasks=True
) )
# If the media has not yet been downloaded schedule it to be downloaded # If the media has not yet been downloaded schedule it to be downloaded
if not instance.media_file_exists: if not instance.media_file_exists:
instance.downloaded = False instance.downloaded = False
instance.media_file = None instance.media_file = None
if (not instance.downloaded and instance.can_download and not instance.skip if (not instance.downloaded and instance.can_download and not instance.skip
and instance.source.download_media): and instance.source.download_media and not instance.is_live):
delete_task_by_media('sync.tasks.download_media', (str(instance.pk),)) delete_task_by_media('sync.tasks.download_media', (str(instance.pk),))
verbose_name = _('Downloading media for "{}"') verbose_name = _('Downloading media for "{}"')
download_media( download_media(

View File

@ -229,6 +229,7 @@ def download_media_metadata(media_id):
source = media.source source = media.source
metadata = media.index_metadata() metadata = media.index_metadata()
media.metadata = json.dumps(metadata, default=json_serial) media.metadata = json.dumps(metadata, default=json_serial)
media.last_crawl = timezone.now()
upload_date = media.upload_date upload_date = media.upload_date
# Media must have a valid upload date # Media must have a valid upload date
if upload_date: if upload_date:

View File

@ -19,6 +19,9 @@
<div class="col m12 xl4 margin-bottom"> <div class="col m12 xl4 margin-bottom">
<a href="{% url 'sync:validate-source' source_type='youtube-playlist' %}" class="btn">Add a YouTube playlist <i class="fab fa-youtube"></i></a> <a href="{% url 'sync:validate-source' source_type='youtube-playlist' %}" class="btn">Add a YouTube playlist <i class="fab fa-youtube"></i></a>
</div> </div>
<div class="col m12 xl4 margin-bottom">
<a href="{% url 'sync:validate-source' source_type='twitch-channel' %}" class="btn">Add a Twitch channel <i class="fab fa-twitch"></i></a>
</div>
</div> </div>
<div class="row no-margin-bottom"> <div class="row no-margin-bottom">
<div class="col s12"> <div class="col s12">

View File

@ -199,3 +199,61 @@ def parse_media_format(format_dict):
'is_hls': is_hls, 'is_hls': is_hls,
'is_dash': is_dash, 'is_dash': is_dash,
} }
def parse_twitch_media_format(format_dict):
'''
This parser primarily adapts the format dict returned by youtube-dl into a
standard form used by the matchers in matching.py. If youtube-dl changes
any internals, update it here.
'''
vcodec_full = format_dict.get('vcodec', '')
vcodec_parts = vcodec_full.split('.')
if len(vcodec_parts) > 0:
vcodec = vcodec_parts[0].strip().upper()
else:
vcodec = None
if vcodec == 'NONE':
vcodec = None
acodec_full = format_dict.get('acodec', '')
acodec_parts = acodec_full.split('.')
if len(acodec_parts) > 0:
acodec = acodec_parts[0].strip().upper()
else:
acodec = None
if acodec == 'NONE':
acodec = None
try:
fps = int(format_dict.get('fps', 0))
except (ValueError, TypeError):
fps = 0
if format_dict.get('format_id').endswith('p60'):
fps = 60
height = format_dict.get('height', 0)
try:
height = int(height)
except (ValueError, TypeError):
height = 0
width = format_dict.get('width', 0)
try:
width = int(width)
except (ValueError, TypeError):
width = 0
format_full = format_dict.get('format_note', '').strip().upper()
return {
'id': format_dict.get('format_id', ''),
'format': f'{height}P',
'format_verbose': format_dict.get('format', ''),
'height': height,
'width': width,
'vcodec': vcodec,
'fps': fps,
'vbr': format_dict.get('tbr', 0),
'acodec': acodec,
'abr': format_dict.get('abr', 0),
'is_60fps': fps > 50,
'is_hdr': False,
'is_hls': True,
'is_dash': False,
}

View File

@ -157,11 +157,13 @@ class ValidateSourceView(FormView):
'youtube-channel': Source.SOURCE_TYPE_YOUTUBE_CHANNEL, 'youtube-channel': Source.SOURCE_TYPE_YOUTUBE_CHANNEL,
'youtube-channel-id': Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID, 'youtube-channel-id': Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID,
'youtube-playlist': Source.SOURCE_TYPE_YOUTUBE_PLAYLIST, 'youtube-playlist': Source.SOURCE_TYPE_YOUTUBE_PLAYLIST,
'twitch-channel': Source.SOURCE_TYPE_TWITCH_CHANNEL,
} }
help_item = { help_item = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _('YouTube channel'), Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _('YouTube channel'),
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: _('YouTube channel ID'), Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: _('YouTube channel ID'),
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: _('YouTube playlist'), Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: _('YouTube playlist'),
Source.SOURCE_TYPE_TWITCH_CHANNEL: _('Twitch channel'),
} }
help_texts = { help_texts = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _( Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _(
@ -183,13 +185,20 @@ class ValidateSourceView(FormView):
'BiGLoNgUnIqUeId</strong> where <strong>BiGLoNgUnIqUeId</strong> is the ' 'BiGLoNgUnIqUeId</strong> where <strong>BiGLoNgUnIqUeId</strong> is the '
'unique ID of the playlist you want to add.' 'unique ID of the playlist you want to add.'
), ),
Source.SOURCE_TYPE_TWITCH_CHANNEL: _(
'Enter a Twitch channel URL into the box below. The URL will be '
'in the format of <strong>https://www.twitch.tv/ChannelId/videos'
'</strong> where <strong>ChannelId</strong> is the '
'is the channel you want to add.'
),
} }
help_examples = { help_examples = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/google', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/google',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: ('https://www.youtube.com/channel/' Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: ('https://www.youtube.com/channel/'
'UCK8sQmJBp8GCxrOtXWBpyEA'), 'UCK8sQmJBp8GCxrOtXWBpyEA'),
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('https://www.youtube.com/playlist?list=' Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('https://www.youtube.com/playlist?list='
'PL590L5WQmH8dpP0RyH5pCfIaDEdt9nk7r') 'PL590L5WQmH8dpP0RyH5pCfIaDEdt9nk7r'),
Source.SOURCE_TYPE_TWITCH_CHANNEL: ('https://www.twitch.tv/twitch/videos')
} }
validation_urls = { validation_urls = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: {
@ -219,11 +228,21 @@ class ValidateSourceView(FormView):
'extract_key': ('qs_args', 'list'), 'extract_key': ('qs_args', 'list'),
'example': 'https://www.youtube.com/playlist?list=PLAYLISTID' 'example': 'https://www.youtube.com/playlist?list=PLAYLISTID'
}, },
Source.SOURCE_TYPE_TWITCH_CHANNEL: {
'scheme': 'https',
'domains': ('www.twitch.tv',),
'path_regex': '^\/([^\/]+)(\/videos)?$',
'path_must_not_match': (),
'qs_args': [],
'extract_key': ('path_regex', 0),
'example': 'https://www.twitch.tv/twitch/videos'
},
} }
prepopulate_fields = { prepopulate_fields = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: ('source_type', 'key', 'name', 'directory'), Source.SOURCE_TYPE_YOUTUBE_CHANNEL: ('source_type', 'key', 'name', 'directory'),
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: ('source_type', 'key'), Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: ('source_type', 'key'),
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('source_type', 'key'), Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('source_type', 'key'),
Source.SOURCE_TYPE_TWITCH_CHANNEL: ('source_type', 'key', 'name', 'directory'),
} }
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
@ -305,13 +324,30 @@ class EditSourceMixin:
'errors or is empty. Check the table at the end of ' 'errors or is empty. Check the table at the end of '
'this page for valid media name variables'), 'this page for valid media name variables'),
'dir_outside_dlroot': _('You cannot specify a directory outside of the ' 'dir_outside_dlroot': _('You cannot specify a directory outside of the '
'base directory (%BASEDIR%)') 'base directory (%BASEDIR%)'),
'not_twitch_vcodec': _('Twitch only supports AVC1 source video codec'),
'not_twitch_acodec': _('Twitch only supports MP4A source audio codec')
} }
def form_valid(self, form: Form): def form_valid(self, form: Form):
# Perform extra validation to make sure the media_format is valid # Perform extra validation to make sure the media_format is valid
obj = form.save(commit=False) obj = form.save(commit=False)
source_type = form.cleaned_data['media_format'] source_type = form.cleaned_data['source_type']
source_vcodec = form.cleaned_data['source_vcodec']
source_acodec = form.cleaned_data['source_acodec']
if source_type == Source.SOURCE_TYPE_TWITCH_CHANNEL and source_vcodec != Source.SOURCE_VCODEC_AVC1:
form.add_error(
'source_vcodec',
ValidationError(self.errors['not_twitch_vcodec'])
)
if source_type == Source.SOURCE_TYPE_TWITCH_CHANNEL and source_acodec != Source.SOURCE_ACODEC_MP4A:
form.add_error(
'source_acodec',
ValidationError(self.errors['not_twitch_acodec'])
)
example_media_file = obj.get_example_media_format() example_media_file = obj.get_example_media_format()
if example_media_file == '': if example_media_file == '':

View File

@ -64,6 +64,16 @@ def get_media_info(url):
return response return response
def get_twitch_media_info(url):
'''
Format Twitch id's to match the id key used in urls
'''
response = get_media_info(url)
for entry in response['entries']:
entry['id'] = entry['id'].lstrip('v')
return response
def download_media(url, media_format, extension, output_file, info_json, def download_media(url, media_format, extension, output_file, info_json,
sponsor_categories="all", sponsor_categories="all",
embed_thumbnail=False, embed_metadata=False, skip_sponsors=True): embed_thumbnail=False, embed_metadata=False, skip_sponsors=True):