twitch support
This commit is contained in:
parent
72c3242e70
commit
f8b86f666e
|
@ -16,10 +16,14 @@ class SourceAdmin(admin.ModelAdmin):
|
||||||
class MediaAdmin(admin.ModelAdmin):
|
class MediaAdmin(admin.ModelAdmin):
|
||||||
|
|
||||||
ordering = ('-created',)
|
ordering = ('-created',)
|
||||||
list_display = ('uuid', 'key', 'source', 'can_download', 'skip', 'downloaded')
|
list_display = ('uuid', 'key', 'source', 'can_download', 'skip', 'is_live', 'downloaded')
|
||||||
readonly_fields = ('uuid', 'created')
|
readonly_fields = ('uuid', 'created')
|
||||||
search_fields = ('uuid', 'source__key', 'key')
|
search_fields = ('uuid', 'source__key', 'key')
|
||||||
|
|
||||||
|
@admin.display(boolean=True)
|
||||||
|
def is_live(self, obj):
|
||||||
|
return obj.is_live
|
||||||
|
|
||||||
|
|
||||||
@admin.register(MediaServer)
|
@admin.register(MediaServer)
|
||||||
class MediaServerAdmin(admin.ModelAdmin):
|
class MediaServerAdmin(admin.ModelAdmin):
|
||||||
|
|
|
@ -0,0 +1,23 @@
|
||||||
|
# Generated by Django 3.2.18 on 2023-03-28 15:40
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('sync', '0017_alter_source_sponsorblock_categories'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='media',
|
||||||
|
name='last_crawl',
|
||||||
|
field=models.DateTimeField(blank=True, db_index=True, help_text='Date and time the metadata of the media was last crawled', null=True, verbose_name='last crawl'),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='source',
|
||||||
|
name='source_type',
|
||||||
|
field=models.CharField(choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist'), ('t', 'Twitch channel')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type'),
|
||||||
|
),
|
||||||
|
]
|
|
@ -14,8 +14,9 @@ from django.utils.translation import gettext_lazy as _
|
||||||
from common.errors import NoFormatException
|
from common.errors import NoFormatException
|
||||||
from common.utils import clean_filename
|
from common.utils import clean_filename
|
||||||
from .youtube import (get_media_info as get_youtube_media_info,
|
from .youtube import (get_media_info as get_youtube_media_info,
|
||||||
download_media as download_youtube_media)
|
download_media as download_youtube_media,
|
||||||
from .utils import seconds_to_timestr, parse_media_format
|
get_twitch_media_info)
|
||||||
|
from .utils import seconds_to_timestr, parse_media_format, parse_twitch_media_format
|
||||||
from .matching import (get_best_combined_format, get_best_audio_format,
|
from .matching import (get_best_combined_format, get_best_audio_format,
|
||||||
get_best_video_format)
|
get_best_video_format)
|
||||||
from .mediaservers import PlexMediaServer
|
from .mediaservers import PlexMediaServer
|
||||||
|
@ -32,12 +33,14 @@ class Source(models.Model):
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL = 'c'
|
SOURCE_TYPE_YOUTUBE_CHANNEL = 'c'
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL_ID = 'i'
|
SOURCE_TYPE_YOUTUBE_CHANNEL_ID = 'i'
|
||||||
SOURCE_TYPE_YOUTUBE_PLAYLIST = 'p'
|
SOURCE_TYPE_YOUTUBE_PLAYLIST = 'p'
|
||||||
|
SOURCE_TYPE_TWITCH_CHANNEL = 't'
|
||||||
SOURCE_TYPES = (SOURCE_TYPE_YOUTUBE_CHANNEL, SOURCE_TYPE_YOUTUBE_CHANNEL_ID,
|
SOURCE_TYPES = (SOURCE_TYPE_YOUTUBE_CHANNEL, SOURCE_TYPE_YOUTUBE_CHANNEL_ID,
|
||||||
SOURCE_TYPE_YOUTUBE_PLAYLIST)
|
SOURCE_TYPE_YOUTUBE_PLAYLIST, SOURCE_TYPE_TWITCH_CHANNEL)
|
||||||
SOURCE_TYPE_CHOICES = (
|
SOURCE_TYPE_CHOICES = (
|
||||||
(SOURCE_TYPE_YOUTUBE_CHANNEL, _('YouTube channel')),
|
(SOURCE_TYPE_YOUTUBE_CHANNEL, _('YouTube channel')),
|
||||||
(SOURCE_TYPE_YOUTUBE_CHANNEL_ID, _('YouTube channel by ID')),
|
(SOURCE_TYPE_YOUTUBE_CHANNEL_ID, _('YouTube channel by ID')),
|
||||||
(SOURCE_TYPE_YOUTUBE_PLAYLIST, _('YouTube playlist')),
|
(SOURCE_TYPE_YOUTUBE_PLAYLIST, _('YouTube playlist')),
|
||||||
|
(SOURCE_TYPE_TWITCH_CHANNEL, _('Twitch channel')),
|
||||||
)
|
)
|
||||||
|
|
||||||
SOURCE_RESOLUTION_360P = '360p'
|
SOURCE_RESOLUTION_360P = '360p'
|
||||||
|
@ -151,30 +154,35 @@ class Source(models.Model):
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
|
SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: '<i class="fab fa-youtube"></i>',
|
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: '<i class="fab fa-youtube"></i>',
|
||||||
SOURCE_TYPE_YOUTUBE_PLAYLIST: '<i class="fab fa-youtube"></i>',
|
SOURCE_TYPE_YOUTUBE_PLAYLIST: '<i class="fab fa-youtube"></i>',
|
||||||
|
SOURCE_TYPE_TWITCH_CHANNEL: '<i class="fab fa-twitch"></i>',
|
||||||
}
|
}
|
||||||
# Format to use to display a URL for the source
|
# Format to use to display a URL for the source
|
||||||
URLS = {
|
URLS = {
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}',
|
SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}',
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}',
|
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}',
|
||||||
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
|
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
|
||||||
|
SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/{key}/videos',
|
||||||
}
|
}
|
||||||
# Format used to create indexable URLs
|
# Format used to create indexable URLs
|
||||||
INDEX_URLS = {
|
INDEX_URLS = {
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos',
|
SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos',
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos',
|
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos',
|
||||||
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
|
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
|
||||||
|
SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/{key}/videos?filter=all&sort=time',
|
||||||
}
|
}
|
||||||
# Callback functions to get a list of media from the source
|
# Callback functions to get a list of media from the source
|
||||||
INDEXERS = {
|
INDEXERS = {
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
|
SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
|
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
|
||||||
SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
|
SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
|
||||||
|
SOURCE_TYPE_TWITCH_CHANNEL: get_twitch_media_info,
|
||||||
}
|
}
|
||||||
# Field names to find the media ID used as the key when storing media
|
# Field names to find the media ID used as the key when storing media
|
||||||
KEY_FIELD = {
|
KEY_FIELD = {
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL: 'id',
|
SOURCE_TYPE_YOUTUBE_CHANNEL: 'id',
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'id',
|
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'id',
|
||||||
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id',
|
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id',
|
||||||
|
SOURCE_TYPE_TWITCH_CHANNEL: 'id',
|
||||||
}
|
}
|
||||||
|
|
||||||
class CapChoices(models.IntegerChoices):
|
class CapChoices(models.IntegerChoices):
|
||||||
|
@ -546,12 +554,21 @@ class Media(models.Model):
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/watch?v={key}',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/watch?v={key}',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/watch?v={key}',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/watch?v={key}',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/videos/{key}',
|
||||||
}
|
}
|
||||||
# Callback functions to get a list of media from the source
|
# Callback functions to get a list of media from the source
|
||||||
INDEXERS = {
|
INDEXERS = {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: get_youtube_media_info,
|
||||||
|
}
|
||||||
|
# Callback functions to get a list of media from the source
|
||||||
|
FORMAT_PARSER = {
|
||||||
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: parse_media_format,
|
||||||
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: parse_media_format,
|
||||||
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: parse_media_format,
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: parse_twitch_media_format,
|
||||||
}
|
}
|
||||||
# Maps standardised names to names used in source metdata
|
# Maps standardised names to names used in source metdata
|
||||||
METADATA_FIELDS = {
|
METADATA_FIELDS = {
|
||||||
|
@ -559,16 +576,19 @@ class Media(models.Model):
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'upload_date',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'upload_date',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'upload_date',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'upload_date',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'upload_date',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'upload_date',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'upload_date',
|
||||||
},
|
},
|
||||||
'title': {
|
'title': {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'title',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'title',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'title',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'title',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'title',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'title',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'title',
|
||||||
},
|
},
|
||||||
'thumbnail': {
|
'thumbnail': {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'thumbnail',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'thumbnail',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'thumbnail',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'thumbnail',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'thumbnail',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'thumbnail',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'thumbnail',
|
||||||
},
|
},
|
||||||
'description': {
|
'description': {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'description',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'description',
|
||||||
|
@ -579,46 +599,61 @@ class Media(models.Model):
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'duration',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'duration',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'duration',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'duration',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'duration',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'duration',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'duration',
|
||||||
},
|
},
|
||||||
'formats': {
|
'formats': {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'formats',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'formats',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'formats',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'formats',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'formats',
|
||||||
},
|
},
|
||||||
'categories': {
|
'categories': {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'categories',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'categories',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'categories',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'categories',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'categories',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'categories',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'categories',
|
||||||
},
|
},
|
||||||
'rating': {
|
'rating': {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'average_rating',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'average_rating',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'average_rating',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'average_rating',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'average_rating',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'average_rating',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'average_rating',
|
||||||
},
|
},
|
||||||
'age_limit': {
|
'age_limit': {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'age_limit',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'age_limit',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'age_limit',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'age_limit',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'age_limit',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'age_limit',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'age_limit',
|
||||||
},
|
},
|
||||||
'uploader': {
|
'uploader': {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'uploader',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'uploader',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'uploader',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'uploader',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'uploader',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'uploader',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'uploader',
|
||||||
},
|
},
|
||||||
'upvotes': {
|
'upvotes': {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'like_count',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'like_count',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'like_count',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'like_count',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'like_count',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'like_count',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'like_count',
|
||||||
},
|
},
|
||||||
'downvotes': {
|
'downvotes': {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'dislike_count',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'dislike_count',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'dislike_count',
|
||||||
},
|
},
|
||||||
'playlist_title': {
|
'playlist_title': {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'playlist_title',
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'playlist_title',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'playlist_title',
|
||||||
|
},
|
||||||
|
'is_live': {
|
||||||
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'is_live',
|
||||||
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'is_live',
|
||||||
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'is_live',
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: 'is_live',
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
STATE_UNKNOWN = 'unknown'
|
STATE_UNKNOWN = 'unknown'
|
||||||
|
@ -626,6 +661,7 @@ class Media(models.Model):
|
||||||
STATE_DOWNLOADING = 'downloading'
|
STATE_DOWNLOADING = 'downloading'
|
||||||
STATE_DOWNLOADED = 'downloaded'
|
STATE_DOWNLOADED = 'downloaded'
|
||||||
STATE_SKIPPED = 'skipped'
|
STATE_SKIPPED = 'skipped'
|
||||||
|
STATE_LIVE = 'live'
|
||||||
STATE_DISABLED_AT_SOURCE = 'source-disabled'
|
STATE_DISABLED_AT_SOURCE = 'source-disabled'
|
||||||
STATE_ERROR = 'error'
|
STATE_ERROR = 'error'
|
||||||
STATES = (STATE_UNKNOWN, STATE_SCHEDULED, STATE_DOWNLOADING, STATE_DOWNLOADED,
|
STATES = (STATE_UNKNOWN, STATE_SCHEDULED, STATE_DOWNLOADING, STATE_DOWNLOADED,
|
||||||
|
@ -638,6 +674,7 @@ class Media(models.Model):
|
||||||
STATE_SKIPPED: '<i class="fas fa-exclamation-circle" title="Skipped"></i>',
|
STATE_SKIPPED: '<i class="fas fa-exclamation-circle" title="Skipped"></i>',
|
||||||
STATE_DISABLED_AT_SOURCE: '<i class="fas fa-stop-circle" title="Media downloading disabled at source"></i>',
|
STATE_DISABLED_AT_SOURCE: '<i class="fas fa-stop-circle" title="Media downloading disabled at source"></i>',
|
||||||
STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>',
|
STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>',
|
||||||
|
STATE_LIVE: '<i class="fas fa-clock" title="Live, download postponed"></i>',
|
||||||
}
|
}
|
||||||
|
|
||||||
uuid = models.UUIDField(
|
uuid = models.UUIDField(
|
||||||
|
@ -798,6 +835,13 @@ class Media(models.Model):
|
||||||
null=True,
|
null=True,
|
||||||
help_text=_('Size of the downloaded media in bytes')
|
help_text=_('Size of the downloaded media in bytes')
|
||||||
)
|
)
|
||||||
|
last_crawl = models.DateTimeField(
|
||||||
|
_('last crawl'),
|
||||||
|
db_index=True,
|
||||||
|
null=True,
|
||||||
|
blank=True,
|
||||||
|
help_text=_('Date and time the metadata of the media was last crawled')
|
||||||
|
)
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return self.key
|
return self.key
|
||||||
|
@ -814,8 +858,9 @@ class Media(models.Model):
|
||||||
return fields.get(self.source.source_type, '')
|
return fields.get(self.source.source_type, '')
|
||||||
|
|
||||||
def iter_formats(self):
|
def iter_formats(self):
|
||||||
|
parser = self.FORMAT_PARSER.get(self.source.source_type)
|
||||||
for fmt in self.formats:
|
for fmt in self.formats:
|
||||||
yield parse_media_format(fmt)
|
yield parser(fmt)
|
||||||
|
|
||||||
def get_best_combined_format(self):
|
def get_best_combined_format(self):
|
||||||
return get_best_combined_format(self)
|
return get_best_combined_format(self)
|
||||||
|
@ -1018,13 +1063,20 @@ class Media(models.Model):
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def description(self):
|
def description(self):
|
||||||
|
if self.source.source_type == Source.SOURCE_TYPE_TWITCH_CHANNEL:
|
||||||
|
chapters = self.loaded_metadata.get('chapters', [])
|
||||||
|
descriptions = [chapter['title'] for chapter in chapters]
|
||||||
|
return ' - '.join(descriptions)
|
||||||
|
|
||||||
field = self.get_metadata_field('description')
|
field = self.get_metadata_field('description')
|
||||||
return self.loaded_metadata.get(field, '').strip()
|
description = self.loaded_metadata.get(field) or ''
|
||||||
|
return description.strip()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def title(self):
|
def title(self):
|
||||||
field = self.get_metadata_field('title')
|
field = self.get_metadata_field('title')
|
||||||
return self.loaded_metadata.get(field, '').strip()
|
title = self.loaded_metadata.get(field) or ''
|
||||||
|
return title.strip()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def slugtitle(self):
|
def slugtitle(self):
|
||||||
|
@ -1034,7 +1086,8 @@ class Media(models.Model):
|
||||||
@property
|
@property
|
||||||
def thumbnail(self):
|
def thumbnail(self):
|
||||||
field = self.get_metadata_field('thumbnail')
|
field = self.get_metadata_field('thumbnail')
|
||||||
return self.loaded_metadata.get(field, '').strip()
|
thumbnail = self.loaded_metadata.get(field) or ''
|
||||||
|
return thumbnail.strip()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def name(self):
|
def name(self):
|
||||||
|
@ -1207,6 +1260,13 @@ class Media(models.Model):
|
||||||
else:
|
else:
|
||||||
return 'video/mp4'
|
return 'video/mp4'
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_live(self):
|
||||||
|
field = self.get_metadata_field('is_live')
|
||||||
|
is_live = self.loaded_metadata.get(field)
|
||||||
|
is_live = True if is_live else False
|
||||||
|
return is_live
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def nfoxml(self):
|
def nfoxml(self):
|
||||||
'''
|
'''
|
||||||
|
@ -1318,6 +1378,8 @@ class Media(models.Model):
|
||||||
return self.STATE_SKIPPED
|
return self.STATE_SKIPPED
|
||||||
if not self.source.download_media:
|
if not self.source.download_media:
|
||||||
return self.STATE_DISABLED_AT_SOURCE
|
return self.STATE_DISABLED_AT_SOURCE
|
||||||
|
if self.is_live:
|
||||||
|
return self.STATE_LIVE
|
||||||
return self.STATE_UNKNOWN
|
return self.STATE_UNKNOWN
|
||||||
|
|
||||||
def get_download_state_icon(self, task=None):
|
def get_download_state_icon(self, task=None):
|
||||||
|
|
|
@ -1,8 +1,10 @@
|
||||||
|
from datetime import timedelta
|
||||||
import os
|
import os
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
|
from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
|
||||||
from django.dispatch import receiver
|
from django.dispatch import receiver
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
from django.utils import timezone
|
||||||
from background_task.signals import task_failed
|
from background_task.signals import task_failed
|
||||||
from background_task.models import Task
|
from background_task.models import Task
|
||||||
from common.logger import log
|
from common.logger import log
|
||||||
|
@ -150,20 +152,28 @@ def media_post_save(sender, instance, created, **kwargs):
|
||||||
post_save.disconnect(media_post_save, sender=Media)
|
post_save.disconnect(media_post_save, sender=Media)
|
||||||
instance.save()
|
instance.save()
|
||||||
post_save.connect(media_post_save, sender=Media)
|
post_save.connect(media_post_save, sender=Media)
|
||||||
# If the media is missing metadata schedule it to be downloaded
|
# If the media is missing metadata schedule it to be downloaded,
|
||||||
if not instance.metadata:
|
# or refresh it if it was live during last run
|
||||||
log.info(f'Scheduling task to download metadata for: {instance.url}')
|
if not instance.metadata or instance.is_live:
|
||||||
|
kwargs = {}
|
||||||
|
if instance.is_live:
|
||||||
|
kwargs['schedule'] = timedelta(hours=1)
|
||||||
|
|
||||||
|
log.info(f'Recheduling task to download metadata for live stream in 1h: {instance.url}')
|
||||||
|
else:
|
||||||
|
log.info(f'Scheduling task to download metadata for: {instance.url}')
|
||||||
verbose_name = _('Downloading metadata for "{}"')
|
verbose_name = _('Downloading metadata for "{}"')
|
||||||
download_media_metadata(
|
download_media_metadata(
|
||||||
str(instance.pk),
|
str(instance.pk),
|
||||||
priority=10,
|
priority=10,
|
||||||
verbose_name=verbose_name.format(instance.pk),
|
verbose_name=verbose_name.format(instance.pk),
|
||||||
remove_existing_tasks=True
|
remove_existing_tasks=True,
|
||||||
|
**kwargs
|
||||||
)
|
)
|
||||||
# If the media is missing a thumbnail schedule it to be downloaded
|
# If the media is missing a thumbnail schedule it to be downloaded
|
||||||
if not instance.thumb_file_exists:
|
if not instance.thumb_file_exists:
|
||||||
instance.thumb = None
|
instance.thumb = None
|
||||||
if not instance.thumb:
|
if not instance.thumb and not instance.is_live:
|
||||||
thumbnail_url = instance.thumbnail
|
thumbnail_url = instance.thumbnail
|
||||||
if thumbnail_url:
|
if thumbnail_url:
|
||||||
log.info(f'Scheduling task to download thumbnail for: {instance.name} '
|
log.info(f'Scheduling task to download thumbnail for: {instance.name} '
|
||||||
|
@ -177,12 +187,13 @@ def media_post_save(sender, instance, created, **kwargs):
|
||||||
verbose_name=verbose_name.format(instance.name),
|
verbose_name=verbose_name.format(instance.name),
|
||||||
remove_existing_tasks=True
|
remove_existing_tasks=True
|
||||||
)
|
)
|
||||||
|
|
||||||
# If the media has not yet been downloaded schedule it to be downloaded
|
# If the media has not yet been downloaded schedule it to be downloaded
|
||||||
if not instance.media_file_exists:
|
if not instance.media_file_exists:
|
||||||
instance.downloaded = False
|
instance.downloaded = False
|
||||||
instance.media_file = None
|
instance.media_file = None
|
||||||
if (not instance.downloaded and instance.can_download and not instance.skip
|
if (not instance.downloaded and instance.can_download and not instance.skip
|
||||||
and instance.source.download_media):
|
and instance.source.download_media and not instance.is_live):
|
||||||
delete_task_by_media('sync.tasks.download_media', (str(instance.pk),))
|
delete_task_by_media('sync.tasks.download_media', (str(instance.pk),))
|
||||||
verbose_name = _('Downloading media for "{}"')
|
verbose_name = _('Downloading media for "{}"')
|
||||||
download_media(
|
download_media(
|
||||||
|
|
|
@ -229,6 +229,7 @@ def download_media_metadata(media_id):
|
||||||
source = media.source
|
source = media.source
|
||||||
metadata = media.index_metadata()
|
metadata = media.index_metadata()
|
||||||
media.metadata = json.dumps(metadata, default=json_serial)
|
media.metadata = json.dumps(metadata, default=json_serial)
|
||||||
|
media.last_crawl = timezone.now()
|
||||||
upload_date = media.upload_date
|
upload_date = media.upload_date
|
||||||
# Media must have a valid upload date
|
# Media must have a valid upload date
|
||||||
if upload_date:
|
if upload_date:
|
||||||
|
|
|
@ -19,6 +19,9 @@
|
||||||
<div class="col m12 xl4 margin-bottom">
|
<div class="col m12 xl4 margin-bottom">
|
||||||
<a href="{% url 'sync:validate-source' source_type='youtube-playlist' %}" class="btn">Add a YouTube playlist <i class="fab fa-youtube"></i></a>
|
<a href="{% url 'sync:validate-source' source_type='youtube-playlist' %}" class="btn">Add a YouTube playlist <i class="fab fa-youtube"></i></a>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="col m12 xl4 margin-bottom">
|
||||||
|
<a href="{% url 'sync:validate-source' source_type='twitch-channel' %}" class="btn">Add a Twitch channel <i class="fab fa-twitch"></i></a>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="row no-margin-bottom">
|
<div class="row no-margin-bottom">
|
||||||
<div class="col s12">
|
<div class="col s12">
|
||||||
|
|
|
@ -199,3 +199,61 @@ def parse_media_format(format_dict):
|
||||||
'is_hls': is_hls,
|
'is_hls': is_hls,
|
||||||
'is_dash': is_dash,
|
'is_dash': is_dash,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_twitch_media_format(format_dict):
|
||||||
|
'''
|
||||||
|
This parser primarily adapts the format dict returned by youtube-dl into a
|
||||||
|
standard form used by the matchers in matching.py. If youtube-dl changes
|
||||||
|
any internals, update it here.
|
||||||
|
'''
|
||||||
|
vcodec_full = format_dict.get('vcodec', '')
|
||||||
|
vcodec_parts = vcodec_full.split('.')
|
||||||
|
if len(vcodec_parts) > 0:
|
||||||
|
vcodec = vcodec_parts[0].strip().upper()
|
||||||
|
else:
|
||||||
|
vcodec = None
|
||||||
|
if vcodec == 'NONE':
|
||||||
|
vcodec = None
|
||||||
|
acodec_full = format_dict.get('acodec', '')
|
||||||
|
acodec_parts = acodec_full.split('.')
|
||||||
|
if len(acodec_parts) > 0:
|
||||||
|
acodec = acodec_parts[0].strip().upper()
|
||||||
|
else:
|
||||||
|
acodec = None
|
||||||
|
if acodec == 'NONE':
|
||||||
|
acodec = None
|
||||||
|
try:
|
||||||
|
fps = int(format_dict.get('fps', 0))
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
fps = 0
|
||||||
|
if format_dict.get('format_id').endswith('p60'):
|
||||||
|
fps = 60
|
||||||
|
height = format_dict.get('height', 0)
|
||||||
|
try:
|
||||||
|
height = int(height)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
height = 0
|
||||||
|
width = format_dict.get('width', 0)
|
||||||
|
try:
|
||||||
|
width = int(width)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
width = 0
|
||||||
|
|
||||||
|
format_full = format_dict.get('format_note', '').strip().upper()
|
||||||
|
return {
|
||||||
|
'id': format_dict.get('format_id', ''),
|
||||||
|
'format': f'{height}P',
|
||||||
|
'format_verbose': format_dict.get('format', ''),
|
||||||
|
'height': height,
|
||||||
|
'width': width,
|
||||||
|
'vcodec': vcodec,
|
||||||
|
'fps': fps,
|
||||||
|
'vbr': format_dict.get('tbr', 0),
|
||||||
|
'acodec': acodec,
|
||||||
|
'abr': format_dict.get('abr', 0),
|
||||||
|
'is_60fps': fps > 50,
|
||||||
|
'is_hdr': False,
|
||||||
|
'is_hls': True,
|
||||||
|
'is_dash': False,
|
||||||
|
}
|
||||||
|
|
|
@ -157,11 +157,13 @@ class ValidateSourceView(FormView):
|
||||||
'youtube-channel': Source.SOURCE_TYPE_YOUTUBE_CHANNEL,
|
'youtube-channel': Source.SOURCE_TYPE_YOUTUBE_CHANNEL,
|
||||||
'youtube-channel-id': Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID,
|
'youtube-channel-id': Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID,
|
||||||
'youtube-playlist': Source.SOURCE_TYPE_YOUTUBE_PLAYLIST,
|
'youtube-playlist': Source.SOURCE_TYPE_YOUTUBE_PLAYLIST,
|
||||||
|
'twitch-channel': Source.SOURCE_TYPE_TWITCH_CHANNEL,
|
||||||
}
|
}
|
||||||
help_item = {
|
help_item = {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _('YouTube channel'),
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _('YouTube channel'),
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: _('YouTube channel ID'),
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: _('YouTube channel ID'),
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: _('YouTube playlist'),
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: _('YouTube playlist'),
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: _('Twitch channel'),
|
||||||
}
|
}
|
||||||
help_texts = {
|
help_texts = {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _(
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _(
|
||||||
|
@ -183,13 +185,20 @@ class ValidateSourceView(FormView):
|
||||||
'BiGLoNgUnIqUeId</strong> where <strong>BiGLoNgUnIqUeId</strong> is the '
|
'BiGLoNgUnIqUeId</strong> where <strong>BiGLoNgUnIqUeId</strong> is the '
|
||||||
'unique ID of the playlist you want to add.'
|
'unique ID of the playlist you want to add.'
|
||||||
),
|
),
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: _(
|
||||||
|
'Enter a Twitch channel URL into the box below. The URL will be '
|
||||||
|
'in the format of <strong>https://www.twitch.tv/ChannelId/videos'
|
||||||
|
'</strong> where <strong>ChannelId</strong> is the '
|
||||||
|
'is the channel you want to add.'
|
||||||
|
),
|
||||||
}
|
}
|
||||||
help_examples = {
|
help_examples = {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/google',
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/google',
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: ('https://www.youtube.com/channel/'
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: ('https://www.youtube.com/channel/'
|
||||||
'UCK8sQmJBp8GCxrOtXWBpyEA'),
|
'UCK8sQmJBp8GCxrOtXWBpyEA'),
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('https://www.youtube.com/playlist?list='
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('https://www.youtube.com/playlist?list='
|
||||||
'PL590L5WQmH8dpP0RyH5pCfIaDEdt9nk7r')
|
'PL590L5WQmH8dpP0RyH5pCfIaDEdt9nk7r'),
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: ('https://www.twitch.tv/twitch/videos')
|
||||||
}
|
}
|
||||||
validation_urls = {
|
validation_urls = {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: {
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: {
|
||||||
|
@ -219,11 +228,21 @@ class ValidateSourceView(FormView):
|
||||||
'extract_key': ('qs_args', 'list'),
|
'extract_key': ('qs_args', 'list'),
|
||||||
'example': 'https://www.youtube.com/playlist?list=PLAYLISTID'
|
'example': 'https://www.youtube.com/playlist?list=PLAYLISTID'
|
||||||
},
|
},
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: {
|
||||||
|
'scheme': 'https',
|
||||||
|
'domains': ('www.twitch.tv',),
|
||||||
|
'path_regex': '^\/([^\/]+)(\/videos)?$',
|
||||||
|
'path_must_not_match': (),
|
||||||
|
'qs_args': [],
|
||||||
|
'extract_key': ('path_regex', 0),
|
||||||
|
'example': 'https://www.twitch.tv/twitch/videos'
|
||||||
|
},
|
||||||
}
|
}
|
||||||
prepopulate_fields = {
|
prepopulate_fields = {
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: ('source_type', 'key', 'name', 'directory'),
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: ('source_type', 'key', 'name', 'directory'),
|
||||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: ('source_type', 'key'),
|
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: ('source_type', 'key'),
|
||||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('source_type', 'key'),
|
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('source_type', 'key'),
|
||||||
|
Source.SOURCE_TYPE_TWITCH_CHANNEL: ('source_type', 'key', 'name', 'directory'),
|
||||||
}
|
}
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
|
@ -305,13 +324,30 @@ class EditSourceMixin:
|
||||||
'errors or is empty. Check the table at the end of '
|
'errors or is empty. Check the table at the end of '
|
||||||
'this page for valid media name variables'),
|
'this page for valid media name variables'),
|
||||||
'dir_outside_dlroot': _('You cannot specify a directory outside of the '
|
'dir_outside_dlroot': _('You cannot specify a directory outside of the '
|
||||||
'base directory (%BASEDIR%)')
|
'base directory (%BASEDIR%)'),
|
||||||
|
'not_twitch_vcodec': _('Twitch only supports AVC1 source video codec'),
|
||||||
|
'not_twitch_acodec': _('Twitch only supports MP4A source audio codec')
|
||||||
}
|
}
|
||||||
|
|
||||||
def form_valid(self, form: Form):
|
def form_valid(self, form: Form):
|
||||||
# Perform extra validation to make sure the media_format is valid
|
# Perform extra validation to make sure the media_format is valid
|
||||||
obj = form.save(commit=False)
|
obj = form.save(commit=False)
|
||||||
source_type = form.cleaned_data['media_format']
|
source_type = form.cleaned_data['source_type']
|
||||||
|
source_vcodec = form.cleaned_data['source_vcodec']
|
||||||
|
source_acodec = form.cleaned_data['source_acodec']
|
||||||
|
|
||||||
|
if source_type == Source.SOURCE_TYPE_TWITCH_CHANNEL and source_vcodec != Source.SOURCE_VCODEC_AVC1:
|
||||||
|
form.add_error(
|
||||||
|
'source_vcodec',
|
||||||
|
ValidationError(self.errors['not_twitch_vcodec'])
|
||||||
|
)
|
||||||
|
|
||||||
|
if source_type == Source.SOURCE_TYPE_TWITCH_CHANNEL and source_acodec != Source.SOURCE_ACODEC_MP4A:
|
||||||
|
form.add_error(
|
||||||
|
'source_acodec',
|
||||||
|
ValidationError(self.errors['not_twitch_acodec'])
|
||||||
|
)
|
||||||
|
|
||||||
example_media_file = obj.get_example_media_format()
|
example_media_file = obj.get_example_media_format()
|
||||||
|
|
||||||
if example_media_file == '':
|
if example_media_file == '':
|
||||||
|
|
|
@ -64,6 +64,16 @@ def get_media_info(url):
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def get_twitch_media_info(url):
|
||||||
|
'''
|
||||||
|
Format Twitch id's to match the id key used in urls
|
||||||
|
'''
|
||||||
|
response = get_media_info(url)
|
||||||
|
for entry in response['entries']:
|
||||||
|
entry['id'] = entry['id'].lstrip('v')
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
def download_media(url, media_format, extension, output_file, info_json,
|
def download_media(url, media_format, extension, output_file, info_json,
|
||||||
sponsor_categories="all",
|
sponsor_categories="all",
|
||||||
embed_thumbnail=False, embed_metadata=False, skip_sponsors=True):
|
embed_thumbnail=False, embed_metadata=False, skip_sponsors=True):
|
||||||
|
|
Loading…
Reference in New Issue