rename app dir

tubesync/sync/admin.py

@@ -0,0 +1,20 @@
+from django.contrib import admin
+from .models import Source, Media
+
+
+@admin.register(Source)
+class SourceAdmin(admin.ModelAdmin):
+
+    ordering = ('-created',)
+    list_display = ('name', 'get_source_type_display', 'last_crawl', 'has_failed')
+    readonly_fields = ('uuid', 'created')
+    search_fields = ('uuid', 'key', 'name')
+
+
+@admin.register(Media)
+class MediaAdmin(admin.ModelAdmin):
+
+    ordering = ('-created',)
+    list_display = ('key', 'source', 'can_download', 'downloaded')
+    readonly_fields = ('uuid', 'created')
+    search_fields = ('uuid', 'source__key', 'key')

tubesync/sync/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class SyncConfig(AppConfig):
+    name = 'sync'

tubesync/sync/forms.py

@@ -0,0 +1,24 @@
+
+from django import forms
+from django.utils.translation import gettext_lazy as _
+
+
+class ValidateSourceForm(forms.Form):
+
+    source_type = forms.CharField(
+        max_length=1,
+        required=True,
+        widget=forms.HiddenInput()
+    )
+    source_url = forms.URLField(
+        label=_('Source URL'),
+        required=True
+    )
+
+
+class ConfirmDeleteSourceForm(forms.Form):
+
+    delete_media = forms.BooleanField(
+        label=_('Also delete downloaded media'),
+        required=False
+    )

tubesync/sync/matching.py

@@ -0,0 +1,386 @@
+'''
+    Match functions take a single Media object instance as its only argument and return
+    two boolean values. The first value is if the match was exact or "best fit", the
+    second argument is the ID of the format that was matched.
+'''
+
+
+from django.conf import settings
+
+
+min_height = getattr(settings, 'VIDEO_HEIGHT_CUTOFF', 360)
+fallback_hd_cutoff = getattr(settings, 'VIDEO_HEIGHT_IS_HD', 500)
+
+
+def get_best_combined_format(media):
+    '''
+        Attempts to see if there is a single, combined audio and video format that
+        exactly matches the source requirements. This is used over separate audio
+        and video formats if possible. Combined formats are the easiest to check
+        for as they must exactly match the source profile be be valid.
+    '''
+    for fmt in media.iter_formats():
+        # Check height matches
+        if media.source.source_resolution.strip().upper() != fmt['format']:
+            continue
+        # Check the video codec matches
+        if media.source.source_vcodec != fmt['vcodec']:
+            continue
+        # Check the audio codec matches
+        if media.source.source_acodec != fmt['acodec']:
+            continue
+        # if the source prefers 60fps, check for it
+        if media.source.prefer_60fps:
+            if not fmt['is_60fps']:
+                continue
+        # If the source prefers HDR, check for it
+        if media.source.prefer_hdr:
+            if not fmt['is_hdr']:
+                continue
+        # If we reach here, we have a combined match!
+        return True, fmt['id']
+    return False, False
+
+
+def get_best_audio_format(media):
+    '''
+        Finds the best match for the source required audio format. If the source
+        has a 'fallback' of fail this can return no match.
+    '''
+    # Order all audio-only formats by bitrate
+    audio_formats = []
+    for fmt in media.iter_formats():
+        # If the format has a video stream, skip it
+        if fmt['vcodec'] is not None:
+            continue
+        audio_formats.append(fmt)
+    audio_formats = list(reversed(sorted(audio_formats, key=lambda k: k['abr'])))
+    if not audio_formats:
+        # Media has no audio formats at all
+        return False, False
+    # Find the highest bitrate audio format with a matching codec
+    for fmt in audio_formats:
+        if media.source.source_acodec == fmt['acodec']:
+            # Matched!
+            return True, fmt['id']
+    # No codecs matched
+    if media.source.can_fallback:
+        # Can fallback, find the next highest bitrate non-matching codec
+        return False, audio_formats[0]
+    else:
+        # Can't fallback
+        return False, False
+
+
+def get_best_video_format(media):
+    '''
+        Finds the best match for the source required video format. If the source
+        has a 'fallback' of fail this can return no match. Resolution is treated
+        as the most important factor to match. This is pretty verbose due to the
+        'soft' matching requirements for prefer_hdr and prefer_60fps.
+    '''
+    # Check if the source wants audio only, fast path to return
+    if media.source.is_audio:
+        return False, False
+    # Filter video-only formats by resolution that matches the source
+    video_formats = []
+    for fmt in media.iter_formats():
+        # If the format has an audio stream, skip it
+        if fmt['acodec'] is not None:
+            continue
+        if media.source.source_resolution.strip().upper() == fmt['format']:
+            video_formats.append(fmt)
+    # Check we matched some streams
+    if not video_formats:
+        # No streams match the requested resolution, see if we can fallback
+        if media.source.can_fallback:
+            # Find the next-best format matches by height
+            for fmt in media.iter_formats():
+                # If the format has an audio stream, skip it
+                if fmt['acodec'] is not None:
+                    continue
+                if (fmt['height'] <= media.source.source_resolution_height and 
+                    fmt['height'] >= min_height):
+                    video_formats.append(fmt)
+        else:
+            # Can't fallback
+            return False, False
+    video_formats = list(reversed(sorted(video_formats, key=lambda k: k['height'])))
+    source_resolution = media.source.source_resolution.strip().upper()
+    source_vcodec = media.source.source_vcodec
+    if not video_formats:
+        # Still no matches
+        return False, False
+    exact_match, best_match = None, None
+    # Of our filtered video formats, check for resolution + codec + hdr + fps match
+    if media.source.prefer_60fps and media.source.prefer_hdr:
+        for fmt in video_formats:
+            # Check for an exact match
+            if (source_resolution == fmt['format'] and
+                source_vcodec == fmt['vcodec'] and 
+                fmt['is_hdr'] and
+                fmt['is_60fps']):
+                # Exact match
+                exact_match, best_match = True, fmt
+                break
+        if media.source.can_fallback:
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for a resolution, hdr and fps match but drop the codec
+                    if (source_resolution == fmt['format'] and 
+                        fmt['is_hdr'] and fmt['is_60fps']):
+                        # Close match
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for a codec, hdr and fps match but drop the resolution
+                    if (source_vcodec == fmt['vcodec'] and 
+                        fmt['is_hdr'] and fmt['is_60fps']):
+                        # Close match
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution, codec and 60fps match
+                    if (source_resolution == fmt['format'] and
+                        source_vcodec == fmt['vcodec'] and
+                        fmt['is_60fps']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution and hdr match
+                    if (source_resolution == fmt['format'] and
+                        fmt['is_hdr']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution and 60fps match
+                    if (source_resolution == fmt['format'] and
+                        fmt['is_60fps']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution, codec and hdr match
+                    if (source_resolution == fmt['format'] and
+                        source_vcodec == fmt['vcodec'] and
+                        fmt['is_hdr']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution and codec
+                    if (source_resolution == fmt['format'] and
+                        source_vcodec == fmt['vcodec']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution
+                    if source_resolution == fmt['format']:
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                # Match the highest resolution
+                exact_match, best_match = False, video_formats[0]
+    # Check for resolution + codec + fps match
+    if media.source.prefer_60fps and not media.source.prefer_hdr:
+        for fmt in video_formats:
+            # Check for an exact match
+            if (source_resolution == fmt['format'] and
+                source_vcodec == fmt['vcodec'] and 
+                fmt['is_60fps'] and
+                not fmt['is_hdr']):
+                # Exact match
+                exact_match, best_match = True, fmt
+                break
+        if media.source.can_fallback:
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for a resolution and fps match but drop the codec
+                    if (source_resolution == fmt['format'] and 
+                        fmt['is_60fps'] and
+                        not fmt['is_hdr']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for a codec and fps match but drop the resolution
+                    if (source_vcodec == fmt['vcodec'] and 
+                        fmt['is_60fps'] and
+                        not fmt['is_hdr']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for a codec and 60fps match
+                    if (source_vcodec == fmt['vcodec'] and 
+                        fmt['is_60fps']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for codec and resolution match bot drop 60fps
+                    if (source_resolution == fmt['format'] and
+                        source_vcodec == fmt['vcodec'] and
+                        not fmt['is_hdr']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for codec and resolution match only
+                    if (source_resolution == fmt['format'] and
+                        source_vcodec == fmt['vcodec']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution
+                    if source_resolution == fmt['format']:
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                # Match the highest resolution
+                exact_match, best_match = False, video_formats[0]
+    # Check for resolution + codec + hdr
+    elif media.source.prefer_hdr and not media.source.prefer_60fps:
+        for fmt in video_formats:
+            # Check for an exact match
+            if (source_resolution == fmt['format'] and
+                source_vcodec == fmt['vcodec'] and 
+                fmt['is_hdr']):
+                # Exact match
+                exact_match, best_match = True, fmt
+                break
+        if media.source.can_fallback:
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for a resolution and fps match but drop the codec
+                    if (source_resolution == fmt['format'] and 
+                        fmt['is_hdr'] and
+                        not fmt['is_60fps']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for a codec and fps match but drop the resolution
+                    if (source_vcodec == fmt['vcodec'] and 
+                        fmt['is_hdr'] and
+                        not fmt['is_60fps']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for a codec and 60fps match
+                    if (source_vcodec == fmt['vcodec'] and 
+                        fmt['is_hdr']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for codec and resolution match bot drop hdr
+                    if (source_resolution == fmt['format'] and
+                        source_vcodec == fmt['vcodec'] and
+                        not fmt['is_60fps']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for codec and resolution match only
+                    if (source_resolution == fmt['format'] and
+                        source_vcodec == fmt['vcodec']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution
+                    if source_resolution == fmt['format']:
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                # Match the highest resolution
+                exact_match, best_match = False, video_formats[0]
+    # check for resolution + codec
+    elif not media.source.prefer_hdr and not media.source.prefer_60fps:
+        for fmt in video_formats:
+            # Check for an exact match
+            if (source_resolution == fmt['format'] and
+                source_vcodec == fmt['vcodec'] and
+                not fmt['is_60fps'] and
+                not fmt['is_hdr']):
+                # Exact match
+                exact_match, best_match = True, fmt
+                break
+        if media.source.can_fallback:
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for a resolution, hdr and fps match but drop the codec
+                    if (source_resolution == fmt['format'] and 
+                        not fmt['is_hdr'] and not fmt['is_60fps']):
+                        # Close match
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for a codec, hdr and fps match but drop the resolution
+                    if (source_vcodec == fmt['vcodec'] and 
+                        not fmt['is_hdr'] and fmt['is_60fps']):
+                        # Close match
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution, codec and hdr match
+                    if (source_resolution == fmt['format'] and
+                        source_vcodec == fmt['vcodec'] and
+                        not fmt['is_hdr']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution, codec and 60fps match
+                    if (source_resolution == fmt['format'] and
+                        source_vcodec == fmt['vcodec'] and
+                        not fmt['is_60fps']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution and codec
+                    if (source_resolution == fmt['format'] and
+                        source_vcodec == fmt['vcodec']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution and not hdr
+                    if (source_resolution == fmt['format'] and
+                        not fmt['is_hdr']):
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                for fmt in video_formats:
+                    # Check for resolution
+                    if source_resolution == fmt['format']:
+                        exact_match, best_match = False, fmt
+                        break
+            if not best_match:
+                # Match the highest resolution
+                exact_match, best_match = False, video_formats[0]
+    # See if we found a match
+    if best_match:
+        # Final check to see if the match we found was good enough
+        if exact_match:
+            return True, best_match['id']
+        elif media.source.can_fallback:
+            # Allow the fallback if it meets requirements
+            if (media.source.fallback == media.source.FALLBACK_NEXT_BEST_HD and
+                best_match['height'] >= fallback_hd_cutoff):
+                return False, best_match['id']
+            elif media.source.fallback == media.source.FALLBACK_NEXT_BEST:
+                return False, best_match['id']
+    # Nope, failed to find match
+    return False, False

tubesync/sync/migrations/0001_initial.py

@@ -0,0 +1,67 @@
+# Generated by Django 3.1.3 on 2020-11-23 06:29
+
+from django.db import migrations, models
+import django.db.models.deletion
+import sync.models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Source',
+            fields=[
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the source', primary_key=True, serialize=False, verbose_name='uuid')),
+                ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the source was created', verbose_name='created')),
+                ('last_crawl', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the source was last crawled', null=True, verbose_name='last crawl')),
+                ('source_type', models.CharField(choices=[('c', 'YouTube channel'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='type')),
+                ('url', models.URLField(db_index=True, help_text='URL of the source', verbose_name='url')),
+                ('key', models.CharField(db_index=True, help_text='Source key, such as exact YouTube channel name or playlist ID', max_length=100, verbose_name='key')),
+                ('name', models.CharField(db_index=True, help_text='Friendly name for the source, used locally', max_length=100, verbose_name='name')),
+                ('directory', models.CharField(help_text='Directory name to save the media into', max_length=100, verbose_name='directory')),
+                ('delete_old_media', models.BooleanField(default=False, help_text='Delete old media after "days to keep" days?', verbose_name='delete old media')),
+                ('days_to_keep', models.PositiveSmallIntegerField(default=14, help_text='If "delete old media" is ticked, the number of days after which to automatically delete media', verbose_name='days to keep')),
+                ('source_profile', models.CharField(choices=[('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('2160p', '2160p (4K)'), ('audio', 'Audio only')], db_index=True, default='1080p', help_text='Source profile, the quality to attempt to download media', max_length=8, verbose_name='source profile')),
+                ('prefer_60fps', models.BooleanField(default=False, help_text='Where possible, prefer 60fps media for this source', verbose_name='prefer 60fps')),
+                ('prefer_hdr', models.BooleanField(default=False, help_text='Where possible, prefer HDR media for this source', verbose_name='prefer hdr')),
+                ('output_format', models.CharField(choices=[('mp4', '.mp4 container'), ('mkv', '.mkv container'), ('mkv', '.webm container'), ('m4a', '.m4a container (audio only)'), ('ogg', '.ogg container (audio only)')], db_index=True, default='mkv', help_text='Output format, the codec and container to save media', max_length=8, verbose_name='output format')),
+                ('fallback', models.CharField(choices=[('f', 'Fail'), ('s', 'Next best SD'), ('h', 'Next best HD')], db_index=True, default='f', help_text='What do do when your first choice is not available', max_length=1, verbose_name='fallback')),
+            ],
+            options={
+                'verbose_name': 'Source',
+                'verbose_name_plural': 'Sources',
+            },
+        ),
+        migrations.CreateModel(
+            name='Media',
+            fields=[
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the media', primary_key=True, serialize=False, verbose_name='uuid')),
+                ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the media was created', verbose_name='created')),
+                ('published', models.DateTimeField(blank=True, db_index=True, help_text='Date and time the media was published on the source', null=True, verbose_name='published')),
+                ('key', models.CharField(db_index=True, help_text='Media key, such as exact YouTube video ID', max_length=100, verbose_name='key')),
+                ('url', models.URLField(db_index=True, help_text='URL of the media', verbose_name='url')),
+                ('thumb', models.ImageField(blank=True, height_field='thumb_height', help_text='Thumbnail', null=True, upload_to=sync.models.get_media_thumb_path, verbose_name='thumb', width_field='thumb_width')),
+                ('thumb_width', models.PositiveSmallIntegerField(blank=True, help_text='Width (X) of the thumbnail', verbose_name='thumb width')),
+                ('thumb_height', models.PositiveSmallIntegerField(blank=True, help_text='Height (Y) of the thumbnail', verbose_name='thumb height')),
+                ('metadata', models.TextField(blank=True, help_text='JSON encoded metadata for the media', null=True, verbose_name='metadata')),
+                ('downloaded', models.BooleanField(db_index=True, default=False, help_text='Media has been downloaded', verbose_name='downloaded')),
+                ('downloaded_audio_codec', models.CharField(blank=True, db_index=True, help_text='Audio codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded audio codec')),
+                ('downloaded_video_codec', models.CharField(blank=True, db_index=True, help_text='Video codec of the downloaded media', max_length=30, null=True, verbose_name='downloaded video codec')),
+                ('downloaded_container', models.CharField(blank=True, db_index=True, help_text='Container format of the downloaded media', max_length=30, null=True, verbose_name='downloaded container format')),
+                ('downloaded_fps', models.PositiveSmallIntegerField(blank=True, db_index=True, help_text='FPS of the downloaded media', null=True, verbose_name='downloaded fps')),
+                ('downloaded_hdr', models.BooleanField(default=False, help_text='Downloaded media has HDR', verbose_name='downloaded hdr')),
+                ('downloaded_filesize', models.PositiveBigIntegerField(blank=True, db_index=True, help_text='Size of the downloaded media in bytes', null=True, verbose_name='downloaded filesize')),
+                ('source', models.ForeignKey(help_text='Source the media belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='media_source', to='sync.source')),
+            ],
+            options={
+                'verbose_name': 'Media',
+                'verbose_name_plural': 'Media',
+            },
+        ),
+    ]

tubesync/sync/migrations/0002_auto_20201126_0504.py

@@ -0,0 +1,37 @@
+# Generated by Django 3.1.3 on 2020-11-26 05:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='media',
+            name='url',
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='fallback',
+            field=models.CharField(choices=[('f', 'Fail, do not download any media'), ('s', 'Get next best SD media instead'), ('h', 'Get next best HD media instead')], db_index=True, default='f', help_text='What do do when media in your source profile is not available', max_length=1, verbose_name='fallback'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='name',
+            field=models.CharField(db_index=True, help_text='Friendly name for the source, used locally in TubeSync only', max_length=100, verbose_name='name'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='output_format',
+            field=models.CharField(choices=[('mp4', '.mp4 container'), ('mkv', '.mkv container'), ('mkv', '.webm container'), ('m4a', '.m4a container (audio only)'), ('ogg', '.ogg container (audio only)')], db_index=True, default='mkv', help_text='Output format, the file format container in which to save media', max_length=8, verbose_name='output format'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='source_type',
+            field=models.CharField(choices=[('c', 'YouTube channel'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type'),
+        ),
+    ]

tubesync/sync/migrations/0003_auto_20201127_0838.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2020-11-27 08:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0002_auto_20201126_0504'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='key',
+            field=models.CharField(db_index=True, help_text='Source key, such as exact YouTube channel name or playlist ID', max_length=100, unique=True, verbose_name='key'),
+        ),
+    ]

tubesync/sync/migrations/0004_remove_source_url.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.1.3 on 2020-11-28 03:11
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0003_auto_20201127_0838'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='source',
+            name='url',
+        ),
+    ]

tubesync/sync/migrations/0005_auto_20201205_0411.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.3 on 2020-12-05 04:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0004_remove_source_url'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='media',
+            name='thumb_height',
+            field=models.PositiveSmallIntegerField(blank=True, help_text='Height (Y) of the thumbnail', null=True, verbose_name='thumb height'),
+        ),
+        migrations.AlterField(
+            model_name='media',
+            name='thumb_width',
+            field=models.PositiveSmallIntegerField(blank=True, help_text='Width (X) of the thumbnail', null=True, verbose_name='thumb width'),
+        ),
+    ]

tubesync/sync/migrations/0006_auto_20201205_0502.py

@@ -0,0 +1,36 @@
+# Generated by Django 3.1.3 on 2020-12-05 05:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0005_auto_20201205_0411'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='source',
+            name='output_format',
+        ),
+        migrations.RemoveField(
+            model_name='source',
+            name='source_profile',
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='source_resolution',
+            field=models.CharField(choices=[('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('2160p', '2160p (4K)'), ('audio', 'Audio only')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='source_vcodec',
+            field=models.CharField(choices=[('M4A', 'M4A'), ('OPUS', 'OPUS'), ('AAC', 'AAC')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='prefer_60fps',
+            field=models.BooleanField(default=True, help_text='Where possible, prefer 60fps media for this source', verbose_name='prefer 60fps'),
+        ),
+    ]

tubesync/sync/migrations/0007_auto_20201205_0509.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2020-12-05 05:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0006_auto_20201205_0502'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='source_vcodec',
+            field=models.CharField(choices=[('M4A', 'M4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'),
+        ),
+    ]

tubesync/sync/migrations/0008_auto_20201205_0512.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.3 on 2020-12-05 05:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0007_auto_20201205_0509'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='source_acodec',
+            field=models.CharField(choices=[('M4A', 'M4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='source_vcodec',
+            field=models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download', max_length=8, verbose_name='source video codec'),
+        ),
+    ]

tubesync/sync/migrations/0009_auto_20201205_0512.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2020-12-05 05:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0008_auto_20201205_0512'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='source_vcodec',
+            field=models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec'),
+        ),
+    ]

tubesync/sync/migrations/0010_auto_20201206_0159.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.1.4 on 2020-12-06 01:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0009_auto_20201205_0512'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='directory',
+            field=models.CharField(db_index=True, help_text='Directory name to save the media into', max_length=100, unique=True, verbose_name='directory'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='fallback',
+            field=models.CharField(choices=[('f', 'Fail, do not download any media'), ('s', 'Get next best SD media or codec instead'), ('h', 'Get next best HD media or codec instead')], db_index=True, default='f', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='name',
+            field=models.CharField(db_index=True, help_text='Friendly name for the source, used locally in TubeSync only', max_length=100, unique=True, verbose_name='name'),
+        ),
+    ]

tubesync/sync/migrations/0011_auto_20201206_0911.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.4 on 2020-12-06 09:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0010_auto_20201206_0159'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='fallback',
+            field=models.CharField(choices=[('f', 'Fail, do not download any media'), ('s', 'Get next best SD media or codec instead'), ('h', 'Get next best HD media or codec instead')], db_index=True, default='h', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='source_resolution',
+            field=models.CharField(choices=[('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('audio', 'Audio only')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution'),
+        ),
+    ]

tubesync/sync/migrations/0012_auto_20201207_0415.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.4 on 2020-12-07 04:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0011_auto_20201206_0911'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='index_schedule',
+            field=models.IntegerField(db_index=True, default=21600, help_text='Schedule of when to index the source for new media', verbose_name='index schedule'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='source_acodec',
+            field=models.CharField(choices=[('MP4A', 'MP4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'),
+        ),
+    ]

tubesync/sync/migrations/0013_auto_20201207_0439.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.4 on 2020-12-07 04:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0012_auto_20201207_0415'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='index_schedule',
+            field=models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours')], db_index=True, default=21600, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule'),
+        ),
+    ]

tubesync/sync/migrations/0014_source_has_errors.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.4 on 2020-12-07 07:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0013_auto_20201207_0439'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='has_errors',
+            field=models.BooleanField(default=False, help_text='Source has errors', verbose_name='has errors'),
+        ),
+    ]

tubesync/sync/migrations/0015_auto_20201207_0744.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.1.4 on 2020-12-07 07:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0014_source_has_errors'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='source',
+            name='has_errors',
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='has_failed',
+            field=models.BooleanField(default=False, help_text='Source has failed to index media', verbose_name='has failed'),
+        ),
+    ]

tubesync/sync/migrations/0016_auto_20201208_0518.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.4 on 2020-12-08 05:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0015_auto_20201207_0744'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='fallback',
+            field=models.CharField(choices=[('f', 'Fail, do not download any media'), ('n', 'Get next best resolution or codec instead'), ('h', 'Get next best resolution but at least HD')], db_index=True, default='h', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback'),
+        ),
+    ]

tubesync/sync/migrations/0017_auto_20201208_0521.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.4 on 2020-12-08 05:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0016_auto_20201208_0518'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='source_resolution',
+            field=models.CharField(choices=[('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('1440p', '1440p (2K)'), ('2160p', '2160p (4K)'), ('4320p', '4320p (8K)'), ('audio', 'Audio only')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution'),
+        ),
+    ]

tubesync/sync/migrations/0018_media_can_download.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.4 on 2020-12-09 08:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0017_auto_20201208_0521'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='media',
+            name='can_download',
+            field=models.BooleanField(db_index=True, default=False, help_text='Media has a matching format and can be downloaded', verbose_name='can download'),
+        ),
+    ]

tubesync/sync/migrations/0019_auto_20201209_0857.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.1.4 on 2020-12-09 08:57
+
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0018_media_can_download'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='media',
+            name='media_file',
+            field=models.FileField(blank=True, help_text='Media file', max_length=200, null=True, upload_to=sync.models.get_media_file_path, verbose_name='media file'),
+        ),
+        migrations.AlterField(
+            model_name='media',
+            name='thumb',
+            field=models.ImageField(blank=True, height_field='thumb_height', help_text='Thumbnail', max_length=200, null=True, upload_to=sync.models.get_media_thumb_path, verbose_name='thumb', width_field='thumb_width'),
+        ),
+    ]

tubesync/sync/models.py

@@ -0,0 +1,676 @@
+import os
+import uuid
+import json
+from datetime import datetime
+from pathlib import Path
+from django.conf import settings
+from django.db import models
+from django.utils.text import slugify
+from django.utils.translation import gettext_lazy as _
+from common.errors import NoFormatException
+from .youtube import (get_media_info as get_youtube_media_info,
+                      download_media as download_youtube_media)
+from .utils import seconds_to_timestr, parse_media_format
+from .matching import (get_best_combined_format, get_best_audio_format, 
+                       get_best_video_format)
+
+
+class Source(models.Model):
+    '''
+        A Source is a source of media. Currently, this is either a YouTube channel
+        or a YouTube playlist.
+    '''
+
+    SOURCE_TYPE_YOUTUBE_CHANNEL = 'c'
+    SOURCE_TYPE_YOUTUBE_PLAYLIST = 'p'
+    SOURCE_TYPES = (SOURCE_TYPE_YOUTUBE_CHANNEL, SOURCE_TYPE_YOUTUBE_PLAYLIST)
+    SOURCE_TYPE_CHOICES = (
+        (SOURCE_TYPE_YOUTUBE_CHANNEL, _('YouTube channel')),
+        (SOURCE_TYPE_YOUTUBE_PLAYLIST, _('YouTube playlist')),
+    )
+
+    SOURCE_RESOLUTION_360P = '360p'
+    SOURCE_RESOLUTION_480P = '480p'
+    SOURCE_RESOLUTION_720P = '720p'
+    SOURCE_RESOLUTION_1080P = '1080p'
+    SOURCE_RESOLUTION_1440P = '1440p'
+    SOURCE_RESOLUTION_2160P = '2160p'
+    SOURCE_RESOLUTION_4320P = '4320p'
+    SOURCE_RESOLUTION_AUDIO = 'audio'
+    SOURCE_RESOLUTIONS = (SOURCE_RESOLUTION_360P, SOURCE_RESOLUTION_480P,
+                          SOURCE_RESOLUTION_720P, SOURCE_RESOLUTION_1080P,
+                          SOURCE_RESOLUTION_1440P, SOURCE_RESOLUTION_2160P,
+                          SOURCE_RESOLUTION_4320P, SOURCE_RESOLUTION_AUDIO)
+    SOURCE_RESOLUTION_CHOICES = (
+        (SOURCE_RESOLUTION_360P, _('360p (SD)')),
+        (SOURCE_RESOLUTION_480P, _('480p (SD)')),
+        (SOURCE_RESOLUTION_720P, _('720p (HD)')),
+        (SOURCE_RESOLUTION_1080P, _('1080p (Full HD)')),
+        (SOURCE_RESOLUTION_1440P, _('1440p (2K)')),
+        (SOURCE_RESOLUTION_2160P, _('2160p (4K)')),
+        (SOURCE_RESOLUTION_4320P, _('4320p (8K)')),
+        (SOURCE_RESOLUTION_AUDIO, _('Audio only')),
+    )
+    RESOLUTION_MAP = {
+        SOURCE_RESOLUTION_360P: 360,
+        SOURCE_RESOLUTION_480P: 480,
+        SOURCE_RESOLUTION_720P: 720,
+        SOURCE_RESOLUTION_1080P: 1080,
+        SOURCE_RESOLUTION_1440P: 1440,
+        SOURCE_RESOLUTION_2160P: 2160,
+        SOURCE_RESOLUTION_4320P: 4320,
+    }
+
+    SOURCE_VCODEC_AVC1 = 'AVC1'
+    SOURCE_VCODEC_VP9 = 'VP9'
+    SOURCE_VCODECS = (SOURCE_VCODEC_AVC1, SOURCE_VCODEC_VP9)
+    SOURCE_VCODECS_PRIORITY = (SOURCE_VCODEC_VP9, SOURCE_VCODEC_AVC1)
+    SOURCE_VCODEC_CHOICES = (
+        (SOURCE_VCODEC_AVC1, _('AVC1 (H.264)')),
+        (SOURCE_VCODEC_VP9, _('VP9')),
+    )
+
+    SOURCE_ACODEC_MP4A = 'MP4A'
+    SOURCE_ACODEC_OPUS = 'OPUS'
+    SOURCE_ACODECS = (SOURCE_ACODEC_MP4A, SOURCE_ACODEC_OPUS)
+    SOURCE_ACODEC_PRIORITY = (SOURCE_ACODEC_OPUS, SOURCE_ACODEC_MP4A)
+    SOURCE_ACODEC_CHOICES = (
+        (SOURCE_ACODEC_MP4A, _('MP4A')),
+        (SOURCE_ACODEC_OPUS, _('OPUS')),
+    )
+
+    FALLBACK_FAIL = 'f'
+    FALLBACK_NEXT_BEST = 'n'
+    FALLBACK_NEXT_BEST_HD = 'h'
+    FALLBACKS = (FALLBACK_FAIL, FALLBACK_NEXT_BEST, FALLBACK_NEXT_BEST_HD)
+    FALLBACK_CHOICES = (
+        (FALLBACK_FAIL, _('Fail, do not download any media')),
+        (FALLBACK_NEXT_BEST, _('Get next best resolution or codec instead')),
+        (FALLBACK_NEXT_BEST_HD, _('Get next best resolution but at least HD'))
+    )
+
+    # Fontawesome icons used for the source on the front end
+    ICONS = {
+        SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
+        SOURCE_TYPE_YOUTUBE_PLAYLIST: '<i class="fab fa-youtube"></i>',
+    }
+    # Format to use to display a URL for the source
+    URLS = {
+        SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}',
+        SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
+    }
+    # Callback functions to get a list of media from the source
+    INDEXERS = {
+        SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
+        SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
+    }
+    # Field names to find the media ID used as the key when storing media
+    KEY_FIELD = {
+        SOURCE_TYPE_YOUTUBE_CHANNEL: 'id',
+        SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id',
+    }
+
+    class IndexSchedule(models.IntegerChoices):
+        EVERY_HOUR = 3600, _('Every hour')
+        EVERY_2_HOURS = 7200, _('Every 2 hours')
+        EVERY_3_HOURS = 10800, _('Every 3 hours')
+        EVERY_4_HOURS = 14400, _('Every 4 hours')
+        EVERY_5_HOURS = 18000, _('Every 5 hours')
+        EVERY_6_HOURS = 21600, _('Every 6 hours')
+        EVERY_12_HOURS = 43200, _('Every 12 hours')
+        EVERY_24_HOURS = 86400, _('Every 24 hours')
+
+    uuid = models.UUIDField(
+        _('uuid'),
+        primary_key=True,
+        editable=False,
+        default=uuid.uuid4,
+        help_text=_('UUID of the source')
+    )
+    created = models.DateTimeField(
+        _('created'),
+        auto_now_add=True,
+        db_index=True,
+        help_text=_('Date and time the source was created')
+    )
+    last_crawl = models.DateTimeField(
+        _('last crawl'),
+        db_index=True,
+        null=True,
+        blank=True,
+        help_text=_('Date and time the source was last crawled')
+    )
+    source_type = models.CharField(
+        _('source type'),
+        max_length=1,
+        db_index=True,
+        choices=SOURCE_TYPE_CHOICES,
+        default=SOURCE_TYPE_YOUTUBE_CHANNEL,
+        help_text=_('Source type')
+    )
+    key = models.CharField(
+        _('key'),
+        max_length=100,
+        db_index=True,
+        unique=True,
+        help_text=_('Source key, such as exact YouTube channel name or playlist ID')
+    )
+    name = models.CharField(
+        _('name'),
+        max_length=100,
+        db_index=True,
+        unique=True,
+        help_text=_('Friendly name for the source, used locally in TubeSync only')
+    )
+    directory = models.CharField(
+        _('directory'),
+        max_length=100,
+        db_index=True,
+        unique=True,
+        help_text=_('Directory name to save the media into')
+    )
+    index_schedule = models.IntegerField(
+        _('index schedule'),
+        choices=IndexSchedule.choices,
+        db_index=True,
+        default=IndexSchedule.EVERY_6_HOURS,
+        help_text=_('Schedule of how often to index the source for new media')
+    )
+    delete_old_media = models.BooleanField(
+        _('delete old media'),
+        default=False,
+        help_text=_('Delete old media after "days to keep" days?')
+    )
+    days_to_keep = models.PositiveSmallIntegerField(
+        _('days to keep'),
+        default=14,
+        help_text=_('If "delete old media" is ticked, the number of days after which '
+                    'to automatically delete media')
+    )
+    source_resolution = models.CharField(
+        _('source resolution'),
+        max_length=8,
+        db_index=True,
+        choices=SOURCE_RESOLUTION_CHOICES,
+        default=SOURCE_RESOLUTION_1080P,
+        help_text=_('Source resolution, desired video resolution to download')
+    )
+    source_vcodec = models.CharField(
+        _('source video codec'),
+        max_length=8,
+        db_index=True,
+        choices=SOURCE_VCODEC_CHOICES,
+        default=SOURCE_VCODEC_VP9,
+        help_text=_('Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)')
+    )
+    source_acodec = models.CharField(
+        _('source audio codec'),
+        max_length=8,
+        db_index=True,
+        choices=SOURCE_ACODEC_CHOICES,
+        default=SOURCE_ACODEC_OPUS,
+        help_text=_('Source audio codec, desired audio encoding format to download')
+    )
+    prefer_60fps = models.BooleanField(
+        _('prefer 60fps'),
+        default=True,
+        help_text=_('Where possible, prefer 60fps media for this source')
+    )
+    prefer_hdr = models.BooleanField(
+        _('prefer hdr'),
+        default=False,
+        help_text=_('Where possible, prefer HDR media for this source')
+    )
+    fallback = models.CharField(
+        _('fallback'),
+        max_length=1,
+        db_index=True,
+        choices=FALLBACK_CHOICES,
+        default=FALLBACK_NEXT_BEST_HD,
+        help_text=_('What do do when media in your source resolution and codecs is not available')
+    )
+    has_failed = models.BooleanField(
+        _('has failed'),
+        default=False,
+        help_text=_('Source has failed to index media')
+    )
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        verbose_name = _('Source')
+        verbose_name_plural = _('Sources')
+
+    @property
+    def icon(self):
+        return self.ICONS.get(self.source_type)
+
+    @property
+    def is_audio(self):
+        return self.source_resolution == self.SOURCE_RESOLUTION_AUDIO
+
+    @property
+    def is_video(self):
+        return not self.is_audio
+
+    @property
+    def extension(self):
+        '''
+            The extension is also used by youtube-dl to set the output container. As
+            it is possible to quite easily pick combinations of codecs and containers
+            which are invalid (e.g. OPUS audio in an MP4 container) just set this for
+            people. All video is set to mkv containers, audio-only is set to m4a or ogg
+            depending on audio codec.
+        '''
+        if self.is_audio:
+            if self.source_acodec == self.SOURCE_ACODEC_MP4A:
+                return 'm4a'
+            elif self.source_acodec == self.SOURCE_ACODEC_OPUS:
+                return 'ogg'
+            else:
+                raise ValueError('Unable to choose audio extension, uknown acodec')
+        else:
+            return 'mkv'
+
+    @classmethod
+    def create_url(obj, source_type, key):
+        url = obj.URLS.get(source_type)
+        return url.format(key=key)
+
+    @property
+    def url(self):
+        return Source.create_url(self.source_type, self.key)
+
+    @property
+    def format_summary(self):
+        if self.source_resolution == Source.SOURCE_RESOLUTION_AUDIO:
+            vc = 'none'
+        else:
+            vc = self.source_vcodec
+        ac = self.source_acodec
+        f = ' 60FPS' if self.is_video and self.prefer_60fps else ''
+        h = ' HDR' if self.is_video and self.prefer_hdr else ''
+        return f'{self.source_resolution} (video:{vc}, audio:{ac}){f}{h}'.strip()
+
+    @property
+    def directory_path(self):
+        if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO:
+            return settings.SYNC_AUDIO_ROOT / self.directory
+        else:
+            return settings.SYNC_VIDEO_ROOT / self.directory
+
+    def make_directory(self):
+        return os.makedirs(self.directory_path, exist_ok=True)
+
+    def directory_exists(self):
+        return (os.path.isdir(self.directory_path) and
+                os.access(self.directory_path, os.W_OK))
+
+    @property
+    def key_field(self):
+        return self.KEY_FIELD.get(self.source_type, '')
+
+    @property
+    def source_resolution_height(self):
+        return self.RESOLUTION_MAP.get(self.source_resolution, 0)
+
+    @property
+    def can_fallback(self):
+        return self.fallback != self.FALLBACK_FAIL
+
+    def index_media(self):
+        '''
+            Index the media source returning a list of media metadata as dicts.
+        '''
+        indexer = self.INDEXERS.get(self.source_type, None)
+        if not callable(indexer):
+            raise Exception(f'Source type f"{self.source_type}" has no indexer')
+        response = indexer(self.url)
+
+        # Account for nested playlists, such as a channel of playlists of playlists
+        def _recurse_playlists(playlist):
+            videos = []
+            if not playlist:
+                return videos
+            entries = playlist.get('entries', [])
+            for entry in entries:
+                if not entry:
+                    continue
+                subentries = entry.get('entries', [])
+                if subentries:
+                    videos = videos + _recurse_playlists(entry)
+                else:
+                    videos.append(entry)
+            return videos
+
+        return _recurse_playlists(response)
+
+
+def get_media_thumb_path(instance, filename):
+    fileid = str(instance.uuid)
+    filename = f'{fileid.lower()}.jpg'
+    prefix = fileid[:2]
+    return Path('thumbs') / prefix / filename
+
+
+def get_media_file_path(instance, filename):
+    return instance.filepath
+
+
+class Media(models.Model):
+    '''
+        Media is a single piece of media, such as a single YouTube video linked to a
+        Source.
+    '''
+
+    # Format to use to display a URL for the media
+    URLS = {
+        Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/watch?v={key}',
+        Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}',
+    }
+    # Maps standardised names to names used in source metdata
+    METADATA_FIELDS = {
+        'upload_date': {
+            Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'upload_date',
+            Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'upload_date',
+        },
+        'title': {
+            Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'title',
+            Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'title',
+        },
+        'thumbnail': {
+            Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'thumbnail',
+            Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'thumbnail',
+        },
+        'description': {
+            Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'description',
+            Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'description',
+        },
+        'duration': {
+            Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'duration',
+            Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'duration',
+        },
+        'formats': {
+            Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'formats',
+            Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats',
+        }
+    }
+
+    uuid = models.UUIDField(
+        _('uuid'),
+        primary_key=True,
+        editable=False,
+        default=uuid.uuid4,
+        help_text=_('UUID of the media')
+    )
+    created = models.DateTimeField(
+        _('created'),
+        auto_now_add=True,
+        db_index=True,
+        help_text=_('Date and time the media was created')
+    )
+    source = models.ForeignKey(
+        Source,
+        on_delete=models.CASCADE,
+        related_name='media_source',
+        help_text=_('Source the media belongs to')
+    )
+    published = models.DateTimeField(
+        _('published'),
+        db_index=True,
+        null=True,
+        blank=True,
+        help_text=_('Date and time the media was published on the source')
+    )
+    key = models.CharField(
+        _('key'),
+        max_length=100,
+        db_index=True,
+        help_text=_('Media key, such as exact YouTube video ID')
+    )
+    thumb = models.ImageField(
+        _('thumb'),
+        upload_to=get_media_thumb_path,
+        max_length=200,
+        blank=True,
+        null=True,
+        width_field='thumb_width',
+        height_field='thumb_height',
+        help_text=_('Thumbnail')
+    )
+    thumb_width = models.PositiveSmallIntegerField(
+        _('thumb width'),
+        blank=True,
+        null=True,
+        help_text=_('Width (X) of the thumbnail')
+    )
+    thumb_height = models.PositiveSmallIntegerField(
+        _('thumb height'),
+        blank=True,
+        null=True,
+        help_text=_('Height (Y) of the thumbnail')
+    )
+    metadata = models.TextField(
+        _('metadata'),
+        blank=True,
+        null=True,
+        help_text=_('JSON encoded metadata for the media')
+    )
+    can_download = models.BooleanField(
+        _('can download'),
+        db_index=True,
+        default=False,
+        help_text=_('Media has a matching format and can be downloaded')
+    )
+    media_file = models.FileField(
+        _('media file'),
+        upload_to=get_media_file_path,
+        max_length=200,
+        blank=True,
+        null=True,
+        help_text=_('Media file')
+    )
+    downloaded = models.BooleanField(
+        _('downloaded'),
+        db_index=True,
+        default=False,
+        help_text=_('Media has been downloaded')
+    )
+    downloaded_audio_codec = models.CharField(
+        _('downloaded audio codec'),
+        max_length=30,
+        db_index=True,
+        blank=True,
+        null=True,
+        help_text=_('Audio codec of the downloaded media')
+    )
+    downloaded_video_codec = models.CharField(
+        _('downloaded video codec'),
+        max_length=30,
+        db_index=True,
+        blank=True,
+        null=True,
+        help_text=_('Video codec of the downloaded media')
+    )
+    downloaded_container = models.CharField(
+        _('downloaded container format'),
+        max_length=30,
+        db_index=True,
+        blank=True,
+        null=True,
+        help_text=_('Container format of the downloaded media')
+    )
+    downloaded_fps = models.PositiveSmallIntegerField(
+        _('downloaded fps'),
+        db_index=True,
+        blank=True,
+        null=True,
+        help_text=_('FPS of the downloaded media')
+    )
+    downloaded_hdr = models.BooleanField(
+        _('downloaded hdr'),
+        default=False,
+        help_text=_('Downloaded media has HDR')
+    )
+    downloaded_filesize = models.PositiveBigIntegerField(
+        _('downloaded filesize'),
+        db_index=True,
+        blank=True,
+        null=True,
+        help_text=_('Size of the downloaded media in bytes')
+    )
+
+    def __str__(self):
+        return self.key
+
+    class Meta:
+        verbose_name = _('Media')
+        verbose_name_plural = _('Media')
+        unique_together = (
+            ('source', 'key'),
+        )
+
+    def get_metadata_field(self, field):
+        fields = self.METADATA_FIELDS.get(field, {})
+        return fields.get(self.source.source_type, '')
+
+    def iter_formats(self):
+        for fmt in self.formats:
+            yield parse_media_format(fmt)
+
+    def get_best_combined_format(self):
+        return get_best_combined_format(self)
+
+    def get_best_audio_format(self):
+        return get_best_audio_format(self)
+
+    def get_best_video_format(self):
+        return get_best_video_format(self)
+
+    def get_format_str(self):
+        '''
+            Returns a youtube-dl compatible format string for the best matches
+            combination of source requirements and available audio and video formats.
+            Returns boolean False if there is no valid downloadable combo.
+        '''
+        if self.source.is_audio:
+            audio_match, audio_format = self.get_best_audio_format()
+            if audio_format:
+                return str(audio_format)
+            else:
+                return False
+        else:
+            combined_match, combined_format = self.get_best_combined_format()
+            if combined_format:
+                return str(combined_format)
+            else:
+                audio_match, audio_format = self.get_best_audio_format()
+                video_match, video_format = self.get_best_video_format()
+                if audio_format and video_format:
+                    return f'{video_format}+{audio_format}'
+                else:
+                    return False
+        return False
+    
+    def get_format_by_code(self, format_code):
+        '''
+            Matches a format code, such as '22', to a processed format dict.
+        '''
+        for fmt in self.iter_formats():
+            if format_code == fmt['id']:
+                return fmt
+        return False
+
+    @property
+    def loaded_metadata(self):
+        try:
+            return json.loads(self.metadata)
+        except Exception as e:
+            return {}
+
+    @property
+    def url(self):
+        url = self.URLS.get(self.source.source_type, '')
+        return url.format(key=self.key)
+
+    @property
+    def description(self):
+        field = self.get_metadata_field('description')
+        return self.loaded_metadata.get(field, '').strip()
+
+    @property
+    def title(self):
+        field = self.get_metadata_field('title')
+        return self.loaded_metadata.get(field, '').strip()
+
+    @property
+    def thumbnail(self):
+        field = self.get_metadata_field('thumbnail')
+        return self.loaded_metadata.get(field, '').strip()
+
+    @property
+    def name(self):
+        title = self.title
+        return title if title else self.key
+
+    @property
+    def upload_date(self):
+        field = self.get_metadata_field('upload_date')
+        upload_date_str = self.loaded_metadata.get(field, '').strip()
+        try:
+            return datetime.strptime(upload_date_str, '%Y%m%d')
+        except (AttributeError, ValueError) as e:
+            return None
+
+    @property
+    def duration(self):
+        field = self.get_metadata_field('duration')
+        return int(self.loaded_metadata.get(field, 0))
+
+    @property
+    def duration_formatted(self):
+        duration = self.duration
+        if duration > 0:
+            return seconds_to_timestr(duration)
+        return '??:??:??'
+
+    @property
+    def formats(self):
+        field = self.get_metadata_field('formats')
+        return self.loaded_metadata.get(field, [])
+
+    @property
+    def filename(self):
+        upload_date = self.upload_date
+        dateobj = upload_date if upload_date else self.created
+        datestr = dateobj.strftime('%Y-%m-%d')
+        source_name = slugify(self.source.name)
+        name = slugify(self.name.replace('&', 'and').replace('+', 'and'))[:50]
+        key = self.key.strip()
+        fmt = self.source.source_resolution.lower()
+        codecs = []
+        vcodec = self.source.source_vcodec.lower()
+        acodec = self.source.source_acodec.lower()
+        if vcodec:
+            codecs.append(vcodec)
+        if acodec:
+            codecs.append(acodec)
+        codecs = '-'.join(codecs)
+        ext = self.source.extension
+        return f'{datestr}_{source_name}_{name}_{key}-{fmt}-{codecs}.{ext}'
+
+    @property
+    def filepath(self):
+        return self.source.directory_path / self.filename
+
+    def download_media(self):
+        format_str = self.get_format_str()
+        if not format_str:
+            raise NoFormatException(f'Cannot download, media "{self.pk}" ({media}) has '
+                                    f'no valid format available')
+        # Download the media with youtube-dl
+        download_youtube_media(self.url, format_str, self.source.extension,
+                               str(self.filepath))
+        # Return the download paramaters
+        return format_str, self.source.extension

tubesync/sync/signals.py

@@ -0,0 +1,132 @@
+from django.conf import settings
+from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
+from django.dispatch import receiver
+from django.utils.translation import gettext_lazy as _
+from background_task.signals import task_failed
+from background_task.models import Task
+from common.logger import log
+from .models import Source, Media
+from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task,
+                    download_media_thumbnail, map_task_to_instance,
+                    check_source_directory_exists, download_media)
+from .utils import delete_file
+
+
+@receiver(pre_save, sender=Source)
+def source_pre_save(sender, instance, **kwargs):
+    # Triggered before a source is saved, if the schedule has been updated recreate
+    # its indexing task
+    try:
+        existing_source = Source.objects.get(pk=instance.pk)
+    except Source.DoesNotExist:
+        # Probably not possible?
+        return
+    if existing_source.index_schedule != instance.index_schedule:
+        # Indexing schedule has changed, recreate the indexing task
+        delete_task_by_source('sync.tasks.index_source_task', instance.pk)
+        verbose_name = _('Index media from source "{}"')
+        index_source_task(
+            str(instance.pk),
+            repeat=instance.index_schedule,
+            queue=str(instance.pk),
+            verbose_name=verbose_name.format(instance.name)
+        )
+
+
+@receiver(post_save, sender=Source)
+def source_post_save(sender, instance, created, **kwargs):
+    # Triggered after a source is saved, Create a new task to check the directory exists
+    check_source_directory_exists(str(instance.pk))
+    if created:
+        # Create a new indexing task for newly created sources
+        delete_task_by_source('sync.tasks.index_source_task', instance.pk)
+        log.info(f'Scheduling media indexing for source: {instance.name}')
+        verbose_name = _('Index media from source "{}"')
+        index_source_task(
+            str(instance.pk),
+            repeat=instance.index_schedule,
+            queue=str(instance.pk),
+            verbose_name=verbose_name.format(instance.name)
+        )
+    # Trigger the post_save signal for each media item linked to this source as various
+    # flags may need to be recalculated
+    for media in Media.objects.filter(source=instance):
+        media.save()
+
+
+@receiver(pre_delete, sender=Source)
+def source_pre_delete(sender, instance, **kwargs):
+    # Triggered before a source is deleted, delete all media objects to trigger
+    # the Media models post_delete signal
+    for media in Media.objects.filter(source=instance):
+        log.info(f'Deleting media for source: {instance.name} item: {media.name}')
+        media.delete()
+
+
+@receiver(post_delete, sender=Source)
+def source_post_delete(sender, instance, **kwargs):
+    # Triggered after a source is deleted
+    log.info(f'Deleting tasks for source: {instance.name}')
+    delete_task_by_source('sync.tasks.index_source_task', instance.pk)
+
+
+@receiver(task_failed, sender=Task)
+def task_task_failed(sender, task_id, completed_task, **kwargs):
+    # Triggered after a task fails by reaching its max retry attempts
+    obj, url = map_task_to_instance(completed_task)
+    if isinstance(obj, Source):
+        log.error(f'Permanent failure for source: {obj} task: {completed_task}')
+        obj.has_failed = True
+        obj.save()
+
+
+@receiver(post_save, sender=Media)
+def media_post_save(sender, instance, created, **kwargs):
+    # Triggered after media is saved
+    if created:
+        # If the media is newly created start a task to download its thumbnail
+        thumbnail_url = instance.thumbnail
+        if thumbnail_url:
+            log.info(f'Scheduling task to download thumbnail for: {instance.name} '
+                     f'from: {thumbnail_url}')
+            verbose_name = _('Downloading thumbnail for "{}"')
+            download_media_thumbnail(
+                str(instance.pk),
+                thumbnail_url,
+                queue=str(instance.source.pk),
+                verbose_name=verbose_name.format(instance.name)
+            )
+    # Recalculate the "can_download" flag, this may need to change if the source
+    # specifications have been changed
+    if instance.get_format_str():
+        if not instance.can_download:
+            instance.can_download = True
+            instance.save()
+    else:
+        if instance.can_download:
+            instance.can_download = True
+            instance.save()
+    # If the media has not yet been downloaded schedule it to be downloaded
+    if not instance.downloaded:
+        delete_task_by_media('sync.tasks.download_media', (str(instance.pk),))
+        verbose_name = _('Downloading media for "{}"')
+        download_media(
+            str(instance.pk),
+            queue=str(instance.source.pk),
+            verbose_name=verbose_name.format(instance.name)
+        )
+
+
+@receiver(pre_delete, sender=Media)
+def media_pre_delete(sender, instance, **kwargs):
+    # Triggered before media is deleted, delete any scheduled tasks
+    log.info(f'Deleting tasks for media: {instance.name}')
+    delete_task_by_media('sync.tasks.download_media', (str(instance.pk),))
+    thumbnail_url = instance.thumbnail
+    if thumbnail_url:
+        delete_task_by_media('sync.tasks.download_media_thumbnail',
+                             (str(instance.pk), thumbnail_url))
+    # Delete media thumbnail if it exists
+    if instance.thumb:
+        log.info(f'Deleting thumbnail for: {instance} path: {instance.thumb.path}')
+        delete_file(instance.thumb.path)

tubesync/sync/tasks.py

@@ -0,0 +1,270 @@
+'''
+    Start, stop and manage scheduled tasks. These are generally triggered by Django
+    signals (see signals.py).
+'''
+
+
+import os
+import json
+import math
+import uuid
+from io import BytesIO
+from hashlib import sha1
+from datetime import timedelta
+from PIL import Image
+from django.conf import settings
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.utils import timezone
+from django.db.utils import IntegrityError
+from background_task import background
+from background_task.models import Task, CompletedTask
+from common.logger import log
+from common.errors import NoMediaException, DownloadFailedException
+from .models import Source, Media
+from .utils import get_remote_image, resize_image_to_height
+
+
+def get_hash(task_name, pk):
+    '''
+        Create a background_task compatible hash for a Task or CompletedTask.
+    '''
+    task_params = json.dumps(((str(pk),), {}), sort_keys=True)
+    return sha1(f'{task_name}{task_params}'.encode('utf-8')).hexdigest()
+
+
+def map_task_to_instance(task):
+    '''
+        Reverse-maps a scheduled backgrond task to an instance. Requires the task name
+        to be a known task function and the first argument to be a UUID. This is used
+        because UUID's are incompatible with background_task's "creator" feature.
+    '''
+    TASK_MAP = {
+        'sync.tasks.index_source_task': Source,
+        'sync.tasks.check_source_directory_exists': Source,
+        'sync.tasks.download_media_thumbnail': Media,
+        'sync.tasks.download_media': Media,
+    }
+    MODEL_URL_MAP = {
+        Source: 'sync:source',
+        Media: 'sync:media-item',
+    }
+    # Unpack 
+    task_func, task_args_str = task.task_name, task.task_params
+    model = TASK_MAP.get(task_func, None)
+    if not model:
+        return None, None
+    url = MODEL_URL_MAP.get(model, None)
+    if not url:
+        return None, None
+    try:
+        task_args = json.loads(task_args_str)
+    except (TypeError, ValueError, AttributeError):
+        return None, None
+    if len(task_args) != 2:
+        return None, None
+    args, kwargs = task_args
+    if len(args) == 0:
+        return None, None
+    instance_uuid_str = args[0]
+    try:
+        instance_uuid = uuid.UUID(instance_uuid_str)
+    except (TypeError, ValueError, AttributeError):
+        return None, None
+    try:
+        instance = model.objects.get(pk=instance_uuid)
+        return instance, url
+    except model.DoesNotExist:
+        return None, None
+
+
+def get_error_message(task):
+    '''
+        Extract an error message from a failed task. This is the last line of the
+        last_error field with the method name removed.
+    '''
+    if not task.has_error():
+        return ''
+    stacktrace_lines = task.last_error.strip().split('\n')
+    if len(stacktrace_lines) == 0:
+        return ''
+    error_message = stacktrace_lines[-1].strip()
+    if ':' not in error_message:
+        return ''
+    return error_message.split(':', 1)[1].strip()
+
+
+def get_source_completed_tasks(source_id, only_errors=False):
+    '''
+        Returns a queryset of CompletedTask objects for a source by source ID.
+    '''
+    q = {'queue': source_id}
+    if only_errors:
+        q['failed_at__isnull'] = False
+    return CompletedTask.objects.filter(**q).order_by('-failed_at')
+
+
+def delete_task_by_source(task_name, source_id):
+    return Task.objects.filter(task_name=task_name, queue=str(source_id)).delete()
+
+
+def delete_task_by_media(task_name, args):
+    return Task.objects.drop_task(task_name, args=args)
+
+
+def cleanup_completed_tasks():
+    days_to_keep = getattr(settings, 'COMPLETED_TASKS_DAYS_TO_KEEP', 30)
+    delta = timezone.now() - timedelta(days=days_to_keep)
+    log.info(f'Deleting completed tasks older than {days_to_keep} days '
+             f'(run_at before {delta})')
+    CompletedTask.objects.filter(run_at__lt=delta).delete()
+
+
+@background(schedule=0)
+def index_source_task(source_id):
+    '''
+        Indexes media available from a Source object.
+    '''
+    try:
+        source = Source.objects.get(pk=source_id)
+    except Source.DoesNotExist:
+        # Task triggered but the Source has been deleted, delete the task
+        delete_index_source_task(source_id)
+        return
+    # Reset any errors
+    source.has_failed = False
+    source.save()
+    # Index the source
+    videos = source.index_media()
+    if not videos:
+        raise NoMediaException(f'Source "{source}" (ID: {source_id}) returned no '
+                               f'media to index, is the source key valid? Check the '
+                               f'source configuration is correct and that the source '
+                               f'is reachable')
+    # Got some media, update the last crawl timestamp
+    source.last_crawl = timezone.now()
+    source.save()
+    log.info(f'Found {len(videos)} media items for source: {source}')
+    for video in videos:
+        # Create or update each video as a Media object
+        key = video.get(source.key_field, None)
+        if not key:
+            # Video has no unique key (ID), it can't be indexed
+            continue
+        try:
+            media = Media.objects.get(key=key)
+        except Media.DoesNotExist:
+            media = Media(key=key)
+        media.source = source
+        media.metadata = json.dumps(video)
+        upload_date = media.upload_date
+        if upload_date:
+            media.published = timezone.make_aware(upload_date)
+        try:
+            media.save()
+            log.info(f'Indexed media: {source} / {media}')
+        except IntegrityError as e:
+            log.error(f'Index media failed: {source} / {media} with "{e}"')
+    # Tack on a cleanup of old completed tasks
+    cleanup_completed_tasks()
+
+
+
+@background(schedule=0)
+def check_source_directory_exists(source_id):
+    '''
+        Checks the output directory for a source exists and is writable, if it does
+        not attempt to create it. This is a task so if there are permission errors
+        they are logged as failed tasks.
+    '''
+    try:
+        source = Source.objects.get(pk=source_id)
+    except Source.DoesNotExist:
+        # Task triggered but the Source has been deleted, delete the task
+        delete_index_source_task(source_id)
+        return
+    # Check the source output directory exists
+    if not source.directory_exists():
+        # Try and create it
+        log.info(f'Creating directory: {source.directory_path}')
+        source.make_directory()
+
+
+@background(schedule=0)
+def download_media_thumbnail(media_id, url):
+    '''
+        Downloads an image from a URL and save it as a local thumbnail attached to a
+        Media instance.
+    '''
+    try:
+        media = Media.objects.get(pk=media_id)
+    except Media.DoesNotExist:
+        # Task triggered but the media no longer exists, do nothing
+        return
+    width = getattr(settings, 'MEDIA_THUMBNAIL_WIDTH', 430)
+    height = getattr(settings, 'MEDIA_THUMBNAIL_HEIGHT', 240)
+    i = get_remote_image(url)
+    log.info(f'Resizing {i.width}x{i.height} thumbnail to '
+             f'{width}x{height}: {url}')
+    i = resize_image_to_height(i, width, height)
+    image_file = BytesIO()
+    i.save(image_file, 'JPEG', quality=80, optimize=True, progressive=True)
+    image_file.seek(0)
+    media.thumb.save(
+        'thumb',
+        SimpleUploadedFile(
+            'thumb',
+            image_file.read(),
+            'image/jpeg',
+        ),
+        save=True
+    )
+    log.info(f'Saved thumbnail for: {media} from: {url}')
+    return True
+
+
+@background(schedule=0)
+def download_media(media_id):
+    '''
+        Downloads the media to disk and attaches it to the Media instance.
+    '''
+    try:
+        media = Media.objects.get(pk=media_id)
+    except Media.DoesNotExist:
+        # Task triggered but the media no longer exists, do nothing
+        return
+    log.info(f'Downloading media: {media} (UUID: {media.pk}) to: {media.filepath}')
+    format_str, container = media.download_media()
+    if os.path.exists(media.filepath):
+        # Media has been downloaded successfully
+        log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: '
+                 f'{media.filepath}')
+        # Link the media file to the object and update info about the download
+        media.media_file.name = str(media.filepath)
+        media.downloaded = True
+        if '+' in format_str:
+            vformat_code, aformat_code = format_str.split('+')
+            aformat = media.get_format_by_code(aformat_code)
+            vformat = media.get_format_by_code(vformat_code)
+            media.downloaded_audio_codec = aformat['acodec']
+            media.downloaded_video_codec = vformat['vcodec']
+            media.downloaded_container = container
+            media.downloaded_fps = vformat['fps']
+            media.downloaded_hdr = vformat['is_hdr']
+            media.downloaded_filesize = os.path.getsize(media.filepath)
+        else:
+            cformat_code = format_str
+            cformat = media.get_format_by_code(cformat_code)
+            media.downloaded_audio_codec = cformat['acodec']
+            media.downloaded_video_codec = cformat['vcodec']
+            media.downloaded_container = container
+            media.downloaded_fps = cformat['fps']
+            media.downloaded_hdr = cformat['is_hdr']
+            media.downloaded_filesize = os.path.getsize(media.filepath)
+        media.save()
+    else:
+        # Expected file doesn't exist on disk
+        err = (f'Failed to download media: {media} (UUID: {media.pk}) to disk, '
+               f'expected outfile does not exist: {media.filepath}')
+        log.error(err)
+        # Raising an error here triggers the task to be re-attempted (or fail)
+        raise DownloadFailedException(err)

tubesync/sync/templates/sync/dashboard.html

@@ -0,0 +1,20 @@
+{% extends 'base.html' %}
+
+{% block headtitle %}Dashboard{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12">
+    <h1 class="truncate">Dashboard</h1>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    <div class="card">
+      <div class="card-content">
+        intro
+      </div>
+    </div>
+  </div>
+</div>
+{% endblock %}

tubesync/sync/templates/sync/media-item.html

@@ -0,0 +1,109 @@
+{% extends 'base.html' %}{% load static %}
+
+{% block headtitle %}Media - {{ media.key }}{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12">
+    <h1 class="truncate">Media <strong>{{ media.key }}</strong></h1>
+    {% if media.title %}<h2 class="truncate"><strong>{{ media.title }}</strong></h2>{% endif %}
+    <p class="truncate"><strong><a href="{{ media.url }}" target="_blank"><i class="fas fa-link"></i> {{ media.url }}</a></strong></p>
+    <p class="truncate">Downloading to: <strong>{{ media.source.directory_path }}</strong></p>
+  </div>
+</div>
+{% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
+<div class="row">
+  <div class="col s12 m7">
+    <div><i class="fas fa-quote-left"></i></div>
+    <p>{% if media.description %}{{ media.description|truncatewords:200 }}{% else %}(Media has no description).{% endif %}</p>
+    <div class="right-align"><i class="fas fa-quote-right"></i></div>
+  </div>
+  <div class="col s12 m5">
+    <div class="card mediacard">
+      <div class="card-image">
+        <img src="{% if media.thumb %}{% url 'sync:media-thumb' pk=media.pk %}{% else %}{% static 'images/nothumb.png' %}{% endif %}">
+      </div>
+    </div>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    <table class="striped">
+      <tr title="The media source">
+        <td class="hide-on-small-only">Source</td>
+        <td><span class="hide-on-med-and-up">Source<br></span><strong><a href="{% url 'sync:source' pk=media.source.pk %}">{{ media.source }}</a></strong></td>
+      </tr>
+      <tr title="The media duration">
+        <td class="hide-on-small-only">Duration</td>
+        <td><span class="hide-on-med-and-up">Duration<br></span><strong>{{ media.duration_formatted }}</strong></td>
+      </tr>
+      <tr title="The desired format">
+        <td class="hide-on-small-only">Desired format</td>
+        <td><span class="hide-on-med-and-up">Desired format<br></span><strong>{{ media.source.format_summary }}</strong></td>
+      </tr>
+      <tr title="Fallback setting on the source">
+        <td class="hide-on-small-only">Fallback</td>
+        <td><span class="hide-on-med-and-up">Fallback<br></span><strong>{{ media.source.get_fallback_display }}</strong></td>
+      </tr>
+      <tr title="Has the media been downloaded?">
+        <td class="hide-on-small-only">Downloaded?</td>
+        <td><span class="hide-on-med-and-up">Downloaded?<br></span><strong>{% if media.downloaded %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
+      {% if media.downloaded %}
+      <tr title="The filename the media will be downloaded as">
+        <td class="hide-on-small-only">Filename</td>
+        <td><span class="hide-on-med-and-up">Filename<br></span><strong>{{ media.filename }}</strong></td>
+      </tr>
+      <tr title="Size of the file on disk">
+        <td class="hide-on-small-only">File size</td>
+        <td><span class="hide-on-med-and-up">File size<br></span><strong>{{ media.downloaded_filesize|filesizeformat }}</strong></td>
+      </tr>
+      <tr title="Codecs used in the downloaded file">
+        <td class="hide-on-small-only">Downloaded codecs</td>
+        <td><span class="hide-on-med-and-up">Downloaded codecs<br></span><strong>audio:{{ media.downloaded_audio_codec }}{% if media.downloaded_video_codec %}, video:{{ media.downloaded_video_codec }}{% endif %}</strong></td>
+      </tr>
+      <tr title="Container file format used in the download file">
+        <td class="hide-on-small-only">Container</td>
+        <td><span class="hide-on-med-and-up">Container<br></span><strong>{{ media.downloaded_container|upper }}</strong></td>
+      </tr>
+      <tr title="Frames per second in the downloaded file">
+        <td class="hide-on-small-only">Downloaded FPS</td>
+        <td><span class="hide-on-med-and-up">Downloaded FPS<br></span><strong>{{ media.downloaded_fps }} FPS</strong></td>
+      </tr>
+      <tr title="Does the downloaded file have high dynamic range?">
+        <td class="hide-on-small-only">Downloaded HDR?</td>
+        <td><span class="hide-on-med-and-up">Downloaded HDR?<br></span><strong>{% if media.downloaded_hdr %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
+      {% else %}
+      <tr title="Can the media be downloaded?">
+        <td class="hide-on-small-only">Can download?</td>
+        <td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if youtube_dl_format %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
+      {% endif %}
+      <tr title="The available media formats">
+        <td class="hide-on-small-only">Available formats</td>
+        <td><span class="hide-on-med-and-up">Available formats<br></span>
+          {% for format in media.formats %}
+          <span class="truncate">
+            ID: <strong>{{ format.format_id }}</strong>
+            {% if format.vcodec|lower != 'none' %}, <strong>{{ format.format_note }} ({{ format.width }}x{{ format.height }})</strong>, fps:<strong>{{ format.fps|lower }}</strong>, video:<strong>{{ format.vcodec }} @{{ format.tbr }}k</strong>{% endif %}
+            {% if format.acodec|lower != 'none' %}, audio:<strong>{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz</strong>{% endif %}
+            {% if format.format_id == combined_format or format.format_id == audio_format or format.format_id == video_format %}<strong>(matched)</strong>{% endif %}
+          </span>
+          {% empty %}
+          Media has no indexed available formats
+          {% endfor %}
+        </td>
+      </tr>
+      <tr title="Best available format for source requirements">
+        <td class="hide-on-small-only">Matched formats</td>
+        <td><span class="hide-on-med-and-up">Matched formats<br></span>
+          Combined: <strong>{% if combined_format %}{{ combined_format }} {% if combined_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}</strong><br>
+          Audio: <strong>{% if audio_format %}{{ audio_format }} {% if audio_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}</strong><br>
+          Video: <strong>{% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}
+        </strong></td>
+      </tr>
+    </table>
+  </div>
+</div>
+{% endblock %}

tubesync/sync/templates/sync/media.html

@@ -0,0 +1,36 @@
+{% extends 'base.html' %}{% load static %}
+
+{% block headtitle %}Media{% if source %} - {{ source }}{% endif %}{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12">
+    <h1 class="truncate">Media</h1>
+  </div>
+</div>
+{% include 'infobox.html' with message=message %}
+<div class="row no-margin-bottom">
+  {% for m in media %}
+  <div class="col s12 m6 l4 xl3">
+    <div class="card mediacard">
+      <a href="{% url 'sync:media-item' pk=m.pk %}" class="">
+        <div class="card-image">
+          <img src="{% if m.thumb %}{% url 'sync:media-thumb' pk=m.pk %}{% else %}{% static 'images/nothumb.png' %}{% endif %}">
+          <span class="card-title truncate">{{ m.source }}<br>
+            <span>{{ m.name }}</span><br>
+            <span>{% if m.can_download %}{% if m.downloaded %}<i class="fas fa-check-circle" title="Downloaded"></i>{% else %}<i class="far fa-clock" title="Queued waiting to download"></i>{% endif %} {{ m.published|date:'Y-m-d' }}{% else %}<i class="fas fa-exclamation-triangle"></i> No matching formats{% endif %}</span>
+          </span>
+        </div>
+      </a>
+    </div>
+  </div>
+  {% empty %}
+  <div class="col s12">
+    <div class="collection">
+      <span class="collection-item no-items"><i class="fas fa-info-circle"></i> No media has been indexed{% if source %} that matches the specified source filter{% endif %}.</span>
+    </div>
+  </div>
+  {% endfor %}
+</div>
+{% include 'pagination.html' with pagination=sources.paginator filter=source.pk %}
+{% endblock %}

tubesync/sync/templates/sync/source-add.html

@@ -0,0 +1,26 @@
+{% extends 'base.html' %}
+
+{% block headtitle %}Add a new source{% endblock %}
+
+{% block content %}
+<div class="row no-margin-bottom">
+  <div class="col s12">
+    <h1>Add a source</h1>
+    <p>
+      You can use this form to add a new source. A source is what's polled on regular
+      basis to find new media to download, such as a channel or playlist.
+    </p>
+  </div>
+</div>
+<div class="row">
+  <form method="post" action="{% url 'sync:add-source' %}" class="col s12 simpleform">
+    {% csrf_token %}
+    {% include 'simpleform.html' with form=form %}
+    <div class="row no-margin-bottom padding-top">
+      <div class="col s12">
+        <button class="btn" type="submit" name="action">Add source <i class="fas fa-fw fa-plus"></i></button>
+      </div>
+    </div>
+  </form>
+</div>
+{% endblock %}

tubesync/sync/templates/sync/source-delete.html

@@ -0,0 +1,28 @@
+{% extends 'base.html' %}
+
+{% block headtitle %}Delete source - {{ source.name }}{% endblock %}
+
+{% block content %}
+<div class="row no-margin-bottom">
+  <div class="col s12">
+    <h1>Delete source <strong>{{ source.name }}</strong></h1>
+    <p>
+      Are you sure you want to delete this source? Deleting a source is permanent.
+      By default, deleting a source does not delete any saved media files. You can
+      tick the &quot;also delete downloaded media&quot; checkbox to also remove save
+      media when you delete the source. Deleting a source cannot be undone.
+    </p>
+  </div>
+</div>
+<div class="row">
+  <form method="post" action="{% url 'sync:delete-source' pk=source.pk %}" class="col s12 simpleform">
+    {% csrf_token %}
+    {% include 'simpleform.html' with form=form %}
+    <div class="row no-margin-bottom padding-top">
+      <div class="col s12">
+        <button class="btn" type="submit" name="action">Really delete source <i class="fas fa-trash-alt"></i></button>
+      </div>
+    </div>
+  </form>
+</div>
+{% endblock %}

tubesync/sync/templates/sync/source-update.html

@@ -0,0 +1,28 @@
+{% extends 'base.html' %}
+
+{% block headtitle %}Update source - {{ source.name }}{% endblock %}
+
+{% block content %}
+<div class="row no-margin-bottom">
+  <div class="col s12">
+    <h1>Update source <strong>{{ source.name }}</strong></h1>
+    <p>
+      You can use this form to update your source. A source is what's polled on regular
+      basis to find new media to download, such as a channel or playlist. Any changes
+      to a source will only apply to new media and will not update media previously
+      downloaded.
+    </p>
+  </div>
+</div>
+<div class="row">
+  <form method="post" action="{% url 'sync:update-source' pk=source.pk %}" class="col s12 simpleform">
+    {% csrf_token %}
+    {% include 'simpleform.html' with form=form %}
+    <div class="row no-margin-bottom padding-top">
+      <div class="col s12">
+        <button class="btn" type="submit" name="action">Update source <i class="fas fa-pen-square"></i></button>
+      </div>
+    </div>
+  </form>
+</div>
+{% endblock %}

tubesync/sync/templates/sync/source-validate.html

@@ -0,0 +1,24 @@
+{% extends 'base.html' %}
+
+{% block headtitle %}Validate a {{ help_item }}{% endblock %}
+
+{% block content %}
+<div class="row no-margin-bottom">
+  <div class="col s12">
+    <h1>Validate a {{ help_item }}</h1>
+    <p>{{ help_text|safe }}</p>
+    <p>Example: <strong>{{ help_example }}</strong></p>
+  </div>
+</div>
+<div class="row">
+  <form method="post" action="{% url 'sync:validate-source' source_type=source_type %}" class="col s12 simpleform">
+    {% csrf_token %}
+    {% include 'simpleform.html' with form=form %}
+    <div class="row no-margin-bottom padding-top">
+      <div class="col s12">
+        <button class="btn" type="submit" name="action">Validate {{ help_item }} <i class="fas fa-check"></i></button>
+      </div>
+    </div>
+  </form>
+</div>
+{% endblock %}

tubesync/sync/templates/sync/source.html

@@ -0,0 +1,130 @@
+{% extends 'base.html' %}
+
+{% block headtitle %}Source - {{ source.name }}{% endblock %}
+
+{% block content %}
+<div class="row no-margin-bottom">
+  <div class="col s12">
+    <h1 class="truncate">Source <strong>{{ source.name }}</strong></h1>
+    <p class="truncate"><strong><a href="{{ source.url }}" target="_blank"><i class="fas fa-link"></i> {{ source.url }}</a></strong></p>
+    <p class="truncate">Saving to: <strong>{{ source.directory_path }}</strong></p>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12 l6 margin-bottom">
+    <a href="{% url 'sync:media' %}?filter={{ source.pk }}" class="btn">View media<span class="hide-on-small-only"> linked to this source</span> <i class="fas fa-fw fa-film"></i></a>
+  </div>
+  <div class="col s12 l6 margin-bottom">
+    <a href="{% url 'sync:tasks-completed' %}?filter={{ source.pk }}" class="btn">View tasks<span class="hide-on-small-only"> linked to this source</span> <i class="far fa-fw fa-clock"></i></a>
+  </div>
+</div>
+{% include 'infobox.html' with message=message %}
+{% if source.has_failed %}{% include 'errorbox.html' with message='This source has encountered permanent failures listed at the bottom of this page, check its settings' %}{% endif %}
+<div class="row">
+  <div class="col s12">
+    <table class="striped">
+      <tr title="The source type">
+        <td class="hide-on-small-only">Type</td>
+        <td><span class="hide-on-med-and-up">Type<br></span><strong>{{ source.get_source_type_display }}</strong></td>
+      </tr>
+      <tr title="Name of the souce in TubeSync for your reference">
+        <td class="hide-on-small-only">Name</td>
+        <td><span class="hide-on-med-and-up">Name<br></span><strong>{{ source.name }}</strong></td>
+      </tr>
+      <tr title="Number of media items downloaded for the source">
+        <td class="hide-on-small-only">Media items</td>
+        <td><span class="hide-on-med-and-up">Media items<br></span><strong><a href="{% url 'sync:media' %}?filter={{ source.pk }}">{{ media|length }}</a></strong></td>
+      </tr>
+      <tr title="Unique key of the source, such as the channel name or playlist ID">
+        <td class="hide-on-small-only">Key</td>
+        <td><span class="hide-on-med-and-up">Key<br></span><strong>{{ source.key }}</strong></td>
+      </tr>
+      <tr title="Directory the source will save media to">
+        <td class="hide-on-small-only">Directory</td>
+        <td><span class="hide-on-med-and-up">Directory<br></span><strong>{{ source.directory }}</strong></td>
+      </tr>
+      <tr title="Schedule of how often to index the source for new media">
+        <td class="hide-on-small-only">Index schedule</td>
+        <td><span class="hide-on-med-and-up">Index schedule<br></span><strong>{{ source.get_index_schedule_display }}</strong></td>
+      </tr>
+      <tr title="When then source was created locally in TubeSync">
+        <td class="hide-on-small-only">Created</td>
+        <td><span class="hide-on-med-and-up">Created<br></span><strong>{{ source.created|date:'Y-m-d H:i:s' }}</strong></td>
+      </tr>
+      <tr title="When the source last checked for available media">
+        <td class="hide-on-small-only">Last crawl</td>
+        <td><span class="hide-on-med-and-up">Last crawl<br></span><strong>{% if source.last_crawl %}{{ source.last_crawl|date:'Y-m-d H-I-S' }}{% else %}Never{% endif %}</strong></td>
+      </tr>
+      <tr title="Quality and type of media the source will attempt to sync">
+        <td class="hide-on-small-only">Source resolution</td>
+        <td><span class="hide-on-med-and-up">Source resolution<br></span><strong>{{ source.get_source_resolution_display }}</strong></td>
+      </tr>
+      {% if source.is_video %}
+      <tr title="Preferred video codec to download">
+        <td class="hide-on-small-only">Source video codec</td>
+        <td><span class="hide-on-med-and-up">Source video codec<br></span><strong>{{ source.get_source_vcodec_display }}</strong></td>
+      </tr>
+      {% endif %}
+      <tr title="Preferred audio codec to download">
+        <td class="hide-on-small-only">Source audio codec</td>
+        <td><span class="hide-on-med-and-up">Source audio codec<br></span><strong>{{ source.get_source_acodec_display }}</strong></td>
+      </tr>
+      <tr title="If available from the source media in 60FPS will be preferred">
+        <td class="hide-on-small-only">Prefer 60FPS?</td>
+        <td><span class="hide-on-med-and-up">Prefer 60FPS?<br></span><strong>{% if source.prefer_60fps %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
+      <tr title="If available from the source media in HDR will be preferred">
+        <td class="hide-on-small-only">Prefer HDR?</td>
+        <td><span class="hide-on-med-and-up">Prefer HDR?<br></span><strong>{% if source.prefer_hdr %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
+      <tr title="Output file extension">
+        <td class="hide-on-small-only">Output extension</td>
+        <td><span class="hide-on-med-and-up">Output extension<br></span><strong>{{ source.extension }}</strong></td>
+      </tr>
+      <tr title="What to do if your source resolution or codecs are unavailable">
+        <td class="hide-on-small-only">Fallback</td>
+        <td><span class="hide-on-med-and-up">Fallback<br></span><strong>{{ source.get_fallback_display }}</strong></td>
+      </tr>
+      {% if source.delete_old_media and source.days_to_keep > 0 %}
+      <tr title="Days after which your media from this source will be locally deleted">
+        <td class="hide-on-small-only">Delete old media</td>
+        <td><span class="hide-on-med-and-up">Delete old media<br></span><strong>After {{ source.days_to_keep }} days</strong></td>
+      </tr>
+      {% else %}
+      <tr title="Media from this source will never be deleted">
+        <td class="hide-on-small-only">Delete old media</td>
+        <td><span class="hide-on-med-and-up">Delete old media<br></span><strong>No, keep forever</strong></td>
+      </tr>
+      {% endif %}
+      <tr title="Unique ID used for this source in TubeSync">
+        <td class="hide-on-small-only">UUID</td>
+        <td><span class="hide-on-med-and-up">UUID<br></span><strong>{{ source.uuid }}</strong></td>
+      </tr>
+    </table>
+  </div>
+</div>
+<div class="row no-margin-bottom padding-top">
+  <div class="col s12 l6 margin-bottom">
+    <a href="{% url 'sync:update-source' pk=source.pk %}" class="btn">Edit source <i class="fas fa-pen-square"></i></a>
+  </div>
+  <div class="col s12 l6 margin-bottom">
+    <a href="{% url 'sync:delete-source' pk=source.pk %}" class="btn delete-button">Delete source <i class="fas fa-trash-alt"></i></a>
+  </div>
+</div>
+{% if errors %}
+<div class="row">
+  <div class="col s12">
+    <h2>Source has encountered {{ errors|length }} Error{{ errors|length|pluralize }}</h2>
+    <div class="collection">
+      {% for task in errors %}
+        <span class="collection-item error-text">
+          <i class="fas fa-exclamation-triangle"></i> <strong>{{ task.verbose_name }}</strong><br>
+          Error: &quot;{{ task.error_message }}&quot;<br>
+          <i class="far fa-clock"></i> Occured at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>
+        </span>
+      {% endfor %}
+    </div>
+  </div>
+</div>
+{% endif %}
+{% endblock %}

tubesync/sync/templates/sync/sources.html

@@ -0,0 +1,40 @@
+{% extends 'base.html' %}
+
+{% block headtitle %}Sources{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12">
+    <h1 class="truncate">Sources</h1>
+  </div>
+</div>
+{% include 'infobox.html' with message=message %}
+<div class="row">
+  <div class="col s12 l6 margin-bottom">
+    <a href="{% url 'sync:validate-source' source_type='youtube-channel' %}" class="btn">Add a YouTube channel <i class="fab fa-youtube"></i></a>
+  </div>
+  <div class="col s12 l6 margin-bottom">
+    <a href="{% url 'sync:validate-source' source_type='youtube-playlist' %}" class="btn">Add a YouTube playlist <i class="fab fa-youtube"></i></a>
+  </div>
+</div>
+<div class="row no-margin-bottom">
+  <div class="col s12">
+    <div class="collection">
+    {% for source in sources %}
+      <a href="{% url 'sync:source' pk=source.pk %}" class="collection-item">
+        {{ source.icon|safe }} <strong>{{ source.name }}</strong> ({{ source.get_source_type_display }} &quot;{{ source.key }}&quot;)<br>
+        {{ source.format_summary }}<br>
+        {% if source.has_failed %}
+        <span class="error-text"><i class="fas fa-exclamation-triangle"></i> <strong>Source has permanent failures</strong></span>
+        {% else %}
+        <strong>{{ source.media_count }}</strong> media items{% if source.delete_old_media and source.days_to_keep > 0 %}, keep {{ source.days_to_keep }} days of media{% endif %}
+        {% endif %}
+      </a>
+    {% empty %}
+      <span class="collection-item no-items"><i class="fas fa-info-circle"></i> You haven't added any sources.</span>
+    {% endfor %}
+    </div>
+  </div>
+</div>
+{% include 'pagination.html' with pagination=sources.paginator %}
+{% endblock %}

tubesync/sync/templates/sync/tasks-completed.html

@@ -0,0 +1,39 @@
+{% extends 'base.html' %}
+
+{% block headtitle %}Tasks - Completed{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12">
+    <h1 class="truncate">Completed tasks</h1>
+  </div>
+</div>
+{% include 'infobox.html' with message=message %}
+<div class="row">
+  <div class="col s12">
+    <div class="collection">
+    {% for task in tasks %}
+      
+      {% if task.has_error %}
+      <span class="collection-item">
+        <i class="fas fa-exclamation-triangle"></i> <strong>{{ task.verbose_name }}</strong><br>
+        Source: &quot;{{ task.queue }}&quot;<br>
+        Error: &quot;{{ task.error_message }}&quot;<br>
+        <i class="far fa-clock"></i> Task ran at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>
+      </span>
+      {% else %}
+      <span class="collection-item">
+        <i class="fas fa-check"></i> <strong>{{ task.verbose_name }}</strong><br>
+        Source: &quot;{{ task.queue }}&quot;<br>
+        <i class="far fa-clock"></i> Task ran at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>
+      </span>
+      {% endif %}
+      </span>
+    {% empty %}
+      <span class="collection-item no-items"><i class="fas fa-info-circle"></i> There have been no completed tasks{% if source %} that match the specified source filter{% endif %}.</span>
+    {% endfor %}
+    </div>
+  </div>
+</div>
+{% include 'pagination.html' with pagination=sources.paginator filter=source.pk %}
+{% endblock %}

tubesync/sync/templates/sync/tasks.html

@@ -0,0 +1,87 @@
+{% extends 'base.html' %}
+
+{% block headtitle %}Tasks{% endblock %}
+
+{% block content %}
+<div class="row">
+  <div class="col s12">
+    <h1>Tasks</h1>
+    <p>
+      Tasks are the background work that TubeSync undertakes to index and download
+      media. This page allows you to see basic overview of what is running and what is
+      scheduled to perform in the future as well as check up on any errors that might
+      have occured.
+    </p>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    <h2>{{ running|length }} Running</h2>
+    <p>
+      Running tasks are tasks which currently being worked on right now.
+    </p>
+    <div class="collection">
+      {% for task in running %}
+        <a href="{% url task.url pk=task.instance.pk %}" class="collection-item">
+          <i class="fas fa-running"></i> <strong>{{ task }}</strong><br>
+          <i class="far fa-clock"></i> Task started at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>
+        </a>
+      {% empty %}
+        <span class="collection-item no-items"><i class="fas fa-info-circle"></i> There are no running tasks.</span>
+      {% endfor %}
+    </div>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    <h2>{{ errors|length }} Error{{ errors|length|pluralize }}</h2>
+    <p>
+      Tasks which generated an error are shown here. Tasks are retried a couple of
+      times, so if there was an intermittent error such as a download got interrupted
+      it will be scheduled to run again.
+    </p>
+    <div class="collection">
+      {% for task in errors %}
+        <a href="{% url task.url pk=task.instance.pk %}" class="collection-item error-text">
+          <i class="fas fa-exclamation-triangle"></i> <strong>{{ task }}</strong>, attempted {{ task.attempts }} time{{ task.attempts|pluralize }}<br>
+          Error: &quot;{{ task.error_message }}&quot;<br>
+          <i class="fas fa-history"></i> Task will be retried at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>
+        </a>
+      {% empty %}
+        <span class="collection-item no-items"><i class="fas fa-info-circle"></i> There are no tasks with errors.</span>
+      {% endfor %}
+    </div>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    <h2>{{ scheduled|length }} Scheduled</h2>
+    <p>
+      Tasks which are scheduled to run in the future or are waiting in a queue to be
+      processed. They can be waiting for an available worker to run immediately, or
+      run in the future at the specified &quot;run at&quot; time.
+    </p>
+    <div class="collection">
+      {% for task in scheduled %}
+        <a href="{% url task.url pk=task.instance.pk %}" class="collection-item">
+          <i class="fas fa-stopwatch"></i> <strong>{{ task }}</strong><br>
+          {% if task.instance.index_schedule %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
+          <i class="fas fa-redo"></i> Task will next run at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>
+        </a>
+      {% empty %}
+        <span class="collection-item no-items"><i class="fas fa-info-circle"></i> There are no scheduled tasks.</span>
+      {% endfor %}
+    </div>
+  </div>
+</div>
+<div class="row">
+  <div class="col s12">
+    <h2>Completed</h2>
+    <p>
+      A record of recently completed tasks is kept for a few days. You can use the button
+      below to view recent tasks which have completed successfully.
+    </p>
+    <a href="{% url 'sync:tasks-completed' %}" class="btn"><span class="hide-on-med-and-down">View </span>Completed tasks <i class="fas fa-check-double"></i></a>
+  </div>
+</div>
+{% endblock %}

tubesync/sync/testdata/README.md

@@ -0,0 +1,8 @@
+# metadata
+
+This directory contains metadata extracted from some test YouTube videos with
+youtube-dl.
+
+They are used to test (with `sync/tests.py`) the format matchers in `sync/matching.py`
+and are not otherwise used in TubeSync. Removing this directory will not break TubeSync
+but will break test running.

tubesync/sync/testdata/metadata.json

@@ -0,0 +1,342 @@
+{
+    "id":"no fancy stuff",
+    "upload_date":"20170911",
+    "license":null,
+    "creator":null,
+    "title":"no fancy stuff",
+    "alt_title":null,
+    "description":"no fancy stuff",
+    "categories":[],
+    "tags":[],
+    "subtitles":{},
+    "automatic_captions":{},
+    "duration":401.0,
+    "age_limit":0,
+    "annotations":null,
+    "chapters":null,
+    "formats":[
+        {
+            "format_id":"249",
+            "player_url":null,
+            "ext":"webm",
+            "format_note":"tiny",
+            "acodec":"opus",
+            "abr":50,
+            "asr":48000,
+            "filesize":2579310,
+            "fps":null,
+            "height":null,
+            "tbr":59.781,
+            "width":null,
+            "vcodec":"none",
+            "format":"249 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"250",
+            "player_url":null,
+            "ext":"webm",
+            "format_note":"tiny",
+            "acodec":"opus",
+            "abr":70,
+            "asr":48000,
+            "filesize":3362359,
+            "fps":null,
+            "height":null,
+            "tbr":78.829,
+            "width":null,
+            "vcodec":"none",
+            "format":"250 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"140",
+            "player_url":null,
+            "ext":"m4a",
+            "format_note":"tiny",
+            "acodec":"mp4a.40.2",
+            "abr":128,
+            "container":"m4a_dash",
+            "asr":44100,
+            "filesize":6365654,
+            "fps":null,
+            "height":null,
+            "tbr":128.191,
+            "width":null,
+            "vcodec":"none",
+            "format":"140 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"251",
+            "player_url":null,
+            "ext":"webm",
+            "format_note":"tiny",
+            "acodec":"opus",
+            "abr":160,
+            "asr":48000,
+            "filesize":6669827,
+            "fps":null,
+            "height":null,
+            "tbr":156.344,
+            "width":null,
+            "vcodec":"none",
+            "format":"251 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"278",
+            "player_url":null,
+            "ext":"webm",
+            "height":144,
+            "format_note":"144p",
+            "container":"webm",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":4080226,
+            "fps":24,
+            "tbr":101.676,
+            "width":256,
+            "acodec":"none",
+            "format":"278 - 256x144 (144p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"160",
+            "player_url":null,
+            "ext":"mp4",
+            "height":144,
+            "format_note":"144p",
+            "vcodec":"avc1.4d400c",
+            "asr":null,
+            "filesize":2277074,
+            "fps":24,
+            "tbr":115.609,
+            "width":256,
+            "acodec":"none",
+            "format":"160 - 256x144 (144p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"242",
+            "player_url":null,
+            "ext":"webm",
+            "height":240,
+            "format_note":"240p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":6385245,
+            "fps":24,
+            "tbr":229.335,
+            "width":426,
+            "acodec":"none",
+            "format":"242 - 426x240 (240p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"133",
+            "player_url":null,
+            "ext":"mp4",
+            "height":240,
+            "format_note":"240p",
+            "vcodec":"avc1.4d4015",
+            "asr":null,
+            "filesize":4049342,
+            "fps":24,
+            "tbr":268.066,
+            "width":426,
+            "acodec":"none",
+            "format":"133 - 426x240 (240p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"243",
+            "player_url":null,
+            "ext":"webm",
+            "height":360,
+            "format_note":"360p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":11534488,
+            "fps":24,
+            "tbr":431.644,
+            "width":640,
+            "acodec":"none",
+            "format":"243 - 640x360 (360p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"134",
+            "player_url":null,
+            "ext":"mp4",
+            "height":360,
+            "format_note":"360p",
+            "vcodec":"avc1.4d401e",
+            "asr":null,
+            "filesize":9802128,
+            "fps":24,
+            "tbr":682.225,
+            "width":640,
+            "acodec":"none",
+            "format":"134 - 640x360 (360p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"244",
+            "player_url":null,
+            "ext":"webm",
+            "height":480,
+            "format_note":"480p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":18862435,
+            "fps":24,
+            "tbr":796.032,
+            "width":854,
+            "acodec":"none",
+            "format":"244 - 854x480 (480p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"135",
+            "player_url":null,
+            "ext":"mp4",
+            "height":480,
+            "format_note":"480p",
+            "vcodec":"avc1.4d401e",
+            "asr":null,
+            "filesize":19207551,
+            "fps":24,
+            "tbr":1237.434,
+            "width":854,
+            "acodec":"none",
+            "format":"135 - 854x480 (480p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"247",
+            "player_url":null,
+            "ext":"webm",
+            "height":720,
+            "format_note":"720p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":36550937,
+            "fps":24,
+            "tbr":1585.561,
+            "width":1280,
+            "acodec":"none",
+            "format":"247 - 1280x720 (720p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"136",
+            "player_url":null,
+            "ext":"mp4",
+            "height":720,
+            "format_note":"720p",
+            "vcodec":"avc1.4d401f",
+            "asr":null,
+            "filesize":34995244,
+            "fps":24,
+            "tbr":2362.234,
+            "width":1280,
+            "acodec":"none",
+            "format":"136 - 1280x720 (720p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"248",
+            "player_url":null,
+            "ext":"webm",
+            "height":1080,
+            "format_note":"1080p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":63659748,
+            "fps":24,
+            "tbr":2747.461,
+            "width":1920,
+            "acodec":"none",
+            "format":"248 - 1920x1080 (1080p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"137",
+            "player_url":null,
+            "ext":"mp4",
+            "height":1080,
+            "format_note":"1080p",
+            "vcodec":"avc1.640028",
+            "asr":null,
+            "filesize":62770019,
+            "fps":24,
+            "tbr":4155.019,
+            "width":1920,
+            "acodec":"none",
+            "format":"137 - 1920x1080 (1080p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"18",
+            "player_url":null,
+            "ext":"mp4",
+            "width":640,
+            "height":360,
+            "acodec":"mp4a.40.2",
+            "abr":96,
+            "vcodec":"avc1.42001E",
+            "asr":44100,
+            "filesize":21611729,
+            "format_note":"360p",
+            "fps":24,
+            "tbr":431.488,
+            "format":"18 - 640x360 (360p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"22",
+            "player_url":null,
+            "ext":"mp4",
+            "width":1280,
+            "height":720,
+            "acodec":"mp4a.40.2",
+            "abr":192,
+            "vcodec":"avc1.64001F",
+            "asr":44100,
+            "filesize":null,
+            "format_note":"720p",
+            "fps":24,
+            "tbr":825.41,
+            "format":"22 - 1280x720 (720p)",
+            "protocol":"https"
+        }
+    ],
+    "is_live":null,
+    "start_time":null,
+    "end_time":null,
+    "series":null,
+    "season_number":null,
+    "episode_number":null,
+    "track":null,
+    "artist":null,
+    "album":null,
+    "release_date":null,
+    "release_year":null,
+    "extractor":"youtube",
+    "extractor_key":"Youtube",
+    "thumbnail":"https://example.com/maxresdefault.webp",
+    "requested_subtitles":null,
+    "format":"137 - 1920x1080 (1080p)+251 - audio only (tiny)",
+    "format_id":"137+251",
+    "width":1920,
+    "height":1080,
+    "resolution":null,
+    "fps":24,
+    "vcodec":"avc1.640028",
+    "vbr":null,
+    "stretched_ratio":null,
+    "acodec":"opus",
+    "abr":160,
+    "ext":"mp4"
+}

tubesync/sync/testdata/metadata_60fps.json

@@ -0,0 +1,342 @@
+{
+    "id":"60fps",
+    "upload_date":"20191111",
+    "license":null,
+    "creator":null,
+    "title":"60fps",
+    "alt_title":null,
+    "description":"60fps",
+    "categories":[],
+    "tags":[],
+    "subtitles":{},
+    "automatic_captions":{},
+    "duration":289.0,
+    "age_limit":0,
+    "annotations":null,
+    "chapters":null,
+    "formats":[
+        {
+            "format_id":"249",
+            "player_url":null,
+            "ext":"webm",
+            "format_note":"tiny",
+            "acodec":"opus",
+            "abr":50,
+            "asr":48000,
+            "filesize":1870819,
+            "fps":null,
+            "height":null,
+            "tbr":58.007,
+            "width":null,
+            "vcodec":"none",
+            "format":"249 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"250",
+            "player_url":null,
+            "ext":"webm",
+            "format_note":"tiny",
+            "acodec":"opus",
+            "abr":70,
+            "asr":48000,
+            "filesize":2449870,
+            "fps":null,
+            "height":null,
+            "tbr":75.669,
+            "width":null,
+            "vcodec":"none",
+            "format":"250 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"140",
+            "player_url":null,
+            "ext":"m4a",
+            "format_note":"tiny",
+            "acodec":"mp4a.40.2",
+            "abr":128,
+            "container":"m4a_dash",
+            "asr":44100,
+            "filesize":4663408,
+            "fps":null,
+            "height":null,
+            "tbr":130.556,
+            "width":null,
+            "vcodec":"none",
+            "format":"140 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"251",
+            "player_url":null,
+            "ext":"webm",
+            "format_note":"tiny",
+            "acodec":"opus",
+            "abr":160,
+            "asr":48000,
+            "filesize":4753701,
+            "fps":null,
+            "height":null,
+            "tbr":146.615,
+            "width":null,
+            "vcodec":"none",
+            "format":"251 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"278",
+            "player_url":null,
+            "ext":"webm",
+            "height":144,
+            "format_note":"144p",
+            "container":"webm",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":2977163,
+            "fps":30,
+            "tbr":88.365,
+            "width":256,
+            "acodec":"none",
+            "format":"278 - 256x144 (144p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"160",
+            "player_url":null,
+            "ext":"mp4",
+            "height":144,
+            "format_note":"144p",
+            "vcodec":"avc1.4d400c",
+            "asr":null,
+            "filesize":2535617,
+            "fps":30,
+            "tbr":111.075,
+            "width":256,
+            "acodec":"none",
+            "format":"160 - 256x144 (144p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"242",
+            "player_url":null,
+            "ext":"webm",
+            "height":240,
+            "format_note":"240p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":5737364,
+            "fps":30,
+            "tbr":192.191,
+            "width":426,
+            "acodec":"none",
+            "format":"242 - 426x240 (240p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"133",
+            "player_url":null,
+            "ext":"mp4",
+            "height":240,
+            "format_note":"240p",
+            "vcodec":"avc1.4d4015",
+            "asr":null,
+            "filesize":5021208,
+            "fps":30,
+            "tbr":245.185,
+            "width":426,
+            "acodec":"none",
+            "format":"133 - 426x240 (240p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"243",
+            "player_url":null,
+            "ext":"webm",
+            "height":360,
+            "format_note":"360p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":11239827,
+            "fps":30,
+            "tbr":407.893,
+            "width":640,
+            "acodec":"none",
+            "format":"243 - 640x360 (360p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"134",
+            "player_url":null,
+            "ext":"mp4",
+            "height":360,
+            "format_note":"360p",
+            "vcodec":"avc1.4d401e",
+            "asr":null,
+            "filesize":8874509,
+            "fps":30,
+            "tbr":497.961,
+            "width":640,
+            "acodec":"none",
+            "format":"134 - 640x360 (360p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"135",
+            "player_url":null,
+            "ext":"mp4",
+            "height":480,
+            "format_note":"480p",
+            "vcodec":"avc1.4d401f",
+            "asr":null,
+            "filesize":15442192,
+            "fps":30,
+            "tbr":745.342,
+            "width":854,
+            "acodec":"none",
+            "format":"135 - 854x480 (480p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"244",
+            "player_url":null,
+            "ext":"webm",
+            "height":480,
+            "format_note":"480p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":18572091,
+            "fps":30,
+            "tbr":754.357,
+            "width":854,
+            "acodec":"none",
+            "format":"244 - 854x480 (480p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"247",
+            "player_url":null,
+            "ext":"webm",
+            "height":720,
+            "format_note":"720p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":33280726,
+            "fps":30,
+            "tbr":1484.764,
+            "width":1280,
+            "acodec":"none",
+            "format":"247 - 1280x720 (720p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"136",
+            "player_url":null,
+            "ext":"mp4",
+            "height":720,
+            "format_note":"720p",
+            "vcodec":"avc1.4d401f",
+            "asr":null,
+            "filesize":37701417,
+            "fps":30,
+            "tbr":2104.312,
+            "width":1280,
+            "acodec":"none",
+            "format":"136 - 1280x720 (720p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"302",
+            "player_url":null,
+            "ext":"webm",
+            "height":720,
+            "format_note":"720p60",
+            "vcodec":"vp9",
+            "fps":60,
+            "asr":null,
+            "filesize":50207586,
+            "tbr":2400.348,
+            "width":1280,
+            "acodec":"none",
+            "format":"302 - 1280x720 (720p60)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"298",
+            "player_url":null,
+            "ext":"mp4",
+            "height":720,
+            "format_note":"720p60",
+            "vcodec":"avc1.4d4020",
+            "fps":60,
+            "asr":null,
+            "filesize":65788990,
+            "tbr":3470.649,
+            "width":1280,
+            "acodec":"none",
+            "format":"298 - 1280x720 (720p60)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"18",
+            "player_url":null,
+            "ext":"mp4",
+            "width":640,
+            "height":360,
+            "acodec":"mp4a.40.2",
+            "abr":96,
+            "vcodec":"avc1.42001E",
+            "asr":44100,
+            "filesize":20192032,
+            "format_note":"360p",
+            "fps":30,
+            "tbr":560.848,
+            "format":"18 - 640x360 (360p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"22",
+            "player_url":null,
+            "ext":"mp4",
+            "width":1280,
+            "height":720,
+            "acodec":"mp4a.40.2",
+            "abr":192,
+            "vcodec":"avc1.64001F",
+            "asr":44100,
+            "filesize":null,
+            "format_note":"720p",
+            "fps":30,
+            "tbr":1176.187,
+            "format":"22 - 1280x720 (720p)",
+            "protocol":"https"
+        }
+    ],
+    "is_live":null,
+    "start_time":null,
+    "end_time":null,
+    "series":null,
+    "season_number":null,
+    "episode_number":null,
+    "track":null,
+    "artist":null,
+    "album":null,
+    "release_date":null,
+    "release_year":null,
+    "extractor":"youtube",
+    "extractor_key":"Youtube",
+    "thumbnail":"https://example.com/maxresdefault.webp",
+    "requested_subtitles":null,
+    "format":"298 - 1280x720 (720p60)+251 - audio only (tiny)",
+    "format_id":"298+251",
+    "width":1280,
+    "height":720,
+    "resolution":null,
+    "fps":60,
+    "vcodec":"avc1.4d4020",
+    "vbr":null,
+    "stretched_ratio":null,
+    "acodec":"opus",
+    "abr":160,
+    "ext":"mp4"
+}

tubesync/sync/testdata/metadata_60fps_hdr.json

@@ -0,0 +1,551 @@
+{
+    "id":"60fpshdr",
+    "upload_date":"20200529",
+    "license":null,
+    "creator":null,
+    "title":"60fps with HDR",
+    "alt_title":null,
+    "description":"60fps with HDR",
+    "categories":[],
+    "tags":[],
+    "subtitles":{},
+    "automatic_captions":{},
+    "duration":1225.0,
+    "age_limit":0,
+    "annotations":null,
+    "chapters":null,
+    "formats":[
+        {
+            "format_id":"249",
+            "player_url":null,
+            "ext":"webm",
+            "format_note":"tiny",
+            "acodec":"opus",
+            "abr":50,
+            "asr":48000,
+            "filesize":8512058,
+            "fps":null,
+            "height":null,
+            "tbr":69.211,
+            "width":null,
+            "vcodec":"none",
+            "format":"249 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"250",
+            "player_url":null,
+            "ext":"webm",
+            "format_note":"tiny",
+            "acodec":"opus",
+            "abr":70,
+            "asr":48000,
+            "filesize":11102913,
+            "fps":null,
+            "height":null,
+            "tbr":91.251,
+            "width":null,
+            "vcodec":"none",
+            "format":"250 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"140",
+            "player_url":null,
+            "ext":"m4a",
+            "format_note":"tiny",
+            "acodec":"mp4a.40.2",
+            "abr":128,
+            "container":"m4a_dash",
+            "asr":44100,
+            "filesize":19821739,
+            "fps":null,
+            "height":null,
+            "tbr":131.659,
+            "width":null,
+            "vcodec":"none",
+            "format":"140 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"251",
+            "player_url":null,
+            "ext":"webm",
+            "format_note":"tiny",
+            "acodec":"opus",
+            "abr":160,
+            "asr":48000,
+            "filesize":21470525,
+            "fps":null,
+            "height":null,
+            "tbr":179.929,
+            "width":null,
+            "vcodec":"none",
+            "format":"251 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"160",
+            "player_url":null,
+            "ext":"mp4",
+            "height":144,
+            "format_note":"144p",
+            "vcodec":"avc1.4d400c",
+            "asr":null,
+            "filesize":7980487,
+            "fps":30,
+            "tbr":111.635,
+            "width":256,
+            "acodec":"none",
+            "format":"160 - 256x144 (144p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"278",
+            "player_url":null,
+            "ext":"webm",
+            "height":144,
+            "format_note":"144p",
+            "container":"webm",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":12325312,
+            "fps":30,
+            "tbr":172.269,
+            "width":256,
+            "acodec":"none",
+            "format":"278 - 256x144 (144p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"330",
+            "player_url":null,
+            "asr":null,
+            "filesize":30914955,
+            "format_note":"144p60 HDR",
+            "fps":60,
+            "height":144,
+            "tbr":248.489,
+            "width":256,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"330 - 256x144 (144p60 HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"133",
+            "player_url":null,
+            "ext":"mp4",
+            "height":240,
+            "format_note":"240p",
+            "vcodec":"avc1.4d4015",
+            "asr":null,
+            "filesize":17341221,
+            "fps":30,
+            "tbr":249.328,
+            "width":426,
+            "acodec":"none",
+            "format":"133 - 426x240 (240p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"242",
+            "player_url":null,
+            "ext":"webm",
+            "height":240,
+            "format_note":"240p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":21200666,
+            "fps":30,
+            "tbr":324.704,
+            "width":426,
+            "acodec":"none",
+            "format":"242 - 426x240 (240p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"331",
+            "player_url":null,
+            "asr":null,
+            "filesize":66206984,
+            "format_note":"240p60 HDR",
+            "fps":60,
+            "height":240,
+            "tbr":505.312,
+            "width":426,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"331 - 426x240 (240p60 HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"243",
+            "player_url":null,
+            "ext":"webm",
+            "height":360,
+            "format_note":"360p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":45145322,
+            "fps":30,
+            "tbr":625.446,
+            "width":640,
+            "acodec":"none",
+            "format":"243 - 640x360 (360p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"134",
+            "player_url":null,
+            "ext":"mp4",
+            "height":360,
+            "format_note":"360p",
+            "vcodec":"avc1.4d401e",
+            "asr":null,
+            "filesize":36679230,
+            "fps":30,
+            "tbr":634.64,
+            "width":640,
+            "acodec":"none",
+            "format":"134 - 640x360 (360p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"244",
+            "player_url":null,
+            "ext":"webm",
+            "height":480,
+            "format_note":"480p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":80021765,
+            "fps":30,
+            "tbr":914.834,
+            "width":854,
+            "acodec":"none",
+            "format":"244 - 854x480 (480p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"332",
+            "player_url":null,
+            "asr":null,
+            "filesize":144756945,
+            "format_note":"360p60 HDR",
+            "fps":60,
+            "height":360,
+            "tbr":1074.178,
+            "width":640,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"332 - 640x360 (360p60 HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"135",
+            "player_url":null,
+            "ext":"mp4",
+            "height":480,
+            "format_note":"480p",
+            "vcodec":"avc1.4d401f",
+            "asr":null,
+            "filesize":61634125,
+            "fps":30,
+            "tbr":1160.027,
+            "width":854,
+            "acodec":"none",
+            "format":"135 - 854x480 (480p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"247",
+            "player_url":null,
+            "ext":"webm",
+            "height":720,
+            "format_note":"720p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":156141801,
+            "fps":30,
+            "tbr":1773.891,
+            "width":1280,
+            "acodec":"none",
+            "format":"247 - 1280x720 (720p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"333",
+            "player_url":null,
+            "asr":null,
+            "filesize":281910480,
+            "format_note":"480p60 HDR",
+            "fps":60,
+            "height":480,
+            "tbr":2002.329,
+            "width":854,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"333 - 854x480 (480p60 HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"136",
+            "player_url":null,
+            "ext":"mp4",
+            "height":720,
+            "format_note":"720p",
+            "vcodec":"avc1.4d401f",
+            "asr":null,
+            "filesize":99428484,
+            "fps":30,
+            "tbr":2315.733,
+            "width":1280,
+            "acodec":"none",
+            "format":"136 - 1280x720 (720p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"302",
+            "player_url":null,
+            "ext":"webm",
+            "height":720,
+            "format_note":"720p60",
+            "vcodec":"vp9",
+            "fps":60,
+            "asr":null,
+            "filesize":234142087,
+            "tbr":2741.606,
+            "width":1280,
+            "acodec":"none",
+            "format":"302 - 1280x720 (720p60)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"298",
+            "player_url":null,
+            "ext":"mp4",
+            "height":720,
+            "format_note":"720p60",
+            "vcodec":"avc1.4d4020",
+            "fps":60,
+            "asr":null,
+            "filesize":293152085,
+            "tbr":4075.248,
+            "width":1280,
+            "acodec":"none",
+            "format":"298 - 1280x720 (720p60)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"334",
+            "player_url":null,
+            "asr":null,
+            "filesize":664805232,
+            "format_note":"720p60 HDR",
+            "fps":60,
+            "height":720,
+            "tbr":4550.044,
+            "width":1280,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"334 - 1280x720 (720p60 HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"303",
+            "player_url":null,
+            "ext":"webm",
+            "height":1080,
+            "format_note":"1080p60",
+            "vcodec":"vp9",
+            "fps":60,
+            "asr":null,
+            "filesize":427360829,
+            "tbr":4604.536,
+            "width":1920,
+            "acodec":"none",
+            "format":"303 - 1920x1080 (1080p60)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"299",
+            "player_url":null,
+            "ext":"mp4",
+            "height":1080,
+            "format_note":"1080p60",
+            "vcodec":"avc1.64002a",
+            "fps":60,
+            "asr":null,
+            "filesize":526570848,
+            "tbr":6840.241,
+            "width":1920,
+            "acodec":"none",
+            "format":"299 - 1920x1080 (1080p60)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"335",
+            "url":"https://r5---sn-ntqe6nee.googlevideo.com/videoplayback?expire=1607507475&ei=s0nQX4HYK_fL3LUPmeGewAc&ip=2404%3Ae80%3A4b4%3A48%3Afdc4%3A9086%3Af1e0%3A9dba&id=o-AK9yHRTiE6MhIQOZwbN_FFFVrcjNjuzNOFHJxuby6t_7&itag=335&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C272%2C278%2C298%2C299%2C302%2C303%2C308%2C315%2C330%2C331%2C332%2C333%2C334%2C335%2C336%2C337&source=youtube&requiressl=yes&mh=lV&mm=31%2C26&mn=sn-ntqe6nee%2Csn-npoe7ned&ms=au%2Conr&mv=m&mvi=5&pl=32&initcwndbps=1763750&vprv=1&mime=video%2Fwebm&ns=LrF8qBcGnL-4AEw1lplberwF&gir=yes&clen=1026563583&dur=1224.666&lmt=1597607257610309&mt=1607485470&fvip=5&keepalive=yes&beids=9466587&c=WEB&txp=5431232&n=aX7vTt4fLxHgqpUTJ&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&sig=AOq0QJ8wRQIhAK5Mx4xsl9cuqeoT6GPYvR6yjqmPixW_aJXlvBTvTG2IAiAaLuFQWsBJY6qD9va96wtyFMz4Q0C7RHW6jjpn1rUlsA%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&lsig=AG3C_xAwRAIgRTWYsm_a49j3OlWI05B9d-3rDTjdk9v-v31Ef0gQqZkCIHXgqek9uOUZTwRAvMWXbGxI8OcIJfZlLDvXriw2qhE2&ratebypass=yes",
+            "player_url":null,
+            "asr":null,
+            "filesize":1026563583,
+            "format_note":"1080p60 HDR",
+            "fps":60,
+            "height":1080,
+            "tbr":7092.87,
+            "width":1920,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"335 - 1920x1080 (1080p60 HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"308",
+            "player_url":null,
+            "ext":"webm",
+            "height":1440,
+            "format_note":"1440p60",
+            "vcodec":"vp9",
+            "fps":60,
+            "asr":null,
+            "filesize":1337282162,
+            "tbr":13646.585,
+            "width":2560,
+            "acodec":"none",
+            "format":"308 - 2560x1440 (1440p60)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"336",
+            "player_url":null,
+            "asr":null,
+            "filesize":2459997480,
+            "format_note":"1440p60 HDR",
+            "fps":60,
+            "height":1440,
+            "tbr":16696.968,
+            "width":2560,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"336 - 2560x1440 (1440p60 HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"272",
+            "player_url":null,
+            "ext":"webm",
+            "height":4320,
+            "format_note":"4320p60",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":3256069660,
+            "fps":60,
+            "tbr":27347.091,
+            "width":7680,
+            "acodec":"none",
+            "format":"272 - 7680x4320 (4320p60)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"315",
+            "player_url":null,
+            "ext":"webm",
+            "height":2160,
+            "format_note":"2160p60",
+            "vcodec":"vp9",
+            "fps":60,
+            "asr":null,
+            "filesize":3608369795,
+            "tbr":27405.936,
+            "width":3840,
+            "acodec":"none",
+            "format":"315 - 3840x2160 (2160p60)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"337",
+            "player_url":null,
+            "asr":null,
+            "filesize":4378262991,
+            "format_note":"2160p60 HDR",
+            "fps":60,
+            "height":2160,
+            "tbr":30090.336,
+            "width":3840,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"337 - 3840x2160 (2160p60 HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"18",
+            "player_url":null,
+            "ext":"mp4",
+            "width":640,
+            "height":360,
+            "acodec":"mp4a.40.2",
+            "abr":96,
+            "vcodec":"avc1.42001E",
+            "asr":44100,
+            "filesize":82807429,
+            "format_note":"360p",
+            "fps":30,
+            "tbr":540.93,
+            "format":"18 - 640x360 (360p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"22",
+            "player_url":null,
+            "ext":"mp4",
+            "width":1280,
+            "height":720,
+            "acodec":"mp4a.40.2",
+            "abr":192,
+            "vcodec":"avc1.64001F",
+            "asr":44100,
+            "filesize":null,
+            "format_note":"720p",
+            "fps":30,
+            "tbr":778.432,
+            "format":"22 - 1280x720 (720p)",
+            "protocol":"https"
+        }
+    ],
+    "is_live":null,
+    "start_time":null,
+    "end_time":null,
+    "series":null,
+    "season_number":null,
+    "episode_number":null,
+    "track":null,
+    "artist":null,
+    "album":null,
+    "release_date":null,
+    "release_year":null,
+    "extractor":"youtube",
+    "extractor_key":"Youtube",
+    "thumbnail":"https://example.com/maxresdefault.webp",
+    "requested_subtitles":null,
+    "format":"337 - 3840x2160 (2160p60 HDR)+251 - audio only (tiny)",
+    "format_id":"337+251",
+    "width":3840,
+    "height":2160,
+    "resolution":null,
+    "fps":60,
+    "vcodec":"vp9.2",
+    "vbr":null,
+    "stretched_ratio":null,
+    "acodec":"opus",
+    "abr":160,
+    "ext":"webm"
+}

tubesync/sync/testdata/metadata_hdr.json

@@ -0,0 +1,502 @@
+{
+    "id":"hdr",
+    "upload_date":"20161109",
+    "license":null,
+    "creator":null,
+    "title":"hdr",
+    "alt_title":null,
+    "description":"hdr",
+    "categories":[],
+    "tags":[],
+    "subtitles":{},
+    "automatic_captions":{},
+    "duration":118.0,
+    "age_limit":0,
+    "annotations":null,
+    "chapters":null,
+    "formats":[
+        {
+            "format_id":"249",
+            "player_url":null,
+            "ext":"webm",
+            "format_note":"tiny",
+            "acodec":"opus",
+            "abr":50,
+            "asr":48000,
+            "filesize":846425,
+            "fps":null,
+            "height":null,
+            "tbr":60.208,
+            "width":null,
+            "vcodec":"none",
+            "format":"249 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"250",
+            "player_url":null,
+            "ext":"webm",
+            "format_note":"tiny",
+            "acodec":"opus",
+            "abr":70,
+            "asr":48000,
+            "filesize":1126262,
+            "fps":null,
+            "height":null,
+            "tbr":79.173,
+            "width":null,
+            "vcodec":"none",
+            "format":"250 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"140",
+            "player_url":null,
+            "ext":"m4a",
+            "format_note":"tiny",
+            "acodec":"mp4a.40.2",
+            "abr":128,
+            "container":"m4a_dash",
+            "asr":44100,
+            "filesize":1902693,
+            "fps":null,
+            "height":null,
+            "tbr":130.432,
+            "width":null,
+            "vcodec":"none",
+            "format":"140 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"251",
+            "player_url":null,
+            "ext":"webm",
+            "format_note":"tiny",
+            "acodec":"opus",
+            "abr":160,
+            "asr":48000,
+            "filesize":2141091,
+            "fps":null,
+            "height":null,
+            "tbr":150.305,
+            "width":null,
+            "vcodec":"none",
+            "format":"251 - audio only (tiny)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"160",
+            "player_url":null,
+            "ext":"mp4",
+            "height":144,
+            "format_note":"144p",
+            "vcodec":"avc1.4d400c",
+            "asr":null,
+            "filesize":458348,
+            "fps":24,
+            "tbr":43.288,
+            "width":256,
+            "acodec":"none",
+            "format":"160 - 256x144 (144p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"133",
+            "player_url":null,
+            "ext":"mp4",
+            "height":240,
+            "format_note":"240p",
+            "vcodec":"avc1.4d4015",
+            "asr":null,
+            "filesize":974592,
+            "fps":24,
+            "tbr":92.086,
+            "width":426,
+            "acodec":"none",
+            "format":"133 - 426x240 (240p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"278",
+            "player_url":null,
+            "ext":"webm",
+            "height":144,
+            "format_note":"144p",
+            "container":"webm",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":1273663,
+            "fps":24,
+            "tbr":95.995,
+            "width":256,
+            "acodec":"none",
+            "format":"278 - 256x144 (144p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"330",
+            "player_url":null,
+            "asr":null,
+            "filesize":1620022,
+            "format_note":"144p HDR",
+            "fps":24,
+            "height":144,
+            "tbr":130.515,
+            "width":256,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"330 - 256x144 (144p HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"242",
+            "player_url":null,
+            "ext":"webm",
+            "height":240,
+            "format_note":"240p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":1565384,
+            "fps":24,
+            "tbr":143.325,
+            "width":426,
+            "acodec":"none",
+            "format":"242 - 426x240 (240p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"331",
+            "player_url":null,
+            "asr":null,
+            "filesize":2910740,
+            "format_note":"240p HDR",
+            "fps":24,
+            "height":240,
+            "tbr":221.449,
+            "width":426,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"331 - 426x240 (240p HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"134",
+            "player_url":null,
+            "ext":"mp4",
+            "height":360,
+            "format_note":"360p",
+            "vcodec":"avc1.4d401e",
+            "asr":null,
+            "filesize":2845501,
+            "fps":24,
+            "tbr":265.501,
+            "width":640,
+            "acodec":"none",
+            "format":"134 - 640x360 (360p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"243",
+            "player_url":null,
+            "ext":"webm",
+            "height":360,
+            "format_note":"360p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":3361873,
+            "fps":24,
+            "tbr":303.494,
+            "width":640,
+            "acodec":"none",
+            "format":"243 - 640x360 (360p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"332",
+            "player_url":null,
+            "asr":null,
+            "filesize":6418173,
+            "format_note":"360p HDR",
+            "fps":24,
+            "height":360,
+            "tbr":478.757,
+            "width":640,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"332 - 640x360 (360p HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"244",
+            "player_url":null,
+            "ext":"webm",
+            "height":480,
+            "format_note":"480p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":6282780,
+            "fps":24,
+            "tbr":545.429,
+            "width":854,
+            "acodec":"none",
+            "format":"244 - 854x480 (480p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"135",
+            "player_url":null,
+            "ext":"mp4",
+            "height":480,
+            "format_note":"480p",
+            "vcodec":"avc1.4d401e",
+            "asr":null,
+            "filesize":6404636,
+            "fps":24,
+            "tbr":619.85,
+            "width":854,
+            "acodec":"none",
+            "format":"135 - 854x480 (480p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"333",
+            "player_url":null,
+            "asr":null,
+            "filesize":12520807,
+            "format_note":"480p HDR",
+            "fps":24,
+            "height":480,
+            "tbr":902.636,
+            "width":854,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"333 - 854x480 (480p HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"136",
+            "player_url":null,
+            "ext":"mp4",
+            "height":720,
+            "format_note":"720p",
+            "vcodec":"avc1.4d401f",
+            "asr":null,
+            "filesize":13646566,
+            "fps":24,
+            "tbr":1318.273,
+            "width":1280,
+            "acodec":"none",
+            "format":"136 - 1280x720 (720p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"247",
+            "player_url":null,
+            "ext":"webm",
+            "height":720,
+            "format_note":"720p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":15047273,
+            "fps":24,
+            "tbr":1335.23,
+            "width":1280,
+            "acodec":"none",
+            "format":"247 - 1280x720 (720p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"334",
+            "player_url":null,
+            "asr":null,
+            "filesize":28956363,
+            "format_note":"720p HDR",
+            "fps":24,
+            "height":720,
+            "tbr":2050.091,
+            "width":1280,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"334 - 1280x720 (720p HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"248",
+            "player_url":null,
+            "ext":"webm",
+            "height":1080,
+            "format_note":"1080p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":29892111,
+            "fps":24,
+            "tbr":2521.353,
+            "width":1920,
+            "acodec":"none",
+            "format":"248 - 1920x1080 (1080p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"137",
+            "player_url":null,
+            "ext":"mp4",
+            "height":1080,
+            "format_note":"1080p",
+            "vcodec":"avc1.640028",
+            "asr":null,
+            "filesize":26734027,
+            "fps":24,
+            "tbr":2548.593,
+            "width":1920,
+            "acodec":"none",
+            "format":"137 - 1920x1080 (1080p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"335",
+            "player_url":null,
+            "asr":null,
+            "filesize":51033319,
+            "format_note":"1080p HDR",
+            "fps":24,
+            "height":1080,
+            "tbr":3550.019,
+            "width":1920,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"335 - 1920x1080 (1080p HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"271",
+            "player_url":null,
+            "ext":"webm",
+            "height":1440,
+            "format_note":"1440p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":80944570,
+            "fps":24,
+            "tbr":6965.068,
+            "width":2560,
+            "acodec":"none",
+            "format":"271 - 2560x1440 (1440p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"336",
+            "player_url":null,
+            "asr":null,
+            "filesize":143288990,
+            "format_note":"1440p HDR",
+            "fps":24,
+            "height":1440,
+            "tbr":10066.781,
+            "width":2560,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"336 - 2560x1440 (1440p HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"313",
+            "player_url":null,
+            "ext":"webm",
+            "height":2160,
+            "format_note":"2160p",
+            "vcodec":"vp9",
+            "asr":null,
+            "filesize":238816879,
+            "fps":24,
+            "tbr":17450.488,
+            "width":3840,
+            "acodec":"none",
+            "format":"313 - 3840x2160 (2160p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"337",
+            "player_url":null,
+            "asr":null,
+            "filesize":329492052,
+            "format_note":"2160p HDR",
+            "fps":24,
+            "height":2160,
+            "tbr":23198.764,
+            "width":3840,
+            "ext":"webm",
+            "vcodec":"vp9.2",
+            "acodec":"none",
+            "format":"337 - 3840x2160 (2160p HDR)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"18",
+            "player_url":null,
+            "ext":"mp4",
+            "width":640,
+            "height":360,
+            "acodec":"mp4a.40.2",
+            "abr":96,
+            "vcodec":"avc1.42001E",
+            "asr":44100,
+            "filesize":6587979,
+            "format_note":"360p",
+            "fps":24,
+            "tbr":448.703,
+            "format":"18 - 640x360 (360p)",
+            "protocol":"https"
+        },
+        {
+            "format_id":"22",
+            "player_url":null,
+            "ext":"mp4",
+            "width":1280,
+            "height":720,
+            "acodec":"mp4a.40.2",
+            "abr":192,
+            "vcodec":"avc1.64001F",
+            "asr":44100,
+            "filesize":null,
+            "format_note":"720p",
+            "fps":24,
+            "tbr":1058.627,
+            "format":"22 - 1280x720 (720p)",
+            "protocol":"https"
+        }
+    ],
+    "is_live":null,
+    "start_time":null,
+    "end_time":null,
+    "series":null,
+    "season_number":null,
+    "episode_number":null,
+    "track":null,
+    "artist":null,
+    "album":null,
+    "release_date":null,
+    "release_year":null,
+    "extractor":"youtube",
+    "extractor_key":"Youtube",
+    "thumbnail":"https://example.com/vi_webp/WW2DKBGCvEs/maxresdefault.webp",
+    "requested_subtitles":null,
+    "format":"337 - 3840x2160 (2160p HDR)+251 - audio only (tiny)",
+    "format_id":"337+251",
+    "width":3840,
+    "height":2160,
+    "resolution":null,
+    "fps":24,
+    "vcodec":"vp9.2",
+    "vbr":null,
+    "stretched_ratio":null,
+    "acodec":"opus",
+    "abr":160,
+    "ext":"webm"
+}

tubesync/sync/urls.py

@@ -0,0 +1,68 @@
+from django.urls import path
+from .views import (DashboardView, SourcesView, ValidateSourceView, AddSourceView,
+                    SourceView, UpdateSourceView, DeleteSourceView, MediaView,
+                    MediaThumbView, MediaItemView, TasksView, CompletedTasksView)
+
+
+app_name = 'sync'
+
+
+urlpatterns = [
+
+    # Dashboard URLs
+
+    path('',
+         DashboardView.as_view(),
+         name='dashboard'),
+    
+    # Source URLs
+
+    path('sources',
+         SourcesView.as_view(),
+         name='sources'),
+
+    path('source-validate/<slug:source_type>',
+         ValidateSourceView.as_view(),
+         name='validate-source'),
+
+    path('source-add',
+         AddSourceView.as_view(),
+         name='add-source'),
+
+    path('source/<uuid:pk>',
+         SourceView.as_view(),
+         name='source'),
+
+    path('source-update/<uuid:pk>',
+         UpdateSourceView.as_view(),
+         name='update-source'),
+
+    path('source-delete/<uuid:pk>',
+         DeleteSourceView.as_view(),
+         name='delete-source'),
+
+    # Media URLs
+
+    path('media',
+         MediaView.as_view(),
+         name='media'),
+
+    path('media-thumb/<uuid:pk>',
+         MediaThumbView.as_view(),
+         name='media-thumb'),
+
+    path('media/<uuid:pk>',
+         MediaItemView.as_view(),
+         name='media-item'),
+
+    # Task URLs
+
+    path('tasks',
+         TasksView.as_view(),
+         name='tasks'),
+
+    path('tasks-completed',
+         CompletedTasksView.as_view(),
+         name='tasks-completed'),
+
+]

tubesync/sync/utils.py

@@ -0,0 +1,173 @@
+import os
+import re
+import math
+from pathlib import Path
+import requests
+from PIL import Image
+from django.conf import settings
+from urllib.parse import urlsplit, parse_qs
+from django.forms import ValidationError
+
+
+def validate_url(url, validator):
+    '''
+        Validate a URL against a dict of validation requirements. Returns an extracted
+        part of the URL if the URL is valid, if invalid raises a ValidationError.
+    '''
+    valid_scheme, valid_netloc, valid_path, invalid_paths, valid_query, \
+        extract_parts = (
+            validator['scheme'], validator['domain'], validator['path_regex'],
+            validator['path_must_not_match'], validator['qs_args'],
+            validator['extract_key']
+    )
+    url_parts = urlsplit(str(url).strip())
+    url_scheme = str(url_parts.scheme).strip().lower()
+    if url_scheme != valid_scheme:
+        raise ValidationError(f'invalid scheme "{url_scheme}" must be "{valid_scheme}"')
+    url_netloc = str(url_parts.netloc).strip().lower()
+    if url_netloc != valid_netloc:
+        raise ValidationError(f'invalid domain "{url_netloc}" must be "{valid_netloc}"')
+    url_path = str(url_parts.path).strip()
+    matches = re.findall(valid_path, url_path)
+    if not matches:
+        raise ValidationError(f'invalid path "{url_path}" must match "{valid_path}"')
+    for invalid_path in invalid_paths:
+        if url_path.lower() == invalid_path.lower():
+            raise ValidationError(f'path "{url_path}" is not valid')
+    url_query = str(url_parts.query).strip()
+    url_query_parts = parse_qs(url_query)
+    for required_query in valid_query:
+        if required_query not in url_query_parts:
+            raise ValidationError(f'invalid query string "{url_query}" must '
+                                  f'contain the parameter "{required_query}"')
+    extract_from, extract_param = extract_parts
+    extract_value = ''
+    if extract_from == 'path_regex':
+        try:
+            submatches = matches[0]
+            try:
+                extract_value = submatches[extract_param]
+            except IndexError:
+                pass
+        except IndexError:
+            pass
+    elif extract_from == 'qs_args':
+        extract_value = url_query_parts[extract_param][0]
+    return extract_value
+
+
+def get_remote_image(url):
+    headers = {
+        'user-agent': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                       '(KHTML, like Gecko) Chrome/69.0.3497.64 Safari/537.36')
+    }
+    r = requests.get(url, headers=headers, stream=True, timeout=60)
+    r.raw.decode_content = True
+    return Image.open(r.raw)
+
+
+def resize_image_to_height(image, width, height):
+    '''
+        Resizes an image to 'height' pixels keeping the ratio. If the resulting width
+        is larger than 'width' then crop it. If the resulting width is smaller than
+        'width' then stretch it.
+    '''
+    image = image.convert('RGB')
+    ratio = image.width / image.height
+    scaled_width = math.ceil(height * ratio)
+    if scaled_width < width:
+        # Width too small, stretch it
+        scaled_width = width
+    image = image.resize((scaled_width, height), Image.ANTIALIAS)
+    if scaled_width > width:
+        # Width too large, crop it
+        delta = scaled_width - width
+        left, upper = round(delta / 2), 0
+        right, lower = (left + width), height
+        image = image.crop((left, upper, right, lower))
+    return image
+
+
+def file_is_editable(filepath):
+    '''
+        Checks that a file exists and the file is in an allowed predefined tuple of
+        directories we want to allow writing or deleting in.
+    '''
+    allowed_paths = (
+        # Media item thumbnails
+        os.path.commonpath([os.path.abspath(str(settings.MEDIA_ROOT))]),
+        # Downloaded video files
+        os.path.commonpath([os.path.abspath(str(settings.SYNC_VIDEO_ROOT))]),
+        # Downloaded audio files
+        os.path.commonpath([os.path.abspath(str(settings.SYNC_AUDIO_ROOT))]),
+    )
+    filepath = os.path.abspath(str(filepath))
+    if not os.path.isfile(filepath):
+        return False
+    for allowed_path in allowed_paths:
+        if allowed_path == os.path.commonpath([allowed_path, filepath]):
+            return True
+    return False
+
+
+def delete_file(filepath):
+    if file_is_editable(filepath):
+        return os.remove(filepath)
+    return False
+
+
+def seconds_to_timestr(seconds):
+   seconds = seconds % (24 * 3600)
+   hour = seconds // 3600
+   seconds %= 3600
+   minutes = seconds // 60
+   seconds %= 60
+   return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds)
+
+
+def parse_media_format(format_dict):
+    '''
+        This parser primarily adapts the format dict returned by youtube-dl into a
+        standard form used by the matchers in matching.py. If youtube-dl changes
+        any internals, update it here.
+    '''
+    vcodec_full = format_dict.get('vcodec', '')
+    vcodec_parts = vcodec_full.split('.')
+    if len(vcodec_parts) > 0:
+        vcodec = vcodec_parts[0].strip().upper()
+    else:
+        vcodec = None
+    if vcodec == 'NONE':
+        vcodec = None
+    acodec_full = format_dict.get('acodec', '')
+    acodec_parts = acodec_full.split('.')
+    if len(acodec_parts) > 0:
+        acodec = acodec_parts[0].strip().upper()
+    else:
+        acodec = None
+    if acodec == 'NONE':
+        acodec = None
+    try:
+        fps = int(format_dict.get('fps', 0))
+    except (ValueError, TypeError):
+        fps = 0
+    format_full = format_dict.get('format_note', '').strip().upper()
+    format_str = format_full[:-2] if format_full.endswith('60') else format_full
+    format_str = format_str.strip()
+    format_str = format_str[:-3] if format_str.endswith('HDR') else format_str
+    format_str = format_str.strip()
+    format_str = format_str[:-2] if format_str.endswith('60') else format_str
+    format_str = format_str.strip()
+    return {
+        'id': format_dict.get('format_id', ''),
+        'format': format_str,
+        'format_verbose': format_dict.get('format', ''),
+        'height': format_dict.get('height', 0),
+        'vcodec': vcodec,
+        'fps': format_dict.get('fps', 0),
+        'vbr': format_dict.get('tbr', 0),
+        'acodec': acodec,
+        'abr': format_dict.get('abr', 0),
+        'is_60fps': fps > 50,
+        'is_hdr': 'HDR' in format_dict.get('format', '').upper(),
+    }

tubesync/sync/views.py

@@ -0,0 +1,480 @@
+from base64 import b64decode
+from django.conf import settings
+from django.http import Http404
+from django.views.generic import TemplateView, ListView, DetailView
+from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateView,
+                                       DeleteView)
+from django.http import HttpResponse
+from django.urls import reverse_lazy
+from django.db.models import Count
+from django.forms import ValidationError
+from django.utils.text import slugify
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+from common.utils import append_uri_params
+from background_task.models import Task, CompletedTask
+from .models import Source, Media
+from .forms import ValidateSourceForm, ConfirmDeleteSourceForm
+from .utils import validate_url
+from .tasks import map_task_to_instance, get_error_message, get_source_completed_tasks
+from . import signals
+from . import youtube
+
+
+class DashboardView(TemplateView):
+    '''
+        The dashboard shows non-interactive totals and summaries.
+    '''
+
+    template_name = 'sync/dashboard.html'
+
+    def dispatch(self, request, *args, **kwargs):
+        return super().dispatch(request, *args, **kwargs)
+
+
+class SourcesView(ListView):
+    '''
+        A bare list of the sources which have been created with their states.
+    '''
+
+    template_name = 'sync/sources.html'
+    context_object_name = 'sources'
+    paginate_by = settings.SOURCES_PER_PAGE
+    messages = {
+        'source-deleted': _('Your selected source has been deleted.'),
+    }
+
+    def __init__(self, *args, **kwargs):
+        self.message = None
+        super().__init__(*args, **kwargs)
+
+    def dispatch(self, request, *args, **kwargs):
+        message_key = request.GET.get('message', '')
+        self.message = self.messages.get(message_key, '')
+        return super().dispatch(request, *args, **kwargs)
+
+    def get_queryset(self):
+        all_sources = Source.objects.all().order_by('name')
+        return all_sources.annotate(media_count=Count('media_source'))
+
+    def get_context_data(self, *args, **kwargs):
+        data = super().get_context_data(*args, **kwargs)
+        data['message'] = self.message
+        return data
+
+
+class ValidateSourceView(FormView):
+    '''
+        Validate a URL and prepopulate a create source view form with confirmed
+        accurate data. The aim here is to streamline onboarding of new sources
+        which otherwise may not be entirely obvious to add, such as the "key"
+        being just a playlist ID or some other reasonably opaque internals.
+    '''
+
+    template_name = 'sync/source-validate.html'
+    form_class = ValidateSourceForm
+    errors = {
+        'invalid_url': _('Invalid URL, the URL must for a "{item}" must be in '
+                         'the format of "{example}". The error was: {error}.'),
+    }
+    source_types = {
+        'youtube-channel': Source.SOURCE_TYPE_YOUTUBE_CHANNEL,
+        'youtube-playlist': Source.SOURCE_TYPE_YOUTUBE_PLAYLIST,
+    }
+    help_item = {
+        Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _('YouTube channel'),
+        Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: _('YouTube playlist'),
+    }
+    help_texts = {
+        Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _(
+            'Enter a YouTube channel URL into the box below. A channel URL will be in '
+            'the format of <strong>https://www.youtube.com/CHANNELNAME</strong> '
+            'where <strong>CHANNELNAME</strong> is the name of the channel you want '
+            'to add.'
+        ),
+        Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: _(
+            'Enter a YouTube playlist URL into the box below. A playlist URL will be '
+            'in the format of <strong>https://www.youtube.com/playlist?list='
+            'BiGLoNgUnIqUeId</strong> where <strong>BiGLoNgUnIqUeId</strong> is the '
+            'unique ID of the playlist you want to add.'
+        ),
+    }
+    help_examples = {
+        Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/google',
+        Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('https://www.youtube.com/playlist?list='
+                                              'PL590L5WQmH8dpP0RyH5pCfIaDEdt9nk7r')
+    }
+    validation_urls = {
+        Source.SOURCE_TYPE_YOUTUBE_CHANNEL: {
+            'scheme': 'https',
+            'domain': 'www.youtube.com',
+            'path_regex': '^\/(c\/)?([^\/]+)$',
+            'path_must_not_match': ('/playlist', '/c/playlist'),
+            'qs_args': [],
+            'extract_key': ('path_regex', 1),
+            'example': 'https://www.youtube.com/SOMECHANNEL'
+        },
+        Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: {
+            'scheme': 'https',
+            'domain': 'www.youtube.com',
+            'path_regex': '^\/(playlist|watch)$',
+            'path_must_not_match': (),
+            'qs_args': ('list',),
+            'extract_key': ('qs_args', 'list'),
+            'example': 'https://www.youtube.com/playlist?list=PLAYLISTID'
+        },
+    }
+    prepopulate_fields = {
+        Source.SOURCE_TYPE_YOUTUBE_CHANNEL: ('source_type', 'key', 'name', 'directory'),
+        Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('source_type', 'key'),
+    }
+
+    def __init__(self, *args, **kwargs):
+        self.source_type_str = ''
+        self.source_type = None
+        self.key = ''
+        super().__init__(*args, **kwargs)
+
+    def dispatch(self, request, *args, **kwargs):
+        self.source_type_str = kwargs.get('source_type', '').strip().lower()
+        self.source_type = self.source_types.get(self.source_type_str, None)
+        if not self.source_type:
+            raise Http404
+        return super().dispatch(request, *args, **kwargs)
+
+    def get_initial(self):
+        initial = super().get_initial()
+        initial['source_type'] = self.source_type
+        return initial
+
+    def get_context_data(self, *args, **kwargs):
+        data = super().get_context_data(*args, **kwargs)
+        data['source_type'] = self.source_type_str
+        data['help_item'] = self.help_item.get(self.source_type)
+        data['help_text'] = self.help_texts.get(self.source_type)
+        data['help_example'] = self.help_examples.get(self.source_type)
+        return data
+
+    def form_valid(self, form):
+        # Perform extra validation on the URL, we need to extract the channel name or
+        # playlist ID and check they are valid
+        source_type = form.cleaned_data['source_type']
+        if source_type not in self.source_types.values():
+            form.add_error(
+                'source_type',
+                ValidationError(self.errors['invalid_source'])
+            )
+        source_url = form.cleaned_data['source_url']
+        validation_url = self.validation_urls.get(source_type)
+        try:
+            self.key = validate_url(source_url, validation_url)
+        except ValidationError as e:
+            error = self.errors.get('invalid_url')
+            item = self.help_item.get(self.source_type)
+            form.add_error(
+                'source_url',
+                ValidationError(error.format(
+                    item=item,
+                    example=validation_url['example'],
+                    error=e.message)
+                )
+            )
+        if form.errors:
+            return super().form_invalid(form)
+        return super().form_valid(form)
+
+    def get_success_url(self):
+        url = reverse_lazy('sync:add-source')
+        fields_to_populate = self.prepopulate_fields.get(self.source_type)
+        fields = {}
+        for field in fields_to_populate:
+            if field == 'source_type':
+                fields[field] = self.source_type
+            elif field in ('key', 'name', 'directory'):
+                fields[field] = self.key
+        return append_uri_params(url, fields)
+
+
+class AddSourceView(CreateView):
+    '''
+        Adds a new source, optionally takes some initial data querystring values to
+        prepopulate some of the more unclear values.
+    '''
+
+    template_name = 'sync/source-add.html'
+    model = Source
+    fields = ('source_type', 'key', 'name', 'directory', 'index_schedule',
+              'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
+              'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback')
+
+    def __init__(self, *args, **kwargs):
+        self.prepopulated_data = {}
+        super().__init__(*args, **kwargs)
+
+    def dispatch(self, request, *args, **kwargs):
+        source_type = request.GET.get('source_type', '')
+        if source_type and source_type in Source.SOURCE_TYPES:
+            self.prepopulated_data['source_type'] = source_type
+        key = request.GET.get('key', '')
+        if key:
+            self.prepopulated_data['key'] = key.strip()
+        name = request.GET.get('name', '')
+        if name:
+            self.prepopulated_data['name'] = slugify(name)
+        directory = request.GET.get('directory', '')
+        if directory:
+            self.prepopulated_data['directory'] = slugify(directory)
+        return super().dispatch(request, *args, **kwargs)
+
+    def get_initial(self):
+        initial = super().get_initial()
+        for k, v in self.prepopulated_data.items():
+            initial[k] = v
+        return initial
+
+    def get_success_url(self):
+        url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
+        return append_uri_params(url, {'message': 'source-created'})
+
+
+class SourceView(DetailView):
+
+    template_name = 'sync/source.html'
+    model = Source
+    messages = {
+        'source-created': _('Your new source has been created'),
+        'source-updated': _('Your source has been updated.'),
+    }
+
+    def __init__(self, *args, **kwargs):
+        self.message = None
+        super().__init__(*args, **kwargs)
+
+    def dispatch(self, request, *args, **kwargs):
+        message_key = request.GET.get('message', '')
+        self.message = self.messages.get(message_key, '')
+        return super().dispatch(request, *args, **kwargs)
+
+    def get_context_data(self, *args, **kwargs):
+        data = super().get_context_data(*args, **kwargs)
+        data['message'] = self.message
+        data['errors'] = []
+        for error in get_source_completed_tasks(self.object.pk, only_errors=True):
+            error_message = get_error_message(error)
+            setattr(error, 'error_message', error_message)
+            data['errors'].append(error)
+        data['media'] = Media.objects.filter(source=self.object).order_by('-published')
+        return data
+
+
+class UpdateSourceView(UpdateView):
+
+    template_name = 'sync/source-update.html'
+    model = Source
+    fields = ('source_type', 'key', 'name', 'directory', 'index_schedule',
+              'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
+              'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback')
+
+    def get_success_url(self):
+        url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
+        return append_uri_params(url, {'message': 'source-updated'})
+
+
+class DeleteSourceView(DeleteView, FormMixin):
+    '''
+        Confirm the deletion of a source with an option to delete all the media
+        associated with the source from disk when the source is deleted.
+    '''
+
+    template_name = 'sync/source-delete.html'
+    model = Source
+    form_class = ConfirmDeleteSourceForm
+    context_object_name = 'source'
+
+    def post(self, request, *args, **kwargs):
+        delete_media_val = request.POST.get('delete_media', False)
+        delete_media = True if delete_media_val is not False else False
+        if delete_media:
+            # TODO: delete media files from disk linked to this source
+            pass
+        return super().post(request, *args, **kwargs)
+
+    def get_success_url(self):
+        url = reverse_lazy('sync:sources')
+        return append_uri_params(url, {'message': 'source-deleted'})
+
+
+class MediaView(ListView):
+    '''
+        A bare list of media added with their states.
+    '''
+
+    template_name = 'sync/media.html'
+    context_object_name = 'media'
+    paginate_by = settings.MEDIA_PER_PAGE
+    messages = {
+        'filter': _('Viewing media filtered for source: <strong>{name}</strong>'),
+    }
+
+    def __init__(self, *args, **kwargs):
+        self.filter_source = None
+        super().__init__(*args, **kwargs)
+
+    def dispatch(self, request, *args, **kwargs):
+        filter_by = request.GET.get('filter', '')
+        if filter_by:
+            try:
+                self.filter_source = Source.objects.get(pk=filter_by)
+            except Source.DoesNotExist:
+                self.filter_source = None
+        return super().dispatch(request, *args, **kwargs)
+
+    def get_queryset(self):
+        if self.filter_source:
+            q = Media.objects.filter(source=self.filter_source)
+        else:
+            q = Media.objects.all()
+        return q.order_by('-published', '-created')
+
+    def get_context_data(self, *args, **kwargs):
+        data = super().get_context_data(*args, **kwargs)
+        data['message'] = ''
+        data['source'] = None
+        if self.filter_source:
+            message = str(self.messages.get('filter', ''))
+            data['message'] = message.format(name=self.filter_source.name)
+            data['source'] = self.filter_source
+        return data
+
+
+class MediaThumbView(DetailView):
+    '''
+        Shows a media thumbnail. Whitenoise doesn't support post-start media image
+        serving and the images here are pretty small so just serve them manually. This
+        isn't fast, but it's not likely to be a serious bottleneck.
+    '''
+
+    model = Media
+
+    def get(self, request, *args, **kwargs):
+        media = self.get_object()
+        if media.thumb:
+            thumb = open(media.thumb.path, 'rb').read()
+            content_type = 'image/jpeg' 
+        else:
+            # No thumbnail on disk, return a blank 1x1 gif
+            thumb = b64decode('R0lGODlhAQABAIABAP///wAAACH5BAEKAAEALAA'
+                              'AAAABAAEAAAICTAEAOw==')
+            content_type = 'image/gif'
+        response = HttpResponse(thumb, content_type=content_type)
+        # Thumbnail media is never updated so we can ask the browser to cache it
+        # for ages, 604800 = 7 days
+        response['Cache-Control'] = 'public, max-age=604800'
+        return response
+
+
+class MediaItemView(DetailView):
+
+    template_name = 'sync/media-item.html'
+    model = Media
+
+    def get_context_data(self, *args, **kwargs):
+        data = super().get_context_data(*args, **kwargs)
+        combined_exact, combined_format = self.object.get_best_combined_format()
+        audio_exact, audio_format = self.object.get_best_audio_format()
+        video_exact, video_format = self.object.get_best_video_format()
+        data['combined_exact'] = combined_exact
+        data['combined_format'] = combined_format
+        data['audio_exact'] = audio_exact
+        data['audio_format'] = audio_format
+        data['video_exact'] = video_exact
+        data['video_format'] = video_format
+        data['youtube_dl_format'] = self.object.get_format_str()
+        return data
+
+
+class TasksView(ListView):
+    '''
+        A list of tasks queued to be completed. This is, for example, scraping for new
+        media or downloading media.
+    '''
+
+    template_name = 'sync/tasks.html'
+    context_object_name = 'tasks'
+
+    def get_queryset(self):
+        return Task.objects.all().order_by('run_at')
+
+    def get_context_data(self, *args, **kwargs):
+        data = super().get_context_data(*args, **kwargs)
+        data['running'] = []
+        data['errors'] = []
+        data['scheduled'] = []
+        queryset = self.get_queryset()
+        now = timezone.now()
+        for task in queryset:
+            obj, url = map_task_to_instance(task)
+            if not obj:
+                # Orphaned task, ignore it (it will be deleted when it fires)
+                continue
+            setattr(task, 'instance', obj)
+            setattr(task, 'url', url)
+            if task.locked_by_pid_running():
+                data['running'].append(task)
+            elif task.has_error():
+                error_message = get_error_message(task)
+                setattr(task, 'error_message', error_message)
+                data['errors'].append(task)
+            else:
+                data['scheduled'].append(task)
+        return data
+
+
+class CompletedTasksView(ListView):
+    '''
+        List of tasks which have been completed with an optional per-source filter.
+    '''
+
+    template_name = 'sync/tasks-completed.html'
+    context_object_name = 'tasks'
+    paginate_by = settings.TASKS_PER_PAGE
+    messages = {
+        'filter': _('Viewing tasks filtered for source: <strong>{name}</strong>'),
+    }
+
+    def __init__(self, *args, **kwargs):
+        self.filter_source = None
+        super().__init__(*args, **kwargs)
+
+    def dispatch(self, request, *args, **kwargs):
+        filter_by = request.GET.get('filter', '')
+        if filter_by:
+            try:
+                self.filter_source = Source.objects.get(pk=filter_by)
+            except Source.DoesNotExist:
+                self.filter_source = None
+        return super().dispatch(request, *args, **kwargs)
+
+    def get_queryset(self):
+        return CompletedTask.objects.all().order_by('-run_at')
+
+    def get_queryset(self):
+        if self.filter_source:
+            q = CompletedTask.objects.filter(queue=str(self.filter_source.pk))
+        else:
+            q = CompletedTask.objects.all()
+        return q.order_by('-run_at')
+
+    def get_context_data(self, *args, **kwargs):
+        data = super().get_context_data(*args, **kwargs)
+        for task in data['tasks']:
+            if task.has_error():
+                error_message = get_error_message(task)
+                setattr(task, 'error_message', error_message)
+        data['message'] = ''
+        data['source'] = None
+        if self.filter_source:
+            message = str(self.messages.get('filter', ''))
+            data['message'] = message.format(name=self.filter_source.name)
+            data['source'] = self.filter_source
+        return data

tubesync/sync/youtube.py

@@ -0,0 +1,61 @@
+'''
+    Wrapper for the youtube-dl library. Used so if there are any library interface
+    updates we only need to udpate them in one place.
+'''
+
+
+from django.conf import settings
+from copy import copy
+from common.logger import log
+import youtube_dl
+
+
+_defaults = getattr(settings, 'YOUTUBE_DEFAULTS', {})
+_defaults.update({'logger': log})
+
+
+class YouTubeError(youtube_dl.utils.DownloadError):
+    '''
+        Generic wrapped error for all errors that could be raised by youtube-dl.
+    '''
+    pass
+
+
+def get_media_info(url):
+    '''
+        Extracts information from a YouTube URL and returns it as a dict. For a channel
+        or playlist this returns a dict of all the videos on the channel or playlist
+        as well as associated metadata.
+    '''
+    opts = copy(_defaults)
+    opts.update({
+        'skip_download': True,
+        'forcejson': True,
+        'simulate': True,
+    })
+    response = {}
+    with youtube_dl.YoutubeDL(opts) as y:
+        try:
+            response = y.extract_info(url, download=False)
+        except youtube_dl.utils.DownloadError as e:
+            raise YouTubeError(f'Failed to extract_info for "{url}": {e}') from e
+    return response
+
+
+def download_media(url, media_format, extension, output_file):
+    '''
+        Downloads a YouTube URL to a file on disk.
+    '''
+    opts = copy(_defaults)
+    opts.update({
+        'format': media_format,
+        'merge_output_format': extension,
+        'outtmpl': output_file,
+        'quiet': True,
+    })
+    with youtube_dl.YoutubeDL(opts) as y:
+        try:
+            return y.download([url])
+        except youtube_dl.utils.DownloadError as e:
+            raise YouTubeError(f'Failed to download for "{url}": {e}') from e
+    return False