more media management, custom logger

app/common/logger.py

@@ -0,0 +1,10 @@
+import logging
+
+
+log = logging.getLogger('tubesync')
+log.setLevel(logging.DEBUG)
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(asctime)s [%(name)s/%(levelname)s] %(message)s')
+ch.setFormatter(formatter)
+log.addHandler(ch)

app/common/static/styles/_colours.scss

@@ -37,6 +37,7 @@ $form-help-text-colour: $colour-light-blue;
 $form-delete-button-background-colour: $colour-red;
 
 $collection-no-items-text-colour: $colour-near-black;
+$collection-text-colour: $colour-near-black;
 $collection-background-hover-colour: $colour-orange;
 $collection-text-hover-colour: $colour-near-white;
 

app/common/static/styles/_template.scss

@@ -88,11 +88,12 @@ main {
     }
 
     .collection {
+        margin: 0.5rem 0 0 0 !important;
         .collection-item {
             display: block;
         }
         a.collection-item {
-            color: $main-link-colour;
+            color: $collection-text-colour;
             text-decoration: none;
             &:hover {
                 background-color: $collection-background-hover-colour !important;

app/sync/migrations/0010_auto_20201206_0159.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.1.4 on 2020-12-06 01:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0009_auto_20201205_0512'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='directory',
+            field=models.CharField(db_index=True, help_text='Directory name to save the media into', max_length=100, unique=True, verbose_name='directory'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='fallback',
+            field=models.CharField(choices=[('f', 'Fail, do not download any media'), ('s', 'Get next best SD media or codec instead'), ('h', 'Get next best HD media or codec instead')], db_index=True, default='f', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='name',
+            field=models.CharField(db_index=True, help_text='Friendly name for the source, used locally in TubeSync only', max_length=100, unique=True, verbose_name='name'),
+        ),
+    ]

app/sync/models.py

@@ -128,11 +128,14 @@ class Source(models.Model):
         _('name'),
         max_length=100,
         db_index=True,
+        unique=True,
         help_text=_('Friendly name for the source, used locally in TubeSync only')
     )
     directory = models.CharField(
         _('directory'),
         max_length=100,
+        db_index=True,
+        unique=True,
         help_text=_('Directory name to save the media into')
     )
     delete_old_media = models.BooleanField(
@@ -200,6 +203,33 @@ class Source(models.Model):
     def icon(self):
         return self.ICONS.get(self.source_type)
 
+    @property
+    def is_audio(self):
+        return self.source_resolution == self.SOURCE_RESOLUTION_AUDIO
+
+    @property
+    def is_video(self):
+        return not self.is_audio
+
+    @property
+    def extension(self):
+        '''
+            The extension is also used by youtube-dl to set the output container. As
+            it is possible to quite easily pick combinations of codecs and containers
+            which are invalid (e.g. OPUS audio in an MP4 container) just set this for
+            people. All video is set to mkv containers, audio-only is set to m4a or ogg
+            depending on audio codec.
+        '''
+        if self.is_audio:
+            if self.source_acodec == self.SOURCE_ACODEC_M4A:
+                return 'm4a'
+            elif self.source_acodec == self.SOURCE_ACODEC_OPUS:
+                return 'ogg'
+            else:
+                raise ValueError('Unable to choose audio extension, uknown acodec')
+        else:
+            return 'mkv'
+
     @classmethod
     def create_url(obj, source_type, key):
         url = obj.URLS.get(source_type)
@@ -387,25 +417,6 @@ class Media(models.Model):
         _metadata_cache[self.pk] = json.loads(self.metadata)
         return _metadata_cache[self.pk]
 
-    @property
-    def extension(self):
-        '''
-            The extension is also used by youtube-dl to set the output container. As
-            it is possible to quite easily pick combinations of codecs and containers
-            which are invalid (e.g. OPUS audio in an MP4 container) just set this for
-            people. All video is set to mkv containers, audio-only is set to m4a or ogg
-            depending on audio codec.
-        '''
-        if self.source.source_resolution == Source.SOURCE_RESOLUTION_AUDIO:
-            if self.source.source_acodec == Source.SOURCE_ACODEC_M4A:
-                return 'm4a'
-            elif self.source.source_acodec == Source.SOURCE_ACODEC_OPUS:
-                return 'ogg'
-            else:
-                raise ValueError('Unable to choose audio extension, uknown acodec')
-        else:
-            return 'mkv'
-
     @property
     def url(self):
         url = self.URLS.get(self.source.source_type, '')
@@ -426,6 +437,12 @@ class Media(models.Model):
     @property
     def filename(self):
         upload_date = self.upload_date.strftime('%Y-%m-%d')
+        source_name = slugify(self.source.name)
         title = slugify(self.title.replace('&', 'and').replace('+', 'and'))
-        ext = self.extension
+        ext = self.source.extension
-        return f'{upload_date}_{title}.{ext}'
+        fn = f'{upload_date}_{source_name}_{title}'[:100]
+        return f'{fn}.{ext}'
+
+    @property
+    def filepath(self):
+        return self.source.directory_path / self.filename

app/sync/signals.py

@@ -1,9 +1,10 @@
 from django.conf import settings
-from django.db.models.signals import post_save, post_delete
+from django.db.models.signals import post_save, pre_delete, post_delete
 from django.dispatch import receiver
 
 from .models import Source, Media
 from .tasks import delete_index_source_task, index_source_task, download_media_thumbnail
+from .utils import delete_file
 
 
 @receiver(post_save, sender=Source)
@@ -14,6 +15,14 @@ def source_post_save(sender, instance, created, **kwargs):
         index_source_task(str(instance.pk), repeat=settings.INDEX_SOURCE_EVERY)
 
 
+@receiver(pre_delete, sender=Source)
+def source_post_delete(sender, instance, **kwargs):
+    # Triggered just before a source is deleted, delete all media objects to trigger
+    # the Media models post_delete signal
+    for media in Media.objects.filter(source=instance):
+        media.delete()
+
+
 @receiver(post_delete, sender=Source)
 def source_post_delete(sender, instance, **kwargs):
     # Triggered when a source is deleted
@@ -33,6 +42,5 @@ def media_post_save(sender, instance, created, **kwargs):
 
 @receiver(post_delete, sender=Media)
 def media_post_delete(sender, instance, **kwargs):
-    # Triggered when media is deleted
+    # Triggered when media is deleted, delete media thumbnail
-    pass
+    delete_file(instance.thumb.path)
-    # TODO: delete thumbnail and media file from disk

app/sync/tasks.py

@@ -10,6 +10,7 @@ from django.conf import settings
 from django.core.files.uploadedfile import SimpleUploadedFile
 from background_task import background
 from background_task.models import Task
+from common.logger import log
 from .models import Source, Media
 from .utils import get_remote_image
 
@@ -23,6 +24,7 @@ def delete_index_source_task(source_id):
         pass
     if task:
         # A scheduled task exists for this Source, delete it
+        log.info(f'Deleting Source index task: {task}')
         task.delete()
 
 
@@ -51,6 +53,7 @@ def index_source_task(source_id):
         media.source = source
         media.metadata = json.dumps(video)
         media.save()
+        log.info(f'Indexed media: {source} / {media}')
 
 
 @background(schedule=0)
@@ -68,6 +71,7 @@ def download_media_thumbnail(media_id, url):
     max_width, max_height = getattr(settings, 'MAX_MEDIA_THUMBNAIL_SIZE', (512, 512))
     if i.width > max_width or i.height > max_height:
         # Image is larger than we want to save, resize it
+        log.info(f'Resizing thumbnail ({i.width}x{i.height}): {url}')
         i.thumbnail(size=(max_width, max_height))
     image_file = BytesIO()
     i.save(image_file, 'JPEG', quality=80, optimize=True, progressive=True)
@@ -81,4 +85,5 @@ def download_media_thumbnail(media_id, url):
         ),
         save=True
     )
+    log.info(f'Saved thumbnail for: {media} from: {url}')
     return True

app/sync/templates/sync/source.html

@@ -8,6 +8,7 @@
     <h1 class="truncate">Source <strong>{{ source.name }}</strong></h1>
     <p class="truncate"><strong><a href="{{ source.url }}" target="_blank"><i class="fas fa-link"></i> {{ source.url }}</a></strong></p>
     <p class="truncate">Saving to: <strong>{{ source.directory_path }}</strong></p>
+    <p><a href="{% url 'sync:media' %}?filter={{ source.pk }}" class="btn">Media<span class="hide-on-small-only"> linked to this source</span> <i class="fas fa-fw fa-film"></i></a></p>
   </div>
 </div>
 <div class="row">
@@ -31,15 +32,25 @@
       </tr>
       <tr title="When then source was created locally in TubeSync">
         <td class="hide-on-small-only">Created</td>
-        <td><span class="hide-on-med-and-up">Created<br></span><strong>{{ source.created|date:'Y-m-d H-I-S' }}</strong></td>
+        <td><span class="hide-on-med-and-up">Created<br></span><strong>{{ source.created|date:'Y-m-d H:i:s' }}</strong></td>
       </tr>
       <tr title="When the source last checked for available media">
         <td class="hide-on-small-only">Last crawl</td>
         <td><span class="hide-on-med-and-up">Last crawl<br></span><strong>{% if source.last_crawl %}{{ source.last_crawl|date:'Y-m-d H-I-S' }}{% else %}Never{% endif %}</strong></td>
       </tr>
       <tr title="Quality and type of media the source will attempt to sync">
-        <td class="hide-on-small-only">Source profile</td>
+        <td class="hide-on-small-only">Source resolution</td>
-        <td><span class="hide-on-med-and-up">Source profile<br></span><strong>{{ source.get_source_profile_display }}</strong></td>
+        <td><span class="hide-on-med-and-up">Source resolution<br></span><strong>{{ source.get_source_resolution_display }}</strong></td>
+      </tr>
+      {% if source.is_video %}
+      <tr title="Preferred video codec to download">
+        <td class="hide-on-small-only">Source video codec</td>
+        <td><span class="hide-on-med-and-up">Source video codec<br></span><strong>{{ source.get_source_vcodec_display }}</strong></td>
+      </tr>
+      {% endif %}
+      <tr title="Preferred audio codec to download">
+        <td class="hide-on-small-only">Source audio codec</td>
+        <td><span class="hide-on-med-and-up">Source audio codec<br></span><strong>{{ source.get_source_acodec_display }}</strong></td>
       </tr>
       <tr title="If available from the source media in 60FPS will be preferred">
         <td class="hide-on-small-only">Prefer 60FPS?</td>
@@ -49,11 +60,11 @@
         <td class="hide-on-small-only">Prefer HDR?</td>
         <td><span class="hide-on-med-and-up">Prefer HDR?<br></span><strong>{% if source.prefer_hdr %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
-      <tr title="Output file container format to sync media in">
+      <tr title="Output file extension">
-        <td class="hide-on-small-only">Output format</td>
+        <td class="hide-on-small-only">Output extension</td>
-        <td><span class="hide-on-med-and-up">Output format<br></span><strong>{{ source.get_output_format_display }}</strong></td>
+        <td><span class="hide-on-med-and-up">Output extension<br></span><strong>{{ source.extension }}</strong></td>
       </tr>
-      <tr title="What to do if your source profile is unavailable">
+      <tr title="What to do if your source resolution or codecs are unavailable">
         <td class="hide-on-small-only">Fallback</td>
         <td><span class="hide-on-med-and-up">Fallback<br></span><strong>{{ source.get_fallback_display }}</strong></td>
       </tr>

app/sync/templates/sync/sources.html

@@ -17,10 +17,9 @@
     <div class="collection">
     {% for source in sources %}
       <a href="{% url 'sync:source' pk=source.pk %}" class="collection-item">
-        {{ source.icon|safe }} <strong>{{ source.name }}</strong><br>
+        {{ source.icon|safe }} <strong>{{ source.name }}</strong>, {{ source.get_source_type_display }}<br>
-        {{ source.get_source_type_display }}<br>
+        {{ source.format_summary }}<br>
-        {{ source.get_source_profile_display }} media in a {{ source.get_output_format_display }}
+        <strong>{{ source.media_count }}</strong> media items{% if source.delete_old_media and source.days_to_keep > 0 %}, keep {{ source.days_to_keep }} days of media{% endif %}
-        {% if source.delete_old_media and source.days_to_keep > 0 %}Delete media after {{ source.days_to_keep }} days{% endif %}
       </a>
     {% empty %}
       <span class="collection-item no-items"><i class="fas fa-info-circle"></i> You haven't added any sources.</span>

app/sync/utils.py

@@ -1,6 +1,9 @@
+import os
 import re
+from pathlib import Path
 import requests
 from PIL import Image
+from django.conf import settings
 from urllib.parse import urlsplit, parse_qs
 from django.forms import ValidationError
 
@@ -54,3 +57,41 @@ def get_remote_image(url):
     r = requests.get(url, headers=headers, stream=True, timeout=60)
     r.raw.decode_content = True
     return Image.open(r.raw)
+
+
+
+def path_is_parent(parent_path, child_path):
+    # Smooth out relative path names, note: if you are concerned about symbolic links, you should use os.path.realpath too
+    parent_path = os.path.abspath(parent_path)
+    child_path = os.path.abspath(child_path)
+
+    # Compare the common path of the parent and child path with the common path of just the parent path. Using the commonpath method on just the parent path will regularise the path name in the same way as the comparison that deals with both paths, removing any trailing path separator
+    return os.path.commonpath([parent_path]) == os.path.commonpath([parent_path, child_path])
+
+
+def file_is_editable(filepath):
+    '''
+        Checks that a file exists and the file is in an allowed predefined tuple of
+        directories we want to allow writing or deleting in.
+    '''
+    allowed_paths = (
+        # Media item thumbnails
+        os.path.commonpath([os.path.abspath(str(settings.MEDIA_ROOT))]),
+        # Downloaded video files
+        os.path.commonpath([os.path.abspath(str(settings.SYNC_VIDEO_ROOT))]),
+        # Downloaded audio files
+        os.path.commonpath([os.path.abspath(str(settings.SYNC_AUDIO_ROOT))]),
+    )
+    filepath = os.path.abspath(str(filepath))
+    if not os.path.isfile(filepath):
+        return False
+    for allowed_path in allowed_paths:
+        if allowed_path == os.path.commonpath([allowed_path, filepath]):
+            return True
+    return False
+
+
+def delete_file(filepath):
+    if file_is_editable(filepath):
+        return os.remove(filepath)
+    return False

app/sync/views.py

@@ -6,6 +6,7 @@ from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateVi
                                        DeleteView)
 from django.http import HttpResponse
 from django.urls import reverse_lazy
+from django.db.models import Count
 from django.forms import ValidationError
 from django.utils.text import slugify
 from django.utils.translation import gettext_lazy as _
@@ -52,7 +53,8 @@ class SourcesView(ListView):
         return super().dispatch(request, *args, **kwargs)
 
     def get_queryset(self):
-        return Source.objects.all().order_by('name')
+        all_sources = Source.objects.all().order_by('name')
+        return all_sources.annotate(media_count=Count('media_source'))
 
     def get_context_data(self, *args, **kwargs):
         data = super().get_context_data(*args, **kwargs)

app/sync/youtube.py

@@ -5,10 +5,13 @@
 
 
 from django.conf import settings
+from copy import copy
+from common.logger import log
 import youtube_dl
 
 
 _defaults = getattr(settings, 'YOUTUBE_DEFAULTS', {})
+_defaults.update({'logger': log})
 
 
 class YouTubeError(youtube_dl.utils.DownloadError):
@@ -24,8 +27,8 @@ def get_media_info(url):
         or playlist this returns a dict of all the videos on the channel or playlist
         as well as associated metadata.
     '''
-
+    opts = copy(_defaults)
-    opts = _defaults.update({
+    opts.update({
         'skip_download': True,
         'forcejson': True,
         'simulate': True,

app/tubesync/settings.py

@@ -124,14 +124,14 @@ SOURCES_PER_PAGE = 25
 MEDIA_PER_PAGE = 25
 
 
-INDEX_SOURCE_EVERY = 60  # Seconds between indexing sources, 21600 = every 6 hours
+INDEX_SOURCE_EVERY = 21600  # Seconds between indexing sources, 21600 = every 6 hours
 
 
 MAX_MEDIA_THUMBNAIL_SIZE = (320, 240)       # Max size in pixels for media thumbnails
 
 
 YOUTUBE_DEFAULTS = {
-    'age_limit': 99,        # Age in years to spoof the client as
+    'age_limit': 99,        # 'Age in years' to spoof
 }