From a0ea2965b8a2c3ef3fc1e9ef6db164a874d2a44b Mon Sep 17 00:00:00 2001 From: meeb Date: Sun, 6 Dec 2020 13:48:10 +1100 Subject: [PATCH] more media management, custom logger --- app/common/logger.py | 10 ++++ app/common/static/styles/_colours.scss | 1 + app/common/static/styles/_template.scss | 3 +- .../migrations/0010_auto_20201206_0159.py | 28 +++++++++ app/sync/models.py | 59 ++++++++++++------- app/sync/signals.py | 16 +++-- app/sync/tasks.py | 5 ++ app/sync/templates/sync/source.html | 25 +++++--- app/sync/templates/sync/sources.html | 7 +-- app/sync/utils.py | 41 +++++++++++++ app/sync/views.py | 4 +- app/sync/youtube.py | 7 ++- app/tubesync/settings.py | 4 +- 13 files changed, 168 insertions(+), 42 deletions(-) create mode 100644 app/common/logger.py create mode 100644 app/sync/migrations/0010_auto_20201206_0159.py diff --git a/app/common/logger.py b/app/common/logger.py new file mode 100644 index 0000000..44deb26 --- /dev/null +++ b/app/common/logger.py @@ -0,0 +1,10 @@ +import logging + + +log = logging.getLogger('tubesync') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s [%(name)s/%(levelname)s] %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) diff --git a/app/common/static/styles/_colours.scss b/app/common/static/styles/_colours.scss index 07cb185..9033b60 100644 --- a/app/common/static/styles/_colours.scss +++ b/app/common/static/styles/_colours.scss @@ -37,6 +37,7 @@ $form-help-text-colour: $colour-light-blue; $form-delete-button-background-colour: $colour-red; $collection-no-items-text-colour: $colour-near-black; +$collection-text-colour: $colour-near-black; $collection-background-hover-colour: $colour-orange; $collection-text-hover-colour: $colour-near-white; diff --git a/app/common/static/styles/_template.scss b/app/common/static/styles/_template.scss index fc5d181..d7e8bb4 100644 --- a/app/common/static/styles/_template.scss +++ b/app/common/static/styles/_template.scss @@ -88,11 +88,12 @@ main { } .collection { + margin: 0.5rem 0 0 0 !important; .collection-item { display: block; } a.collection-item { - color: $main-link-colour; + color: $collection-text-colour; text-decoration: none; &:hover { background-color: $collection-background-hover-colour !important; diff --git a/app/sync/migrations/0010_auto_20201206_0159.py b/app/sync/migrations/0010_auto_20201206_0159.py new file mode 100644 index 0000000..87e30a9 --- /dev/null +++ b/app/sync/migrations/0010_auto_20201206_0159.py @@ -0,0 +1,28 @@ +# Generated by Django 3.1.4 on 2020-12-06 01:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0009_auto_20201205_0512'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='directory', + field=models.CharField(db_index=True, help_text='Directory name to save the media into', max_length=100, unique=True, verbose_name='directory'), + ), + migrations.AlterField( + model_name='source', + name='fallback', + field=models.CharField(choices=[('f', 'Fail, do not download any media'), ('s', 'Get next best SD media or codec instead'), ('h', 'Get next best HD media or codec instead')], db_index=True, default='f', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback'), + ), + migrations.AlterField( + model_name='source', + name='name', + field=models.CharField(db_index=True, help_text='Friendly name for the source, used locally in TubeSync only', max_length=100, unique=True, verbose_name='name'), + ), + ] diff --git a/app/sync/models.py b/app/sync/models.py index d4c4ae9..5b46aed 100644 --- a/app/sync/models.py +++ b/app/sync/models.py @@ -128,11 +128,14 @@ class Source(models.Model): _('name'), max_length=100, db_index=True, + unique=True, help_text=_('Friendly name for the source, used locally in TubeSync only') ) directory = models.CharField( _('directory'), max_length=100, + db_index=True, + unique=True, help_text=_('Directory name to save the media into') ) delete_old_media = models.BooleanField( @@ -200,6 +203,33 @@ class Source(models.Model): def icon(self): return self.ICONS.get(self.source_type) + @property + def is_audio(self): + return self.source_resolution == self.SOURCE_RESOLUTION_AUDIO + + @property + def is_video(self): + return not self.is_audio + + @property + def extension(self): + ''' + The extension is also used by youtube-dl to set the output container. As + it is possible to quite easily pick combinations of codecs and containers + which are invalid (e.g. OPUS audio in an MP4 container) just set this for + people. All video is set to mkv containers, audio-only is set to m4a or ogg + depending on audio codec. + ''' + if self.is_audio: + if self.source_acodec == self.SOURCE_ACODEC_M4A: + return 'm4a' + elif self.source_acodec == self.SOURCE_ACODEC_OPUS: + return 'ogg' + else: + raise ValueError('Unable to choose audio extension, uknown acodec') + else: + return 'mkv' + @classmethod def create_url(obj, source_type, key): url = obj.URLS.get(source_type) @@ -387,25 +417,6 @@ class Media(models.Model): _metadata_cache[self.pk] = json.loads(self.metadata) return _metadata_cache[self.pk] - @property - def extension(self): - ''' - The extension is also used by youtube-dl to set the output container. As - it is possible to quite easily pick combinations of codecs and containers - which are invalid (e.g. OPUS audio in an MP4 container) just set this for - people. All video is set to mkv containers, audio-only is set to m4a or ogg - depending on audio codec. - ''' - if self.source.source_resolution == Source.SOURCE_RESOLUTION_AUDIO: - if self.source.source_acodec == Source.SOURCE_ACODEC_M4A: - return 'm4a' - elif self.source.source_acodec == Source.SOURCE_ACODEC_OPUS: - return 'ogg' - else: - raise ValueError('Unable to choose audio extension, uknown acodec') - else: - return 'mkv' - @property def url(self): url = self.URLS.get(self.source.source_type, '') @@ -426,6 +437,12 @@ class Media(models.Model): @property def filename(self): upload_date = self.upload_date.strftime('%Y-%m-%d') + source_name = slugify(self.source.name) title = slugify(self.title.replace('&', 'and').replace('+', 'and')) - ext = self.extension - return f'{upload_date}_{title}.{ext}' + ext = self.source.extension + fn = f'{upload_date}_{source_name}_{title}'[:100] + return f'{fn}.{ext}' + + @property + def filepath(self): + return self.source.directory_path / self.filename diff --git a/app/sync/signals.py b/app/sync/signals.py index bdbbcd3..3dfa5c4 100644 --- a/app/sync/signals.py +++ b/app/sync/signals.py @@ -1,9 +1,10 @@ from django.conf import settings -from django.db.models.signals import post_save, post_delete +from django.db.models.signals import post_save, pre_delete, post_delete from django.dispatch import receiver from .models import Source, Media from .tasks import delete_index_source_task, index_source_task, download_media_thumbnail +from .utils import delete_file @receiver(post_save, sender=Source) @@ -14,6 +15,14 @@ def source_post_save(sender, instance, created, **kwargs): index_source_task(str(instance.pk), repeat=settings.INDEX_SOURCE_EVERY) +@receiver(pre_delete, sender=Source) +def source_post_delete(sender, instance, **kwargs): + # Triggered just before a source is deleted, delete all media objects to trigger + # the Media models post_delete signal + for media in Media.objects.filter(source=instance): + media.delete() + + @receiver(post_delete, sender=Source) def source_post_delete(sender, instance, **kwargs): # Triggered when a source is deleted @@ -33,6 +42,5 @@ def media_post_save(sender, instance, created, **kwargs): @receiver(post_delete, sender=Media) def media_post_delete(sender, instance, **kwargs): - # Triggered when media is deleted - pass - # TODO: delete thumbnail and media file from disk + # Triggered when media is deleted, delete media thumbnail + delete_file(instance.thumb.path) diff --git a/app/sync/tasks.py b/app/sync/tasks.py index e51ceb1..544043a 100644 --- a/app/sync/tasks.py +++ b/app/sync/tasks.py @@ -10,6 +10,7 @@ from django.conf import settings from django.core.files.uploadedfile import SimpleUploadedFile from background_task import background from background_task.models import Task +from common.logger import log from .models import Source, Media from .utils import get_remote_image @@ -23,6 +24,7 @@ def delete_index_source_task(source_id): pass if task: # A scheduled task exists for this Source, delete it + log.info(f'Deleting Source index task: {task}') task.delete() @@ -51,6 +53,7 @@ def index_source_task(source_id): media.source = source media.metadata = json.dumps(video) media.save() + log.info(f'Indexed media: {source} / {media}') @background(schedule=0) @@ -68,6 +71,7 @@ def download_media_thumbnail(media_id, url): max_width, max_height = getattr(settings, 'MAX_MEDIA_THUMBNAIL_SIZE', (512, 512)) if i.width > max_width or i.height > max_height: # Image is larger than we want to save, resize it + log.info(f'Resizing thumbnail ({i.width}x{i.height}): {url}') i.thumbnail(size=(max_width, max_height)) image_file = BytesIO() i.save(image_file, 'JPEG', quality=80, optimize=True, progressive=True) @@ -81,4 +85,5 @@ def download_media_thumbnail(media_id, url): ), save=True ) + log.info(f'Saved thumbnail for: {media} from: {url}') return True diff --git a/app/sync/templates/sync/source.html b/app/sync/templates/sync/source.html index e48e8f2..2ba753d 100644 --- a/app/sync/templates/sync/source.html +++ b/app/sync/templates/sync/source.html @@ -8,6 +8,7 @@

Source {{ source.name }}

{{ source.url }}

Saving to: {{ source.directory_path }}

+

Media linked to this source

@@ -31,15 +32,25 @@ Created - Created
{{ source.created|date:'Y-m-d H-I-S' }} + Created
{{ source.created|date:'Y-m-d H:i:s' }} Last crawl Last crawl
{% if source.last_crawl %}{{ source.last_crawl|date:'Y-m-d H-I-S' }}{% else %}Never{% endif %} - Source profile - Source profile
{{ source.get_source_profile_display }} + Source resolution + Source resolution
{{ source.get_source_resolution_display }} + + {% if source.is_video %} + + Source video codec + Source video codec
{{ source.get_source_vcodec_display }} + + {% endif %} + + Source audio codec + Source audio codec
{{ source.get_source_acodec_display }} Prefer 60FPS? @@ -49,11 +60,11 @@ Prefer HDR? Prefer HDR?
{% if source.prefer_hdr %}{% else %}{% endif %} - - Output format - Output format
{{ source.get_output_format_display }} + + Output extension + Output extension
{{ source.extension }} - + Fallback Fallback
{{ source.get_fallback_display }} diff --git a/app/sync/templates/sync/sources.html b/app/sync/templates/sync/sources.html index 8593971..35feb69 100644 --- a/app/sync/templates/sync/sources.html +++ b/app/sync/templates/sync/sources.html @@ -17,10 +17,9 @@
{% for source in sources %} - {{ source.icon|safe }} {{ source.name }}
- {{ source.get_source_type_display }}
- {{ source.get_source_profile_display }} media in a {{ source.get_output_format_display }} - {% if source.delete_old_media and source.days_to_keep > 0 %}Delete media after {{ source.days_to_keep }} days{% endif %} + {{ source.icon|safe }} {{ source.name }}, {{ source.get_source_type_display }}
+ {{ source.format_summary }}
+ {{ source.media_count }} media items{% if source.delete_old_media and source.days_to_keep > 0 %}, keep {{ source.days_to_keep }} days of media{% endif %} {% empty %} You haven't added any sources. diff --git a/app/sync/utils.py b/app/sync/utils.py index 5bde56b..2e0366b 100644 --- a/app/sync/utils.py +++ b/app/sync/utils.py @@ -1,6 +1,9 @@ +import os import re +from pathlib import Path import requests from PIL import Image +from django.conf import settings from urllib.parse import urlsplit, parse_qs from django.forms import ValidationError @@ -54,3 +57,41 @@ def get_remote_image(url): r = requests.get(url, headers=headers, stream=True, timeout=60) r.raw.decode_content = True return Image.open(r.raw) + + + +def path_is_parent(parent_path, child_path): + # Smooth out relative path names, note: if you are concerned about symbolic links, you should use os.path.realpath too + parent_path = os.path.abspath(parent_path) + child_path = os.path.abspath(child_path) + + # Compare the common path of the parent and child path with the common path of just the parent path. Using the commonpath method on just the parent path will regularise the path name in the same way as the comparison that deals with both paths, removing any trailing path separator + return os.path.commonpath([parent_path]) == os.path.commonpath([parent_path, child_path]) + + +def file_is_editable(filepath): + ''' + Checks that a file exists and the file is in an allowed predefined tuple of + directories we want to allow writing or deleting in. + ''' + allowed_paths = ( + # Media item thumbnails + os.path.commonpath([os.path.abspath(str(settings.MEDIA_ROOT))]), + # Downloaded video files + os.path.commonpath([os.path.abspath(str(settings.SYNC_VIDEO_ROOT))]), + # Downloaded audio files + os.path.commonpath([os.path.abspath(str(settings.SYNC_AUDIO_ROOT))]), + ) + filepath = os.path.abspath(str(filepath)) + if not os.path.isfile(filepath): + return False + for allowed_path in allowed_paths: + if allowed_path == os.path.commonpath([allowed_path, filepath]): + return True + return False + + +def delete_file(filepath): + if file_is_editable(filepath): + return os.remove(filepath) + return False diff --git a/app/sync/views.py b/app/sync/views.py index 78f55af..74cc4bb 100644 --- a/app/sync/views.py +++ b/app/sync/views.py @@ -6,6 +6,7 @@ from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateVi DeleteView) from django.http import HttpResponse from django.urls import reverse_lazy +from django.db.models import Count from django.forms import ValidationError from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ @@ -52,7 +53,8 @@ class SourcesView(ListView): return super().dispatch(request, *args, **kwargs) def get_queryset(self): - return Source.objects.all().order_by('name') + all_sources = Source.objects.all().order_by('name') + return all_sources.annotate(media_count=Count('media_source')) def get_context_data(self, *args, **kwargs): data = super().get_context_data(*args, **kwargs) diff --git a/app/sync/youtube.py b/app/sync/youtube.py index 0c3ac97..7857a1b 100644 --- a/app/sync/youtube.py +++ b/app/sync/youtube.py @@ -5,10 +5,13 @@ from django.conf import settings +from copy import copy +from common.logger import log import youtube_dl _defaults = getattr(settings, 'YOUTUBE_DEFAULTS', {}) +_defaults.update({'logger': log}) class YouTubeError(youtube_dl.utils.DownloadError): @@ -24,8 +27,8 @@ def get_media_info(url): or playlist this returns a dict of all the videos on the channel or playlist as well as associated metadata. ''' - - opts = _defaults.update({ + opts = copy(_defaults) + opts.update({ 'skip_download': True, 'forcejson': True, 'simulate': True, diff --git a/app/tubesync/settings.py b/app/tubesync/settings.py index 77d882d..4f3b394 100644 --- a/app/tubesync/settings.py +++ b/app/tubesync/settings.py @@ -124,14 +124,14 @@ SOURCES_PER_PAGE = 25 MEDIA_PER_PAGE = 25 -INDEX_SOURCE_EVERY = 60 # Seconds between indexing sources, 21600 = every 6 hours +INDEX_SOURCE_EVERY = 21600 # Seconds between indexing sources, 21600 = every 6 hours MAX_MEDIA_THUMBNAIL_SIZE = (320, 240) # Max size in pixels for media thumbnails YOUTUBE_DEFAULTS = { - 'age_limit': 99, # Age in years to spoof the client as + 'age_limit': 99, # 'Age in years' to spoof }