From 60a7305fb7ceae7a23d9b7d1eb91a928f756f44e Mon Sep 17 00:00:00 2001 From: meeb Date: Sun, 6 Dec 2020 18:33:48 +1100 Subject: [PATCH] tweak media thumb downloading, more work on media interface --- app/common/static/styles/_colours.scss | 10 ++++---- app/common/templates/pagination.html | 2 +- app/sync/models.py | 32 ++++++++++++++++++++++---- app/sync/signals.py | 3 ++- app/sync/tasks.py | 20 ++++++++++++---- app/sync/templates/sync/media.html | 5 ++-- app/sync/templates/sync/sources.html | 2 +- app/sync/utils.py | 10 -------- app/sync/views.py | 7 +++--- app/tubesync/settings.py | 7 +++--- 10 files changed, 61 insertions(+), 37 deletions(-) diff --git a/app/common/static/styles/_colours.scss b/app/common/static/styles/_colours.scss index 9033b60..3b42365 100644 --- a/app/common/static/styles/_colours.scss +++ b/app/common/static/styles/_colours.scss @@ -41,12 +41,15 @@ $collection-text-colour: $colour-near-black; $collection-background-hover-colour: $colour-orange; $collection-text-hover-colour: $colour-near-white; -$mediacard-title-background-colour: $colour-white; -$mediacard-title-text-colour: $colour-black; +$mediacard-title-background-colour: $colour-black; +$mediacard-title-text-colour: $colour-white; $box-error-background-colour: $colour-red; $box-error-text-colour: $colour-near-white; +$infobox-background-colour: $colour-near-black; +$infobox-text-colour: $colour-near-white; + $pagination-background-colour: $colour-near-white; $pagination-text-colour: $colour-near-black; $pagination-border-colour: $colour-light-blue; @@ -56,6 +59,3 @@ $pagination-border-hover-colour: $colour-light-blue; $pagination-current-background-colour: $colour-orange; $pagination-current-text-colour: $colour-near-white; $pagination-current-border-colour: $colour-orange; - -$infobox-background-colour: $colour-near-white; -$infobox-text-colour: $colour-near-black; diff --git a/app/common/templates/pagination.html b/app/common/templates/pagination.html index 938d629..ae8f138 100644 --- a/app/common/templates/pagination.html +++ b/app/common/templates/pagination.html @@ -3,7 +3,7 @@
diff --git a/app/sync/models.py b/app/sync/models.py index 5b46aed..7fedb7f 100644 --- a/app/sync/models.py +++ b/app/sync/models.py @@ -188,7 +188,7 @@ class Source(models.Model): max_length=1, db_index=True, choices=FALLBACK_CHOICES, - default=FALLBACK_FAIL, + default=FALLBACK_NEXT_HD, help_text=_('What do do when media in your source resolution and codecs is not available') ) @@ -269,7 +269,22 @@ class Source(models.Model): if not callable(indexer): raise Exception(f'Source type f"{self.source_type}" has no indexer') response = indexer(self.url) - return response.get('entries', []) + + # Account for nested playlists, such as a channel of playlists of playlists + def _recurse_playlists(playlist): + videos = [] + entries = playlist.get('entries', []) + for entry in entries: + if not entry: + continue + subentries = entry.get('entries', []) + if subentries: + videos = videos + _recurse_playlists(entry) + else: + videos.append(entry) + return videos + + return _recurse_playlists(response) def get_media_thumb_path(instance, filename): @@ -426,21 +441,28 @@ class Media(models.Model): def title(self): return self.loaded_metadata.get('title', '').strip() + @property + def name(self): + title = self.title + return title if title else self.key + @property def upload_date(self): upload_date_str = self.loaded_metadata.get('upload_date', '').strip() try: return datetime.strptime(upload_date_str, '%Y%m%d') - except ValueError as e: + except (AttributeError, ValueError) as e: return None @property def filename(self): - upload_date = self.upload_date.strftime('%Y-%m-%d') + upload_date = self.upload_date + dateobj = upload_date if upload_date else self.created + datestr = dateobj.strftime('%Y-%m-%d') source_name = slugify(self.source.name) title = slugify(self.title.replace('&', 'and').replace('+', 'and')) ext = self.source.extension - fn = f'{upload_date}_{source_name}_{title}'[:100] + fn = f'{datestr}_{source_name}_{title}'[:100] return f'{fn}.{ext}' @property diff --git a/app/sync/signals.py b/app/sync/signals.py index 3dfa5c4..196f050 100644 --- a/app/sync/signals.py +++ b/app/sync/signals.py @@ -43,4 +43,5 @@ def media_post_save(sender, instance, created, **kwargs): @receiver(post_delete, sender=Media) def media_post_delete(sender, instance, **kwargs): # Triggered when media is deleted, delete media thumbnail - delete_file(instance.thumb.path) + if instance.thumb: + delete_file(instance.thumb.path) diff --git a/app/sync/tasks.py b/app/sync/tasks.py index 544043a..fd19244 100644 --- a/app/sync/tasks.py +++ b/app/sync/tasks.py @@ -5,9 +5,12 @@ import json +import math from io import BytesIO +from PIL import Image from django.conf import settings from django.core.files.uploadedfile import SimpleUploadedFile +from django.utils import timezone from background_task import background from background_task.models import Task from common.logger import log @@ -52,6 +55,12 @@ def index_source_task(source_id): media = Media(key=key) media.source = source media.metadata = json.dumps(video) + upload_date = media.upload_date + if upload_date: + if timezone.is_aware(upload_date): + media.published = upload_date + else: + media.published = timezone.make_aware(upload_date) media.save() log.info(f'Indexed media: {source} / {media}') @@ -68,11 +77,12 @@ def download_media_thumbnail(media_id, url): # Task triggered but the media no longer exists, ignore task return i = get_remote_image(url) - max_width, max_height = getattr(settings, 'MAX_MEDIA_THUMBNAIL_SIZE', (512, 512)) - if i.width > max_width or i.height > max_height: - # Image is larger than we want to save, resize it - log.info(f'Resizing thumbnail ({i.width}x{i.height}): {url}') - i.thumbnail(size=(max_width, max_height)) + ratio = i.width / i.height + new_height = getattr(settings, 'MEDIA_THUMBNAIL_HEIGHT', 240) + new_width = math.ceil(new_height * ratio) + log.info(f'Resizing {i.width}x{i.height} thumbnail to ' + f'{new_width}x{new_height}: {url}') + i = i.resize((new_width, new_height), Image.ANTIALIAS) image_file = BytesIO() i.save(image_file, 'JPEG', quality=80, optimize=True, progressive=True) image_file.seek(0) diff --git a/app/sync/templates/sync/media.html b/app/sync/templates/sync/media.html index d779797..3a97bda 100644 --- a/app/sync/templates/sync/media.html +++ b/app/sync/templates/sync/media.html @@ -12,7 +12,8 @@
{{ m.source }}
- {{ m }} + {{ m.name }}
+ {{ m.published|date:'Y-m-d' }}
@@ -26,5 +27,5 @@ {% endfor %} -{% include 'pagination.html' with pagination=sources.paginator %} +{% include 'pagination.html' with pagination=sources.paginator filter=source.pk %} {% endblock %} diff --git a/app/sync/templates/sync/sources.html b/app/sync/templates/sync/sources.html index 35feb69..c2feedc 100644 --- a/app/sync/templates/sync/sources.html +++ b/app/sync/templates/sync/sources.html @@ -17,7 +17,7 @@
{% for source in sources %} - {{ source.icon|safe }} {{ source.name }}, {{ source.get_source_type_display }}
+ {{ source.icon|safe }} {{ source.name }} ({{ source.get_source_type_display }})
{{ source.format_summary }}
{{ source.media_count }} media items{% if source.delete_old_media and source.days_to_keep > 0 %}, keep {{ source.days_to_keep }} days of media{% endif %} diff --git a/app/sync/utils.py b/app/sync/utils.py index 2e0366b..0db29a6 100644 --- a/app/sync/utils.py +++ b/app/sync/utils.py @@ -59,16 +59,6 @@ def get_remote_image(url): return Image.open(r.raw) - -def path_is_parent(parent_path, child_path): - # Smooth out relative path names, note: if you are concerned about symbolic links, you should use os.path.realpath too - parent_path = os.path.abspath(parent_path) - child_path = os.path.abspath(child_path) - - # Compare the common path of the parent and child path with the common path of just the parent path. Using the commonpath method on just the parent path will regularise the path name in the same way as the comparison that deals with both paths, removing any trailing path separator - return os.path.commonpath([parent_path]) == os.path.commonpath([parent_path, child_path]) - - def file_is_editable(filepath): ''' Checks that a file exists and the file is in an allowed predefined tuple of diff --git a/app/sync/views.py b/app/sync/views.py index 74cc4bb..2b6a648 100644 --- a/app/sync/views.py +++ b/app/sync/views.py @@ -304,9 +304,10 @@ class MediaView(ListView): def get_queryset(self): if self.filter_source: - return Media.objects.filter(source=self.filter_source).order_by('-created') + q = Media.objects.filter(source=self.filter_source) else: - return Media.objects.all().order_by('-created') + q = Media.objects.all() + return q.order_by('-published', '-created') def get_context_data(self, *args, **kwargs): data = super().get_context_data(*args, **kwargs) @@ -314,10 +315,8 @@ class MediaView(ListView): data['source'] = None if self.filter_source: message = str(self.messages.get('filter', '')) - print(message) data['message'] = message.format(name=self.filter_source.name) data['source'] = self.filter_source - print(data) return data diff --git a/app/tubesync/settings.py b/app/tubesync/settings.py index 4f9bcf1..1a032e9 100644 --- a/app/tubesync/settings.py +++ b/app/tubesync/settings.py @@ -120,19 +120,20 @@ BACKGROUND_TASK_ASYNC_THREADS = 2 # Number of async tasks to run at on BACKGROUND_TASK_PRIORITY_ORDERING = 'DESC' # Process high priority tasks first -SOURCES_PER_PAGE = 25 -MEDIA_PER_PAGE = 25 +SOURCES_PER_PAGE = 36 +MEDIA_PER_PAGE = 36 INDEX_SOURCE_EVERY = 21600 # Seconds between indexing sources, 21600 = every 6 hours -MAX_MEDIA_THUMBNAIL_SIZE = (320, 240) # Max size in pixels for media thumbnails +MEDIA_THUMBNAIL_HEIGHT = 240 # Height in pixels to resize thumbnails to YOUTUBE_DEFAULTS = { 'no_color': True, # Do not use colours in output 'age_limit': 99, # 'Age in years' to spoof + 'ignoreerrors': True, # Skip on errors (such as unavailable videos in playlists) }