more media management, custom logger
This commit is contained in:
meeb
parent
149d1357e6
commit
a0ea2965b8
|
|
@ -0,0 +1,10 @@
|
|||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger('tubesync')
|
||||
log.setLevel(logging.DEBUG)
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter('%(asctime)s [%(name)s/%(levelname)s] %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
log.addHandler(ch)
|
||||
|
|
@ -37,6 +37,7 @@ $form-help-text-colour: $colour-light-blue;
|
|||
$form-delete-button-background-colour: $colour-red;
|
||||
|
||||
$collection-no-items-text-colour: $colour-near-black;
|
||||
$collection-text-colour: $colour-near-black;
|
||||
$collection-background-hover-colour: $colour-orange;
|
||||
$collection-text-hover-colour: $colour-near-white;
|
||||
|
||||
|
|
|
|||
|
|
@ -88,11 +88,12 @@ main {
|
|||
}
|
||||
|
||||
.collection {
|
||||
margin: 0.5rem 0 0 0 !important;
|
||||
.collection-item {
|
||||
display: block;
|
||||
}
|
||||
a.collection-item {
|
||||
color: $main-link-colour;
|
||||
color: $collection-text-colour;
|
||||
text-decoration: none;
|
||||
&:hover {
|
||||
background-color: $collection-background-hover-colour !important;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
# Generated by Django 3.1.4 on 2020-12-06 01:59
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0009_auto_20201205_0512'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='source',
|
||||
name='directory',
|
||||
field=models.CharField(db_index=True, help_text='Directory name to save the media into', max_length=100, unique=True, verbose_name='directory'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='source',
|
||||
name='fallback',
|
||||
field=models.CharField(choices=[('f', 'Fail, do not download any media'), ('s', 'Get next best SD media or codec instead'), ('h', 'Get next best HD media or codec instead')], db_index=True, default='f', help_text='What do do when media in your source resolution and codecs is not available', max_length=1, verbose_name='fallback'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='source',
|
||||
name='name',
|
||||
field=models.CharField(db_index=True, help_text='Friendly name for the source, used locally in TubeSync only', max_length=100, unique=True, verbose_name='name'),
|
||||
),
|
||||
]
|
||||
|
|
@ -128,11 +128,14 @@ class Source(models.Model):
|
|||
_('name'),
|
||||
max_length=100,
|
||||
db_index=True,
|
||||
unique=True,
|
||||
help_text=_('Friendly name for the source, used locally in TubeSync only')
|
||||
)
|
||||
directory = models.CharField(
|
||||
_('directory'),
|
||||
max_length=100,
|
||||
db_index=True,
|
||||
unique=True,
|
||||
help_text=_('Directory name to save the media into')
|
||||
)
|
||||
delete_old_media = models.BooleanField(
|
||||
|
|
@ -200,6 +203,33 @@ class Source(models.Model):
|
|||
def icon(self):
|
||||
return self.ICONS.get(self.source_type)
|
||||
|
||||
@property
|
||||
def is_audio(self):
|
||||
return self.source_resolution == self.SOURCE_RESOLUTION_AUDIO
|
||||
|
||||
@property
|
||||
def is_video(self):
|
||||
return not self.is_audio
|
||||
|
||||
@property
|
||||
def extension(self):
|
||||
'''
|
||||
The extension is also used by youtube-dl to set the output container. As
|
||||
it is possible to quite easily pick combinations of codecs and containers
|
||||
which are invalid (e.g. OPUS audio in an MP4 container) just set this for
|
||||
people. All video is set to mkv containers, audio-only is set to m4a or ogg
|
||||
depending on audio codec.
|
||||
'''
|
||||
if self.is_audio:
|
||||
if self.source_acodec == self.SOURCE_ACODEC_M4A:
|
||||
return 'm4a'
|
||||
elif self.source_acodec == self.SOURCE_ACODEC_OPUS:
|
||||
return 'ogg'
|
||||
else:
|
||||
raise ValueError('Unable to choose audio extension, uknown acodec')
|
||||
else:
|
||||
return 'mkv'
|
||||
|
||||
@classmethod
|
||||
def create_url(obj, source_type, key):
|
||||
url = obj.URLS.get(source_type)
|
||||
|
|
@ -387,25 +417,6 @@ class Media(models.Model):
|
|||
_metadata_cache[self.pk] = json.loads(self.metadata)
|
||||
return _metadata_cache[self.pk]
|
||||
|
||||
@property
|
||||
def extension(self):
|
||||
'''
|
||||
The extension is also used by youtube-dl to set the output container. As
|
||||
it is possible to quite easily pick combinations of codecs and containers
|
||||
which are invalid (e.g. OPUS audio in an MP4 container) just set this for
|
||||
people. All video is set to mkv containers, audio-only is set to m4a or ogg
|
||||
depending on audio codec.
|
||||
'''
|
||||
if self.source.source_resolution == Source.SOURCE_RESOLUTION_AUDIO:
|
||||
if self.source.source_acodec == Source.SOURCE_ACODEC_M4A:
|
||||
return 'm4a'
|
||||
elif self.source.source_acodec == Source.SOURCE_ACODEC_OPUS:
|
||||
return 'ogg'
|
||||
else:
|
||||
raise ValueError('Unable to choose audio extension, uknown acodec')
|
||||
else:
|
||||
return 'mkv'
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
url = self.URLS.get(self.source.source_type, '')
|
||||
|
|
@ -426,6 +437,12 @@ class Media(models.Model):
|
|||
@property
|
||||
def filename(self):
|
||||
upload_date = self.upload_date.strftime('%Y-%m-%d')
|
||||
source_name = slugify(self.source.name)
|
||||
title = slugify(self.title.replace('&', 'and').replace('+', 'and'))
|
||||
ext = self.extension
|
||||
return f'{upload_date}_{title}.{ext}'
|
||||
ext = self.source.extension
|
||||
fn = f'{upload_date}_{source_name}_{title}'[:100]
|
||||
return f'{fn}.{ext}'
|
||||
|
||||
@property
|
||||
def filepath(self):
|
||||
return self.source.directory_path / self.filename
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
from django.conf import settings
|
||||
from django.db.models.signals import post_save, post_delete
|
||||
from django.db.models.signals import post_save, pre_delete, post_delete
|
||||
from django.dispatch import receiver
|
||||
|
||||
from .models import Source, Media
|
||||
from .tasks import delete_index_source_task, index_source_task, download_media_thumbnail
|
||||
from .utils import delete_file
|
||||
|
||||
|
||||
@receiver(post_save, sender=Source)
|
||||
|
|
@ -14,6 +15,14 @@ def source_post_save(sender, instance, created, **kwargs):
|
|||
index_source_task(str(instance.pk), repeat=settings.INDEX_SOURCE_EVERY)
|
||||
|
||||
|
||||
@receiver(pre_delete, sender=Source)
|
||||
def source_post_delete(sender, instance, **kwargs):
|
||||
# Triggered just before a source is deleted, delete all media objects to trigger
|
||||
# the Media models post_delete signal
|
||||
for media in Media.objects.filter(source=instance):
|
||||
media.delete()
|
||||
|
||||
|
||||
@receiver(post_delete, sender=Source)
|
||||
def source_post_delete(sender, instance, **kwargs):
|
||||
# Triggered when a source is deleted
|
||||
|
|
@ -33,6 +42,5 @@ def media_post_save(sender, instance, created, **kwargs):
|
|||
|
||||
@receiver(post_delete, sender=Media)
|
||||
def media_post_delete(sender, instance, **kwargs):
|
||||
# Triggered when media is deleted
|
||||
pass
|
||||
# TODO: delete thumbnail and media file from disk
|
||||
# Triggered when media is deleted, delete media thumbnail
|
||||
delete_file(instance.thumb.path)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from django.conf import settings
|
|||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from background_task import background
|
||||
from background_task.models import Task
|
||||
from common.logger import log
|
||||
from .models import Source, Media
|
||||
from .utils import get_remote_image
|
||||
|
||||
|
|
@ -23,6 +24,7 @@ def delete_index_source_task(source_id):
|
|||
pass
|
||||
if task:
|
||||
# A scheduled task exists for this Source, delete it
|
||||
log.info(f'Deleting Source index task: {task}')
|
||||
task.delete()
|
||||
|
||||
|
||||
|
|
@ -51,6 +53,7 @@ def index_source_task(source_id):
|
|||
media.source = source
|
||||
media.metadata = json.dumps(video)
|
||||
media.save()
|
||||
log.info(f'Indexed media: {source} / {media}')
|
||||
|
||||
|
||||
@background(schedule=0)
|
||||
|
|
@ -68,6 +71,7 @@ def download_media_thumbnail(media_id, url):
|
|||
max_width, max_height = getattr(settings, 'MAX_MEDIA_THUMBNAIL_SIZE', (512, 512))
|
||||
if i.width > max_width or i.height > max_height:
|
||||
# Image is larger than we want to save, resize it
|
||||
log.info(f'Resizing thumbnail ({i.width}x{i.height}): {url}')
|
||||
i.thumbnail(size=(max_width, max_height))
|
||||
image_file = BytesIO()
|
||||
i.save(image_file, 'JPEG', quality=80, optimize=True, progressive=True)
|
||||
|
|
@ -81,4 +85,5 @@ def download_media_thumbnail(media_id, url):
|
|||
),
|
||||
save=True
|
||||
)
|
||||
log.info(f'Saved thumbnail for: {media} from: {url}')
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
<h1 class="truncate">Source <strong>{{ source.name }}</strong></h1>
|
||||
<p class="truncate"><strong><a href="{{ source.url }}" target="_blank"><i class="fas fa-link"></i> {{ source.url }}</a></strong></p>
|
||||
<p class="truncate">Saving to: <strong>{{ source.directory_path }}</strong></p>
|
||||
<p><a href="{% url 'sync:media' %}?filter={{ source.pk }}" class="btn">Media<span class="hide-on-small-only"> linked to this source</span> <i class="fas fa-fw fa-film"></i></a></p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
|
|
@ -31,15 +32,25 @@
|
|||
</tr>
|
||||
<tr title="When then source was created locally in TubeSync">
|
||||
<td class="hide-on-small-only">Created</td>
|
||||
<td><span class="hide-on-med-and-up">Created<br></span><strong>{{ source.created|date:'Y-m-d H-I-S' }}</strong></td>
|
||||
<td><span class="hide-on-med-and-up">Created<br></span><strong>{{ source.created|date:'Y-m-d H:i:s' }}</strong></td>
|
||||
</tr>
|
||||
<tr title="When the source last checked for available media">
|
||||
<td class="hide-on-small-only">Last crawl</td>
|
||||
<td><span class="hide-on-med-and-up">Last crawl<br></span><strong>{% if source.last_crawl %}{{ source.last_crawl|date:'Y-m-d H-I-S' }}{% else %}Never{% endif %}</strong></td>
|
||||
</tr>
|
||||
<tr title="Quality and type of media the source will attempt to sync">
|
||||
<td class="hide-on-small-only">Source profile</td>
|
||||
<td><span class="hide-on-med-and-up">Source profile<br></span><strong>{{ source.get_source_profile_display }}</strong></td>
|
||||
<td class="hide-on-small-only">Source resolution</td>
|
||||
<td><span class="hide-on-med-and-up">Source resolution<br></span><strong>{{ source.get_source_resolution_display }}</strong></td>
|
||||
</tr>
|
||||
{% if source.is_video %}
|
||||
<tr title="Preferred video codec to download">
|
||||
<td class="hide-on-small-only">Source video codec</td>
|
||||
<td><span class="hide-on-med-and-up">Source video codec<br></span><strong>{{ source.get_source_vcodec_display }}</strong></td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
<tr title="Preferred audio codec to download">
|
||||
<td class="hide-on-small-only">Source audio codec</td>
|
||||
<td><span class="hide-on-med-and-up">Source audio codec<br></span><strong>{{ source.get_source_acodec_display }}</strong></td>
|
||||
</tr>
|
||||
<tr title="If available from the source media in 60FPS will be preferred">
|
||||
<td class="hide-on-small-only">Prefer 60FPS?</td>
|
||||
|
|
@ -49,11 +60,11 @@
|
|||
<td class="hide-on-small-only">Prefer HDR?</td>
|
||||
<td><span class="hide-on-med-and-up">Prefer HDR?<br></span><strong>{% if source.prefer_hdr %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
||||
</tr>
|
||||
<tr title="Output file container format to sync media in">
|
||||
<td class="hide-on-small-only">Output format</td>
|
||||
<td><span class="hide-on-med-and-up">Output format<br></span><strong>{{ source.get_output_format_display }}</strong></td>
|
||||
<tr title="Output file extension">
|
||||
<td class="hide-on-small-only">Output extension</td>
|
||||
<td><span class="hide-on-med-and-up">Output extension<br></span><strong>{{ source.extension }}</strong></td>
|
||||
</tr>
|
||||
<tr title="What to do if your source profile is unavailable">
|
||||
<tr title="What to do if your source resolution or codecs are unavailable">
|
||||
<td class="hide-on-small-only">Fallback</td>
|
||||
<td><span class="hide-on-med-and-up">Fallback<br></span><strong>{{ source.get_fallback_display }}</strong></td>
|
||||
</tr>
|
||||
|
|
|
|||
|
|
@ -17,10 +17,9 @@
|
|||
<div class="collection">
|
||||
{% for source in sources %}
|
||||
<a href="{% url 'sync:source' pk=source.pk %}" class="collection-item">
|
||||
{{ source.icon|safe }} <strong>{{ source.name }}</strong><br>
|
||||
{{ source.get_source_type_display }}<br>
|
||||
{{ source.get_source_profile_display }} media in a {{ source.get_output_format_display }}
|
||||
{% if source.delete_old_media and source.days_to_keep > 0 %}Delete media after {{ source.days_to_keep }} days{% endif %}
|
||||
{{ source.icon|safe }} <strong>{{ source.name }}</strong>, {{ source.get_source_type_display }}<br>
|
||||
{{ source.format_summary }}<br>
|
||||
<strong>{{ source.media_count }}</strong> media items{% if source.delete_old_media and source.days_to_keep > 0 %}, keep {{ source.days_to_keep }} days of media{% endif %}
|
||||
</a>
|
||||
{% empty %}
|
||||
<span class="collection-item no-items"><i class="fas fa-info-circle"></i> You haven't added any sources.</span>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import requests
|
||||
from PIL import Image
|
||||
from django.conf import settings
|
||||
from urllib.parse import urlsplit, parse_qs
|
||||
from django.forms import ValidationError
|
||||
|
||||
|
|
@ -54,3 +57,41 @@ def get_remote_image(url):
|
|||
r = requests.get(url, headers=headers, stream=True, timeout=60)
|
||||
r.raw.decode_content = True
|
||||
return Image.open(r.raw)
|
||||
|
||||
|
||||
|
||||
def path_is_parent(parent_path, child_path):
|
||||
# Smooth out relative path names, note: if you are concerned about symbolic links, you should use os.path.realpath too
|
||||
parent_path = os.path.abspath(parent_path)
|
||||
child_path = os.path.abspath(child_path)
|
||||
|
||||
# Compare the common path of the parent and child path with the common path of just the parent path. Using the commonpath method on just the parent path will regularise the path name in the same way as the comparison that deals with both paths, removing any trailing path separator
|
||||
return os.path.commonpath([parent_path]) == os.path.commonpath([parent_path, child_path])
|
||||
|
||||
|
||||
def file_is_editable(filepath):
|
||||
'''
|
||||
Checks that a file exists and the file is in an allowed predefined tuple of
|
||||
directories we want to allow writing or deleting in.
|
||||
'''
|
||||
allowed_paths = (
|
||||
# Media item thumbnails
|
||||
os.path.commonpath([os.path.abspath(str(settings.MEDIA_ROOT))]),
|
||||
# Downloaded video files
|
||||
os.path.commonpath([os.path.abspath(str(settings.SYNC_VIDEO_ROOT))]),
|
||||
# Downloaded audio files
|
||||
os.path.commonpath([os.path.abspath(str(settings.SYNC_AUDIO_ROOT))]),
|
||||
)
|
||||
filepath = os.path.abspath(str(filepath))
|
||||
if not os.path.isfile(filepath):
|
||||
return False
|
||||
for allowed_path in allowed_paths:
|
||||
if allowed_path == os.path.commonpath([allowed_path, filepath]):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def delete_file(filepath):
|
||||
if file_is_editable(filepath):
|
||||
return os.remove(filepath)
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateVi
|
|||
DeleteView)
|
||||
from django.http import HttpResponse
|
||||
from django.urls import reverse_lazy
|
||||
from django.db.models import Count
|
||||
from django.forms import ValidationError
|
||||
from django.utils.text import slugify
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
|
@ -52,7 +53,8 @@ class SourcesView(ListView):
|
|||
return super().dispatch(request, *args, **kwargs)
|
||||
|
||||
def get_queryset(self):
|
||||
return Source.objects.all().order_by('name')
|
||||
all_sources = Source.objects.all().order_by('name')
|
||||
return all_sources.annotate(media_count=Count('media_source'))
|
||||
|
||||
def get_context_data(self, *args, **kwargs):
|
||||
data = super().get_context_data(*args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -5,10 +5,13 @@
|
|||
|
||||
|
||||
from django.conf import settings
|
||||
from copy import copy
|
||||
from common.logger import log
|
||||
import youtube_dl
|
||||
|
||||
|
||||
_defaults = getattr(settings, 'YOUTUBE_DEFAULTS', {})
|
||||
_defaults.update({'logger': log})
|
||||
|
||||
|
||||
class YouTubeError(youtube_dl.utils.DownloadError):
|
||||
|
|
@ -24,8 +27,8 @@ def get_media_info(url):
|
|||
or playlist this returns a dict of all the videos on the channel or playlist
|
||||
as well as associated metadata.
|
||||
'''
|
||||
|
||||
opts = _defaults.update({
|
||||
opts = copy(_defaults)
|
||||
opts.update({
|
||||
'skip_download': True,
|
||||
'forcejson': True,
|
||||
'simulate': True,
|
||||
|
|
|
|||
|
|
@ -124,14 +124,14 @@ SOURCES_PER_PAGE = 25
|
|||
MEDIA_PER_PAGE = 25
|
||||
|
||||
|
||||
INDEX_SOURCE_EVERY = 60 # Seconds between indexing sources, 21600 = every 6 hours
|
||||
INDEX_SOURCE_EVERY = 21600 # Seconds between indexing sources, 21600 = every 6 hours
|
||||
|
||||
|
||||
MAX_MEDIA_THUMBNAIL_SIZE = (320, 240) # Max size in pixels for media thumbnails
|
||||
|
||||
|
||||
YOUTUBE_DEFAULTS = {
|
||||
'age_limit': 99, # Age in years to spoof the client as
|
||||
'age_limit': 99, # 'Age in years' to spoof
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue