diff --git a/tubesync/common/static/images/favicon-sources/tubesync-016.png b/tubesync/common/static/images/favicon-sources/tubesync-016.png new file mode 100644 index 0000000..3f95ce1 Binary files /dev/null and b/tubesync/common/static/images/favicon-sources/tubesync-016.png differ diff --git a/tubesync/common/static/images/favicon-sources/tubesync-024.png b/tubesync/common/static/images/favicon-sources/tubesync-024.png new file mode 100644 index 0000000..3019f7a Binary files /dev/null and b/tubesync/common/static/images/favicon-sources/tubesync-024.png differ diff --git a/tubesync/common/static/images/favicon-sources/tubesync-032.png b/tubesync/common/static/images/favicon-sources/tubesync-032.png new file mode 100644 index 0000000..2ad15cf Binary files /dev/null and b/tubesync/common/static/images/favicon-sources/tubesync-032.png differ diff --git a/tubesync/common/static/images/favicon-sources/tubesync-048.png b/tubesync/common/static/images/favicon-sources/tubesync-048.png new file mode 100644 index 0000000..6a8217a Binary files /dev/null and b/tubesync/common/static/images/favicon-sources/tubesync-048.png differ diff --git a/tubesync/common/static/images/favicon-sources/tubesync-064.png b/tubesync/common/static/images/favicon-sources/tubesync-064.png new file mode 100644 index 0000000..61aa895 Binary files /dev/null and b/tubesync/common/static/images/favicon-sources/tubesync-064.png differ diff --git a/tubesync/common/static/images/favicon-sources/tubesync-096.png b/tubesync/common/static/images/favicon-sources/tubesync-096.png new file mode 100644 index 0000000..6623932 Binary files /dev/null and b/tubesync/common/static/images/favicon-sources/tubesync-096.png differ diff --git a/tubesync/common/static/images/favicon-sources/tubesync-128.png b/tubesync/common/static/images/favicon-sources/tubesync-128.png new file mode 100644 index 0000000..18cb5d9 Binary files /dev/null and b/tubesync/common/static/images/favicon-sources/tubesync-128.png differ diff --git a/tubesync/common/static/images/favicon.ico b/tubesync/common/static/images/favicon.ico index 04e6460..cf2e89a 100644 Binary files a/tubesync/common/static/images/favicon.ico and b/tubesync/common/static/images/favicon.ico differ diff --git a/tubesync/common/static/images/tubesync.png b/tubesync/common/static/images/tubesync.png new file mode 100644 index 0000000..6191fa8 Binary files /dev/null and b/tubesync/common/static/images/tubesync.png differ diff --git a/tubesync/common/templates/tubesync-coloured.svg b/tubesync/common/templates/tubesync-coloured.svg new file mode 100644 index 0000000..ea87b83 --- /dev/null +++ b/tubesync/common/templates/tubesync-coloured.svg @@ -0,0 +1,83 @@ + + diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 27e116c..bf9a7f0 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -5,6 +5,7 @@ from datetime import datetime from pathlib import Path from django.conf import settings from django.db import models +from django.core.files.storage import FileSystemStorage from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ from common.errors import NoFormatException @@ -15,6 +16,9 @@ from .matching import (get_best_combined_format, get_best_audio_format, get_best_video_format) +media_file_storage = FileSystemStorage(location=settings.DOWNLOAD_ROOT) + + class Source(models.Model): ''' A Source is a source of media. Currently, this is either a YouTube channel @@ -295,10 +299,11 @@ class Source(models.Model): @property def directory_path(self): + download_dir = Path(media_file_storage.location) if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO: - return settings.SYNC_AUDIO_ROOT / self.directory + return download_dir / settings.DOWNLOAD_AUDIO_DIR / self.directory else: - return settings.SYNC_VIDEO_ROOT / self.directory + return download_dir / settings.DOWNLOAD_VIDEO_DIR / self.directory def make_directory(self): return os.makedirs(self.directory_path, exist_ok=True) @@ -483,6 +488,7 @@ class Media(models.Model): max_length=200, blank=True, null=True, + storage=media_file_storage, help_text=_('Media file') ) downloaded = models.BooleanField( @@ -656,26 +662,29 @@ class Media(models.Model): @property def filename(self): + if self.media_file: + return os.path.basename(self.media_file.name) upload_date = self.upload_date dateobj = upload_date if upload_date else self.created datestr = dateobj.strftime('%Y-%m-%d') - source_name = slugify(self.source.name) - name = slugify(self.name.replace('&', 'and').replace('+', 'and'))[:50] - key = self.key.strip() - fmt = self.source.source_resolution.lower() - if self.source.is_audio(): - codecs = self.source.source_acodec.lower() + source_name = slugify(self.source.name).replace('_', '-') + name = slugify(self.name.replace('&', 'and').replace('+', 'and')) + name = name.replace('_', '-')[:80] + key = self.key.strip().replace('_', '-')[:20] + fmt = [] + if self.source.is_audio: + fmt.append(self.source.source_acodec.lower()) else: - codecs = [] - vcodec = self.source.source_vcodec.lower() - acodec = self.source.source_acodec.lower() - if vcodec: - codecs.append(vcodec) - if acodec: - codecs.append(acodec) - codecs = '-'.join(codecs) + fmt.append(self.source.source_resolution.lower()) + fmt.append(self.source.source_vcodec.lower()) + fmt.append(self.source.source_acodec.lower()) + if self.source.prefer_60fps: + fmt.append('60fps') + if self.source.prefer_hdr: + fmt.append('hdr') + fmt = '-'.join(fmt) ext = self.source.extension - return f'{datestr}_{source_name}_{name}_{key}-{fmt}-{codecs}.{ext}' + return f'{datestr}_{source_name}_{name}_{key}_{fmt}.{ext}' @property def filepath(self): diff --git a/tubesync/sync/templates/sync/media-item.html b/tubesync/sync/templates/sync/media-item.html index 836a6e9..48c04d0 100644 --- a/tubesync/sync/templates/sync/media-item.html +++ b/tubesync/sync/templates/sync/media-item.html @@ -32,9 +32,13 @@
+ {% if task.locked_by_pid_running %} + {{ task }}
+ Task started at {{ task.run_at|date:'Y-m-d H:i:s' }} + {% else %} {{ task }}
- {% if task.instance.index_schedule %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.
{% endif %} Task will run {% if task.run_now %}immediately{% else %}at {{ task.run_at|date:'Y-m-d H:i:s' }}{% endif %} + {% endif %}
@@ -80,6 +84,7 @@ Container Container
{{ media.downloaded_container|upper }} + {% if media.downloaded_video_codec %} Downloaded FPS Downloaded FPS
{{ media.downloaded_fps }} FPS @@ -88,6 +93,7 @@ Downloaded HDR? Downloaded HDR?
{% if media.downloaded_hdr %}{% else %}{% endif %} + {% endif %} {% else %} Can download? @@ -98,12 +104,12 @@ Available formats Available formats
 {% for format in media.formats %} - +
ID: {{ format.format_id }} {% if format.vcodec|lower != 'none' %}, {{ format.format_note }} ({{ format.width }}x{{ format.height }}), fps:{{ format.fps|lower }}, video:{{ format.vcodec }} @{{ format.tbr }}k{% endif %} {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz{% endif %} {% if format.format_id == combined_format or format.format_id == audio_format or format.format_id == video_format %}(matched){% endif %} - +
{% empty %} Media has no indexed available formats {% endfor %} diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index c3f82a2..63b5a9c 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -96,10 +96,8 @@ def file_is_editable(filepath): allowed_paths = ( # Media item thumbnails os.path.commonpath([os.path.abspath(str(settings.MEDIA_ROOT))]), - # Downloaded video files - os.path.commonpath([os.path.abspath(str(settings.SYNC_VIDEO_ROOT))]), - # Downloaded audio files - os.path.commonpath([os.path.abspath(str(settings.SYNC_AUDIO_ROOT))]), + # Downloaded media files + os.path.commonpath([os.path.abspath(str(settings.DOWNLOAD_ROOT))]), ) filepath = os.path.abspath(str(filepath)) if not os.path.isfile(filepath): diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 562da19..637df6a 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -4,6 +4,7 @@ ''' +import os from django.conf import settings from copy import copy from common.logger import log @@ -11,7 +12,6 @@ import youtube_dl _defaults = getattr(settings, 'YOUTUBE_DEFAULTS', {}) -_defaults.update({'logger': log}) class YouTubeError(youtube_dl.utils.DownloadError): @@ -32,6 +32,7 @@ def get_media_info(url): 'skip_download': True, 'forcejson': True, 'simulate': True, + 'logger': log }) response = {} with youtube_dl.YoutubeDL(opts) as y: @@ -46,12 +47,37 @@ def download_media(url, media_format, extension, output_file): ''' Downloads a YouTube URL to a file on disk. ''' + + def hook(event): + filename = os.path.basename(event['filename']) + if event['status'] == 'error': + log.error(f'[youtube-dl] error occured downloading: {filename}') + elif event['status'] == 'downloading': + p = round((event['downloaded_bytes'] / event['total_bytes']) * 100, -1) + if p > hook.download_progress: + hook.download_progress = p + eta = event.get('_eta_str', '?').strip() + percent_done = event.get('_percent_str', '?').strip() + speed = event.get('_speed_str', '?').strip() + total = event.get('_total_bytes_str', '?').strip() + log.info(f'[youtube-dl] downloading: {filename} - {percent_done} of ' + f'{total} at {speed}, {eta} remaining') + elif event['status'] == 'finished': + total_size_str = event.get('_total_bytes_str', '?').strip() + elapsed_str = event.get('_elapsed_str', '?').strip() + log.info(f'[youtube-dl] finished downloading: {filename} - ' + f'{total_size_str} in {elapsed_str}') + else: + log.warn(f'[youtube-dl] unknown event: {str(event)}') + hook.download_progress = 0 + opts = copy(_defaults) opts.update({ 'format': media_format, 'merge_output_format': extension, 'outtmpl': output_file, 'quiet': True, + 'progress_hooks': [hook], }) with youtube_dl.YoutubeDL(opts) as y: try: diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 3b71cbc..c51183b 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -25,5 +25,4 @@ BACKGROUND_TASK_ASYNC_THREADS = int(os.get('TUBESYNC_WORKERS', 2)) MEDIA_ROOT = ROOT_DIR / 'config' / 'media' -SYNC_VIDEO_ROOT = ROOT_DIR / 'downloads' / 'video' -SYNC_AUDIO_ROOT = ROOT_DIR / 'downloads' / 'audio' +DOWNLOAD_ROOT = ROOT_DIR / 'downloads' diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 2120836..3facb96 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -98,8 +98,9 @@ STATIC_URL = '/static/' STATIC_ROOT = BASE_DIR / 'static' #MEDIA_URL = '/media/' MEDIA_ROOT = BASE_DIR / 'media' -SYNC_VIDEO_ROOT = BASE_DIR / 'downloads' / 'video' -SYNC_AUDIO_ROOT = BASE_DIR / 'downloads' / 'audio' +DOWNLOAD_ROOT = BASE_DIR / 'downloads' +DOWNLOAD_VIDEO_DIR = 'video' +DOWNLOAD_AUDIO_DIR = 'audio' SASS_PROCESSOR_ROOT = STATIC_ROOT @@ -115,9 +116,9 @@ HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',) MAX_ATTEMPTS = 10 # Number of times tasks will be retried MAX_RUN_TIME = 1800 # Maximum amount of time in seconds a task can run -BACKGROUND_TASK_RUN_ASYNC = True # Run tasks async in the background -BACKGROUND_TASK_ASYNC_THREADS = 2 # Number of async tasks to run at once -BACKGROUND_TASK_PRIORITY_ORDERING = 'ASC' # Use 'niceness' task priority ordering +BACKGROUND_TASK_RUN_ASYNC = False # Run tasks async in the background +BACKGROUND_TASK_ASYNC_THREADS = 1 # Number of async tasks to run at once +BACKGROUND_TASK_PRIORITY_ORDERING = 'ASC' # Use 'niceness' task priority ordering COMPLETED_TASKS_DAYS_TO_KEEP = 30 # Number of days to keep completed tasks