tweak logo and favicon, add django filestorage for download location, rework youtube-dl logging

tubesync/common/templates/tubesync-coloured.svg

@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   class="tubesync-logo"
+   width="128.00024"
+   height="127.99981"
+   viewBox="0 0 128.00024 127.99981"
+   version="1.1"
+   id="svg12"
+   sodipodi:docname="tubesync-coloured.svg"
+   inkscape:version="0.92.3 (2405546, 2018-03-11)">
+  <metadata
+     id="metadata18">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs16" />
+  <sodipodi:namedview
+     pagecolor="#e71d36"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="2560"
+     inkscape:window-height="1377"
+     id="namedview14"
+     showgrid="false"
+     inkscape:zoom="3.6875"
+     inkscape:cx="37.841732"
+     inkscape:cy="44.580651"
+     inkscape:window-x="0"
+     inkscape:window-y="30"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="g8"
+     inkscape:document-units="mm"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0" />
+  <g
+     transform="translate(1.6901452e-4,-168.99998)"
+     id="g10">
+    <g
+       transform="matrix(0.26458333,0,0,0.26458333,-178.7695,-21.269183)"
+       id="g8">
+      <path
+         class="logo-icon"
+         style="fill:#fdfffc;fill-opacity:1"
+         d="M 870.96867,851.62418 C 851.49721,840.45108 835.711,849.60421 835.711,872.05194 v 165.65436 c 0,22.4701 15.78621,31.6114 35.25767,20.4488 L 1015.707,975.11897 c 19.4781,-11.17706 19.4781,-29.28555 0,-40.45997 z"
+         id="path2"
+         inkscape:connector-curvature="0" />
+      <path
+         class="logo-left-arrow"
+         style="fill:#011627"
+         d="m 915.5666,719.13545 c -49.31563,0.40129 -97.71637,16.0713 -138.32422,44.93359 -46.94068,33.30533 -80.46025,82.61479 -94.34374,138.39844 -16.42739,66.00354 -4.54978,134.53452 32.26367,190.96092 l -35.81445,42.1816 150.73242,27.3672 -51.84571,-144.0586 -33.3164,39.4121 c -24.77897,-43.9323 -31.90369,-95.22673 -19.50391,-145.04876 11.39316,-45.77667 38.85626,-86.19287 77.29102,-113.47657 37.9033,-26.96389 84.27909,-39.83411 130.61718,-36.20703 l 3.41602,-43.69336 c -7.06243,-0.57189 -14.12679,-0.82686 -21.17188,-0.76953 z"
+         id="path4"
+         inkscape:connector-curvature="0" />
+      <path
+         class="logo-right-arrow"
+         style="fill:#011627"
+         d="m 1004.9729,759.05928 51.8476,144.05859 33.3165,-39.41211 c 24.7789,43.93234 31.904,95.22486 19.5039,145.04684 -11.3931,45.7767 -38.8579,86.1948 -77.293,113.4785 -37.90331,26.9639 -84.27746,39.8322 -130.61528,36.2051 l -3.41602,43.6934 c 56.4994,4.5751 113.0853,-11.1767 159.4942,-44.1622 46.9407,-33.3053 80.4616,-82.6166 94.3984,-138.6132 16.4271,-66.00353 4.5498,-134.53452 -32.2637,-190.96094 l 35.8145,-42.18164 z"
+         id="path6"
+         inkscape:connector-curvature="0" />
+    </g>
+  </g>
+</svg>

tubesync/sync/models.py

@@ -5,6 +5,7 @@ from datetime import datetime
 from pathlib import Path
 from django.conf import settings
 from django.db import models
+from django.core.files.storage import FileSystemStorage
 from django.utils.text import slugify
 from django.utils.translation import gettext_lazy as _
 from common.errors import NoFormatException
@@ -15,6 +16,9 @@ from .matching import (get_best_combined_format, get_best_audio_format,
                        get_best_video_format)
 
 
+media_file_storage = FileSystemStorage(location=settings.DOWNLOAD_ROOT)
+
+
 class Source(models.Model):
     '''
         A Source is a source of media. Currently, this is either a YouTube channel
@@ -295,10 +299,11 @@ class Source(models.Model):
 
     @property
     def directory_path(self):
+        download_dir = Path(media_file_storage.location)
         if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO:
-            return settings.SYNC_AUDIO_ROOT / self.directory
+            return download_dir / settings.DOWNLOAD_AUDIO_DIR / self.directory
         else:
-            return settings.SYNC_VIDEO_ROOT / self.directory
+            return download_dir / settings.DOWNLOAD_VIDEO_DIR / self.directory
 
     def make_directory(self):
         return os.makedirs(self.directory_path, exist_ok=True)
@@ -483,6 +488,7 @@ class Media(models.Model):
         max_length=200,
         blank=True,
         null=True,
+        storage=media_file_storage,
         help_text=_('Media file')
     )
     downloaded = models.BooleanField(
@@ -656,26 +662,29 @@ class Media(models.Model):
 
     @property
     def filename(self):
+        if self.media_file:
+            return os.path.basename(self.media_file.name)
         upload_date = self.upload_date
         dateobj = upload_date if upload_date else self.created
         datestr = dateobj.strftime('%Y-%m-%d')
-        source_name = slugify(self.source.name)
+        source_name = slugify(self.source.name).replace('_', '-')
-        name = slugify(self.name.replace('&', 'and').replace('+', 'and'))[:50]
+        name = slugify(self.name.replace('&', 'and').replace('+', 'and'))
-        key = self.key.strip()
+        name = name.replace('_', '-')[:80]
-        fmt = self.source.source_resolution.lower()
+        key = self.key.strip().replace('_', '-')[:20]
-        if self.source.is_audio():
+        fmt = []
-            codecs = self.source.source_acodec.lower()
+        if self.source.is_audio:
+            fmt.append(self.source.source_acodec.lower())
         else:
-            codecs = []
+            fmt.append(self.source.source_resolution.lower())
-            vcodec = self.source.source_vcodec.lower()
+            fmt.append(self.source.source_vcodec.lower())
-            acodec = self.source.source_acodec.lower()
+            fmt.append(self.source.source_acodec.lower())
-            if vcodec:
+        if self.source.prefer_60fps:
-                codecs.append(vcodec)
+            fmt.append('60fps')
-            if acodec:
+        if self.source.prefer_hdr:
-                codecs.append(acodec)
+            fmt.append('hdr')
-            codecs = '-'.join(codecs)
+        fmt = '-'.join(fmt)
         ext = self.source.extension
-        return f'{datestr}_{source_name}_{name}_{key}-{fmt}-{codecs}.{ext}'
+        return f'{datestr}_{source_name}_{name}_{key}_{fmt}.{ext}'
 
     @property
     def filepath(self):

tubesync/sync/templates/sync/media-item.html

@@ -32,9 +32,13 @@
   <div class="col s12">
     <div class="collection">
       <span class="collection-item">
+        {% if task.locked_by_pid_running %}
+        <i class="fas fa-running"></i> <strong>{{ task }}</strong><br>
+        <i class="far fa-clock"></i> Task started at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>
+        {% else %}
         <i class="fas fa-stopwatch"></i> <strong>{{ task }}</strong><br>
-        {% if task.instance.index_schedule %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
         <i class="fas fa-redo"></i> Task will run {% if task.run_now %}<strong>immediately</strong>{% else %}at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>{% endif %}
+        {% endif %}
       </span>
     </div>
   </div>
@@ -80,6 +84,7 @@
         <td class="hide-on-small-only">Container</td>
         <td><span class="hide-on-med-and-up">Container<br></span><strong>{{ media.downloaded_container|upper }}</strong></td>
       </tr>
+      {% if media.downloaded_video_codec %}
       <tr title="Frames per second in the downloaded file">
         <td class="hide-on-small-only">Downloaded FPS</td>
         <td><span class="hide-on-med-and-up">Downloaded FPS<br></span><strong>{{ media.downloaded_fps }} FPS</strong></td>
@@ -88,6 +93,7 @@
         <td class="hide-on-small-only">Downloaded HDR?</td>
         <td><span class="hide-on-med-and-up">Downloaded HDR?<br></span><strong>{% if media.downloaded_hdr %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
+      {% endif %}
       {% else %}
       <tr title="Can the media be downloaded?">
         <td class="hide-on-small-only">Can download?</td>
@@ -98,12 +104,12 @@
         <td class="hide-on-small-only">Available formats</td>
         <td><span class="hide-on-med-and-up">Available formats<br></span>
           {% for format in media.formats %}
-          <span class="truncate">
+          <div>
             ID: <strong>{{ format.format_id }}</strong>
             {% if format.vcodec|lower != 'none' %}, <strong>{{ format.format_note }} ({{ format.width }}x{{ format.height }})</strong>, fps:<strong>{{ format.fps|lower }}</strong>, video:<strong>{{ format.vcodec }} @{{ format.tbr }}k</strong>{% endif %}
             {% if format.acodec|lower != 'none' %}, audio:<strong>{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz</strong>{% endif %}
             {% if format.format_id == combined_format or format.format_id == audio_format or format.format_id == video_format %}<strong>(matched)</strong>{% endif %}
-          </span>
+          </div>
           {% empty %}
           Media has no indexed available formats
           {% endfor %}

tubesync/sync/utils.py

@@ -96,10 +96,8 @@ def file_is_editable(filepath):
     allowed_paths = (
         # Media item thumbnails
         os.path.commonpath([os.path.abspath(str(settings.MEDIA_ROOT))]),
-        # Downloaded video files
+        # Downloaded media files
-        os.path.commonpath([os.path.abspath(str(settings.SYNC_VIDEO_ROOT))]),
+        os.path.commonpath([os.path.abspath(str(settings.DOWNLOAD_ROOT))]),
-        # Downloaded audio files
-        os.path.commonpath([os.path.abspath(str(settings.SYNC_AUDIO_ROOT))]),
     )
     filepath = os.path.abspath(str(filepath))
     if not os.path.isfile(filepath):

tubesync/sync/youtube.py

@@ -4,6 +4,7 @@
 '''
 
 
+import os
 from django.conf import settings
 from copy import copy
 from common.logger import log
@@ -11,7 +12,6 @@ import youtube_dl
 
 
 _defaults = getattr(settings, 'YOUTUBE_DEFAULTS', {})
-_defaults.update({'logger': log})
 
 
 class YouTubeError(youtube_dl.utils.DownloadError):
@@ -32,6 +32,7 @@ def get_media_info(url):
         'skip_download': True,
         'forcejson': True,
         'simulate': True,
+        'logger': log
     })
     response = {}
     with youtube_dl.YoutubeDL(opts) as y:
@@ -46,12 +47,37 @@ def download_media(url, media_format, extension, output_file):
     '''
         Downloads a YouTube URL to a file on disk.
     '''
+
+    def hook(event):
+        filename = os.path.basename(event['filename'])
+        if event['status'] == 'error':
+            log.error(f'[youtube-dl] error occured downloading: {filename}')
+        elif event['status'] == 'downloading':
+            p = round((event['downloaded_bytes'] / event['total_bytes']) * 100, -1)
+            if p > hook.download_progress:
+                hook.download_progress = p
+                eta = event.get('_eta_str', '?').strip()
+                percent_done = event.get('_percent_str', '?').strip()
+                speed = event.get('_speed_str', '?').strip()
+                total = event.get('_total_bytes_str', '?').strip()
+                log.info(f'[youtube-dl] downloading: {filename} - {percent_done} of '
+                         f'{total} at {speed}, {eta} remaining')
+        elif event['status'] == 'finished':
+            total_size_str = event.get('_total_bytes_str', '?').strip()
+            elapsed_str = event.get('_elapsed_str', '?').strip()
+            log.info(f'[youtube-dl] finished downloading: {filename} - '
+                     f'{total_size_str} in {elapsed_str}')
+        else:
+            log.warn(f'[youtube-dl] unknown event: {str(event)}')
+    hook.download_progress = 0
+
     opts = copy(_defaults)
     opts.update({
         'format': media_format,
         'merge_output_format': extension,
         'outtmpl': output_file,
         'quiet': True,
+        'progress_hooks': [hook],
     })
     with youtube_dl.YoutubeDL(opts) as y:
         try:

tubesync/tubesync/local_settings.py.container

@@ -25,5 +25,4 @@ BACKGROUND_TASK_ASYNC_THREADS = int(os.get('TUBESYNC_WORKERS', 2))
 
 
 MEDIA_ROOT = ROOT_DIR / 'config' / 'media'
-SYNC_VIDEO_ROOT = ROOT_DIR / 'downloads' / 'video'
+DOWNLOAD_ROOT = ROOT_DIR / 'downloads'
-SYNC_AUDIO_ROOT = ROOT_DIR / 'downloads' / 'audio'

tubesync/tubesync/settings.py

@@ -98,8 +98,9 @@ STATIC_URL = '/static/'
 STATIC_ROOT = BASE_DIR / 'static'
 #MEDIA_URL = '/media/'
 MEDIA_ROOT = BASE_DIR / 'media'
-SYNC_VIDEO_ROOT = BASE_DIR / 'downloads' / 'video'
+DOWNLOAD_ROOT = BASE_DIR / 'downloads'
-SYNC_AUDIO_ROOT = BASE_DIR / 'downloads' / 'audio'
+DOWNLOAD_VIDEO_DIR = 'video'
+DOWNLOAD_AUDIO_DIR = 'audio'
 SASS_PROCESSOR_ROOT = STATIC_ROOT
 
 
@@ -115,9 +116,9 @@ HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',)
 
 MAX_ATTEMPTS = 10                           # Number of times tasks will be retried
 MAX_RUN_TIME = 1800                         # Maximum amount of time in seconds a task can run
-BACKGROUND_TASK_RUN_ASYNC = True            # Run tasks async in the background
+BACKGROUND_TASK_RUN_ASYNC = False           # Run tasks async in the background
-BACKGROUND_TASK_ASYNC_THREADS = 2           # Number of async tasks to run at once
+BACKGROUND_TASK_ASYNC_THREADS = 1           # Number of async tasks to run at once
-BACKGROUND_TASK_PRIORITY_ORDERING = 'ASC'  # Use 'niceness' task priority ordering
+BACKGROUND_TASK_PRIORITY_ORDERING = 'ASC'   # Use 'niceness' task priority ordering
 COMPLETED_TASKS_DAYS_TO_KEEP = 30           # Number of days to keep completed tasks