bump to 0.12.1, resolves #340 and #341

Adding new "manual_skip" field; adapt UI

.github/workflows/ci.yaml

@@ -7,9 +7,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    branches:
-      - main
 
 jobs:
   test:

.gitignore

@@ -130,3 +130,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+Pipfile.lock

Dockerfile

@@ -10,7 +10,8 @@ ENV DEBIAN_FRONTEND="noninteractive" \
   LANGUAGE="en_US.UTF-8" \
   LANG="en_US.UTF-8" \
   LC_ALL="en_US.UTF-8" \
-  TERM="xterm"
+  TERM="xterm" \
+  S6_CMD_WAIT_FOR_SERVICES_MAXTIME="0"
 
 # Install third party software
 RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \

README.md

@@ -350,6 +350,10 @@ etc.). Configuration of this is beyond the scope of this README.
 
 Just `amd64` for the moment. Others may be made available if there is demand.
 
+### The pipenv install fails with "Locking failed"!
+
+Make sure that you have `mysql_config` or `mariadb_config` available, as required by the python module `mysqlclient`. On Debian-based systems this is usually found in the package `libmysqlclient-dev`
+
 
 # Advanced configuration
 

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/timeout-up

@@ -1 +0,0 @@
-60000

tubesync/common/static/styles/fontawesome/_icons.scss

@@ -65,6 +65,7 @@ readers do not read off random characters that represent icons */
 .#{$fa-css-prefix}-arrows-alt-h:before { content: fa-content($fa-var-arrows-alt-h); }
 .#{$fa-css-prefix}-arrows-alt-v:before { content: fa-content($fa-var-arrows-alt-v); }
 .#{$fa-css-prefix}-artstation:before { content: fa-content($fa-var-artstation); }
+.#{$fa-css-prefix}-arrow-rotate-right:before { content: fa-content($fa-var-arrow-rotate-right); }
 .#{$fa-css-prefix}-assistive-listening-systems:before { content: fa-content($fa-var-assistive-listening-systems); }
 .#{$fa-css-prefix}-asterisk:before { content: fa-content($fa-var-asterisk); }
 .#{$fa-css-prefix}-asymmetrik:before { content: fa-content($fa-var-asymmetrik); }

tubesync/common/static/styles/fontawesome/_variables.scss

@@ -80,6 +80,7 @@ $fa-var-arrow-right: \f061;
 $fa-var-arrow-up: \f062;
 $fa-var-arrows-alt: \f0b2;
 $fa-var-arrows-alt-h: \f337;
+$fa-var-arrow-rotate-right: \f01e;
 $fa-var-arrows-alt-v: \f338;
 $fa-var-artstation: \f77a;
 $fa-var-assistive-listening-systems: \f2a2;

tubesync/common/static/styles/materializecss/components/forms/_checkboxes.scss

@@ -14,7 +14,7 @@
   // Text Label Style
   + span:not(.lever) {
     position: relative;
-    padding-left: 35px;
+    padding-left: 27px;
     cursor: pointer;
     display: inline-block;
     height: 25px;

tubesync/common/static/styles/tubesync.scss

@@ -17,3 +17,16 @@ html {
     visibility: visible;
     opacity: 1;
 }
+
+.flex-collection-container {
+    display: flex !important;
+    align-items: center;
+}
+
+.flex-grow {
+    flex-grow: 1;
+}
+
+.help-text > i {
+    padding-right: 6px;
+}

tubesync/sync/migrations/0015_auto_20230213_0603.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.17 on 2023-02-13 06:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0014_alter_media_media_file'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='media',
+            name='manual_skip',
+            field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\' be downloaded', verbose_name='manual_skip'),
+        ),
+        migrations.AlterField(
+            model_name='media',
+            name='skip',
+            field=models.BooleanField(db_index=True, default=False, help_text='INTERNAL FLAG - Media will be skipped and not downloaded', verbose_name='skip'),
+        ),
+    ]

tubesync/sync/models.py

@@ -679,7 +679,13 @@ class Media(models.Model):
         _('skip'),
         db_index=True,
         default=False,
-        help_text=_('Media will be skipped and not downloaded')
+        help_text=_('INTERNAL FLAG - Media will be skipped and not downloaded')
+    )
+    manual_skip = models.BooleanField(
+        _('manual_skip'),
+        db_index=True,
+        default=False,
+        help_text=_('Media marked as "skipped", won\' be downloaded')
     )
     downloaded = models.BooleanField(
         _('downloaded'),
@@ -1136,6 +1142,31 @@ class Media(models.Model):
             return False
         return os.path.exists(self.media_file.path)
 
+    @property
+    def content_type(self):
+        if not self.downloaded:
+            return 'video/mp4'
+        vcodec = self.downloaded_video_codec
+        if vcodec is None:
+            acodec = self.downloaded_audio_codec
+            if acodec is None:
+                raise TypeError() # nothing here.
+            
+            acodec = acodec.lower()
+            if acodec == "mp4a":
+                return "audio/mp4"
+            elif acodec == "opus":
+                return "audio/opus"
+            else:
+                # fall-fall-back.
+                return 'audio/ogg'
+
+        vcodec = vcodec.lower()
+        if vcodec == 'vp9':
+            return 'video/webm'
+        else:
+            return 'video/mp4'
+
     @property
     def nfoxml(self):
         '''

tubesync/sync/signals.py

@@ -93,6 +93,10 @@ def task_task_failed(sender, task_id, completed_task, **kwargs):
 
 @receiver(post_save, sender=Media)
 def media_post_save(sender, instance, created, **kwargs):
+    # If the media is skipped manually, bail.
+    if instance.manual_skip:
+        return
+
     # Triggered after media is saved
     cap_changed = False
     can_download_changed = False

tubesync/sync/tasks.py

@@ -153,7 +153,6 @@ def index_source_task(source_id):
         source = Source.objects.get(pk=source_id)
     except Source.DoesNotExist:
         # Task triggered but the Source has been deleted, delete the task
-        delete_index_source_task(source_id)
         return
     # Reset any errors
     source.has_failed = False
@@ -202,7 +201,6 @@ def check_source_directory_exists(source_id):
         source = Source.objects.get(pk=source_id)
     except Source.DoesNotExist:
         # Task triggered but the Source has been deleted, delete the task
-        delete_index_source_task(source_id)
         return
     # Check the source output directory exists
     if not source.directory_exists():
@@ -223,6 +221,11 @@ def download_media_metadata(media_id):
         log.error(f'Task download_media_metadata(pk={media_id}) called but no '
                   f'media exists with ID: {media_id}')
         return
+    
+    if media.manual_skip:
+        log.info(f'Task for ID: {media_id} skipped, due to task being manually skipped.')
+        return
+
     source = media.source
     metadata = media.index_metadata()
     media.metadata = json.dumps(metadata, default=json_serial)

tubesync/sync/templates/sync/media-item.html

@@ -10,15 +10,23 @@
     <p class="truncate"><strong><a href="{{ media.url }}" target="_blank"><i class="fas fa-link"></i> {{ media.url }}</a></strong></p>
     <p class="truncate">Downloading to: <strong>{{ media.source.directory_path }}</strong></p>
     {% if download_state == 'downloaded' %}
+    {% if media.source.is_audio %}
+    <audio controls src="{% url 'sync:media-content' pk=media.pk %}"></audio>
+    {% else %}
     <video controls style="width: 100%">
       <source src="{% url 'sync:media-content' pk=media.pk %}">
     </video>
-    <p class="truncate"><a href="{% url 'sync:media-content' pk=media.pk %}" download="{{ media.filename }}">Download</a></p>
+    {% endif %}
+
+    <p class="truncate"><a href="{% url 'sync:media-content' pk=media.pk %}" download="{{ media.filename }}"><strong><i class="fas fa-download"></i> Download</strong></a></p>
     {% endif %}
   </div>
 </div>
+{% if media.manual_skip %}{% include 'errorbox.html' with message='Media is marked to be skipped and will not be downloaded.' %}
+{% else %}
   {% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
-{% if media.skip %}{% include 'errorbox.html' with message='Media is marked to be skipped and will not be downloaded.' %}{% endif %}
+  {% if media.skip %}{% include 'errorbox.html' with message='This media may be skipped due to error(s).' %}{% endif %}
+{% endif %}
 {% include 'infobox.html' with message=message %}
 <div class="row">
   <div class="col s12 m7">
@@ -162,10 +170,10 @@
 {% else %}
 <div class="row">
   <div class="col s12">
-    {% if media.skip %}
+    {% if media.manual_skip %}
-      <a href="{% url 'sync:enable-media' pk=media.pk %}" class="btn">Enable (unskip) media <i class="fas fa-cloud-download-alt"></i></a>
+      <a href="{% url 'sync:enable-media' pk=media.pk %}" class="btn">Unskip media (manually) <i class="fas fa-cloud-download-alt"></i></a>
     {% else %}
-      <a href="{% url 'sync:skip-media' pk=media.pk %}" class="btn delete-button">Skip media <i class="fas fa-times-circle"></i></a>
+      <a href="{% url 'sync:skip-media' pk=media.pk %}" class="btn delete-button">Manually mark media to be skipped <i class="fas fa-times-circle"></i></a>
     {% endif %}
   </div>
 </div>

tubesync/sync/templates/sync/media.html

@@ -36,8 +36,10 @@
             {% if m.downloaded %}
               <i class="fas fa-check-circle" title="Downloaded"></i> {{ m.download_date|date:'Y-m-d' }}
             {% else %}
-              {% if m.skip %}
+              {% if m.manual_skip %}
-              <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped</span>
+              <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Manually skipped</span>
+              {% elif m.skip %}
+              <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped by system</span>
               {% elif not m.source.download_media %}
               <span class="error-text"><i class="fas fa-times" title="Not downloading media for this source"></i> Disabled at source</span>
               {% elif not m.has_metadata %}

tubesync/sync/templates/sync/sources.html

@@ -24,7 +24,8 @@
   <div class="col s12">
     <div class="collection">
     {% for source in sources %}
-      <a href="{% url 'sync:source' pk=source.pk %}" class="collection-item">
+      <span class="collection-item flex-collection-container">
+        <a href="{% url 'sync:source' pk=source.pk %}" class="flex-grow">
           {{ source.icon|safe }} <strong>{{ source.name }}</strong> ({{ source.get_source_type_display }} &quot;{{ source.key }}&quot;)<br>
           {{ source.format_summary }}<br>
           {% if source.has_failed %}
@@ -33,6 +34,8 @@
           <strong>{{ source.media_count }}</strong> media items, <strong>{{ source.downloaded_count }}</strong> downloaded{% if source.delete_old_media and source.days_to_keep > 0 %}, keeping {{ source.days_to_keep }} days of media{% endif %}
           {% endif %}
         </a>
+        <a href="{% url 'sync:source-sync-now' pk=source.pk %}" class="collection-item"><i class="fas fa-arrow-rotate-right"></i></a>
+      </span>
     {% empty %}
       <span class="collection-item no-items"><i class="fas fa-info-circle"></i> You haven't added any sources.</span>
     {% endfor %}

tubesync/sync/templates/sync/tasks.html

@@ -66,7 +66,7 @@
       {% for task in scheduled %}
         <a href="{% url task.url pk=task.instance.pk %}" class="collection-item">
           <i class="fas fa-stopwatch"></i> <strong>{{ task }}</strong><br>
-          {% if task.instance.index_schedule %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
+          {% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
           <i class="fas fa-redo"></i> Task will run {% if task.run_now %}<strong>immediately</strong>{% else %}at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>{% endif %}
         </a>
       {% empty %}

tubesync/sync/urls.py

@@ -28,6 +28,10 @@ urlpatterns = [
          ValidateSourceView.as_view(),
          name='validate-source'),
 
+    path('source-sync-now/<uuid:pk>',
+         SourcesView.as_view(),
+         name='source-sync-now'),
+
     path('source-add',
          AddSourceView.as_view(),
          name='add-source'),

tubesync/sync/views.py

@@ -1,18 +1,22 @@
 import os
 import json
 from base64 import b64decode
+import pathlib
+import sys
 from django.conf import settings
-from django.http import Http404
+from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect
 from django.views.generic import TemplateView, ListView, DetailView
 from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateView,
                                        DeleteView)
 from django.views.generic.detail import SingleObjectMixin
+from django.core.exceptions import SuspiciousFileOperation
 from django.http import HttpResponse
 from django.urls import reverse_lazy
 from django.db import IntegrityError
 from django.db.models import Q, Count, Sum, When, Case
-from django.forms import ValidationError
+from django.forms import Form, ValidationError
 from django.utils.text import slugify
+from django.utils._os import safe_join
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from common.utils import append_uri_params
@@ -92,8 +96,27 @@ class SourcesView(ListView):
     paginate_by = settings.SOURCES_PER_PAGE
     messages = {
         'source-deleted': _('Your selected source has been deleted.'),
+        'source-refreshed': _('The source has been scheduled to be synced now.')
     }
 
+    def get(self, *args, **kwargs):
+        if args[0].path.startswith("/source-sync-now/"):
+            sobj = Source.objects.get(pk=kwargs["pk"])
+            if sobj is None:
+                return HttpResponseNotFound()
+            
+            verbose_name = _('Index media from source "{}" once')
+            index_source_task(
+                str(sobj.pk),
+                queue=str(sobj.pk),
+                repeat=0,
+                verbose_name=verbose_name.format(sobj.name))
+            url = reverse_lazy('sync:sources')
+            url = append_uri_params(url, {'message': 'source-refreshed'})
+            return HttpResponseRedirect(url)
+        else:
+            return super().get(self, *args, **kwargs)    
+
     def __init__(self, *args, **kwargs):
         self.message = None
         super().__init__(*args, **kwargs)
@@ -279,20 +302,35 @@ class EditSourceMixin:
         'invalid_media_format': _('Invalid media format, the media format contains '
                                   'errors or is empty. Check the table at the end of '
                                   'this page for valid media name variables'),
+        'dir_outside_dlroot': _('You cannot specify a directory outside of the '
+                                'base directory (%BASEDIR%)')
     }
 
-    def form_valid(self, form):
+    def form_valid(self, form: Form):
         # Perform extra validation to make sure the media_format is valid
         obj = form.save(commit=False)
         source_type = form.cleaned_data['media_format']
         example_media_file = obj.get_example_media_format()
+        
         if example_media_file == '':
             form.add_error(
                 'media_format',
                 ValidationError(self.errors['invalid_media_format'])
             )
+
+        # Check for suspicious file path(s)
+        try:
+            targetCheck = form.cleaned_data['directory']+"/.virt"
+            newdir = safe_join(settings.DOWNLOAD_ROOT,targetCheck)
+        except SuspiciousFileOperation:
+            form.add_error(
+                'directory',
+                ValidationError(self.errors['dir_outside_dlroot'].replace("%BASEDIR%",str(settings.DOWNLOAD_ROOT)))
+            )
+
         if form.errors:
             return super().form_invalid(form)
+
         return super().form_valid(form)
 
 
@@ -448,16 +486,16 @@ class MediaView(ListView):
             if self.show_skipped:
                 q = Media.objects.filter(source=self.filter_source)
             elif self.only_skipped:
-                q = Media.objects.filter(source=self.filter_source, skip=True)
+                q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=True) | Q(manual_skip=True)))
             else:
-                q = Media.objects.filter(source=self.filter_source, skip=False)
+                q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=False) & Q(manual_skip=False)))
         else:
             if self.show_skipped:
                 q = Media.objects.all()
             elif self.only_skipped:
-                q = Media.objects.filter(skip=True)
+                q = Media.objects.filter(Q(skip=True)|Q(manual_skip=True))
             else:
-                q = Media.objects.filter(skip=False)
+                q = Media.objects.filter(Q(skip=False)&Q(manual_skip=False))
         return q.order_by('-published', '-created')
 
     def get_context_data(self, *args, **kwargs):
@@ -629,6 +667,7 @@ class MediaSkipView(FormView, SingleObjectMixin):
         self.object.downloaded_filesize = None
         # Mark it to be skipped
         self.object.skip = True
+        self.object.manual_skip = True
         self.object.save()
         return super().form_valid(form)
 
@@ -657,6 +696,7 @@ class MediaEnableView(FormView, SingleObjectMixin):
     def form_valid(self, form):
         # Mark it as not skipped
         self.object.skip = False
+        self.object.manual_skip = False
         self.object.save()
         return super().form_valid(form)
 
@@ -677,8 +717,35 @@ class MediaContent(DetailView):
 
     def dispatch(self, request, *args, **kwargs):
         self.object = self.get_object()
+        # development direct file stream - DO NOT USE PRODUCTIVLY
+        if settings.DEBUG and 'runserver' in sys.argv:
+            # get media URL
+            pth = self.object.media_file.url
+            # remove "/media-data/"
+            pth = pth.split("/media-data/",1)[1]
+            # remove "/" (incase of absolute path)
+            pth = pth.split(str(settings.DOWNLOAD_ROOT).lstrip("/"),1)
 
+            # if we do not have a "/" at the beginning, it is not a absolute path...
+            if len(pth) > 1:
+                pth = pth[1]
+            else:
+                pth = pth[0]
+            
+            
+            # build final path
+            filepth = pathlib.Path(str(settings.DOWNLOAD_ROOT) + pth)
+            
+            if filepth.exists():
+                # return file
+                response = FileResponse(open(filepth,'rb'))
+                return response
+            else:
+                return HttpResponseNotFound()
+            
+        else:
             headers = {
+                'Content-Type': self.object.content_type,
                 'X-Accel-Redirect': self.object.media_file.url,
             }
             return HttpResponse(headers=headers)

tubesync/sync/youtube.py

@@ -64,7 +64,7 @@ def get_media_info(url):
     return response
 
 
-def download_media(url, media_format, extension, output_file, info_json):
+def download_media(url, media_format, extension, output_file, info_json, sponsor_categories="all"):
     '''
         Downloads a YouTube URL to a file on disk.
     '''
@@ -100,7 +100,17 @@ def download_media(url, media_format, extension, output_file, info_json):
         else:
             log.warn(f'[youtube-dl] unknown event: {str(event)}')
     hook.download_progress = 0
-
+    postprocessors = []
+    postprocessors.append({
+        'key': 'FFmpegMetadata',
+        'add_chapters': True,
+        'add_metadata': True
+    })
+    # Pending configuration options from PR #338
+    #postprocessors.append({
+    #    'key': 'SponsorBlock',
+    #    'categories': [sponsor_categories]
+    #})
     opts = get_yt_opts()
     opts.update({
         'format': media_format,
@@ -108,7 +118,8 @@ def download_media(url, media_format, extension, output_file, info_json):
         'outtmpl': output_file,
         'quiet': True,
         'progress_hooks': [hook],
-        'writeinfojson': info_json
+        'writeinfojson': info_json,
+        'postprocessors': postprocessors,
     })
     with yt_dlp.YoutubeDL(opts) as y:
         try:

tubesync/tubesync/settings.py

@@ -6,7 +6,7 @@ CONFIG_BASE_DIR = BASE_DIR
 DOWNLOADS_BASE_DIR = BASE_DIR
 
 
-VERSION = '0.12.0'
+VERSION = '0.12.1'
 SECRET_KEY = ''
 DEBUG = False
 ALLOWED_HOSTS = []