move TIME_ZONE set by env var from local_settings to settings, resolves #462

Following Delete files on disk #426

.idea/.gitignore

@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Editor-based HTTP Client requests
-/httpRequests/
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/tubesync.iml" filepath="$PROJECT_DIR$/.idea/tubesync.iml" />
-    </modules>
-  </component>
-</project>

.idea/tubesync.iml

@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-  <component name="PyDocumentationSettings">
-    <option name="format" value="PLAIN" />
-    <option name="myDocStringFormat" value="Plain" />
-  </component>
-  <component name="TemplatesService">
-    <option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
-    <option name="TEMPLATE_FOLDERS">
-      <list>
-        <option value="$MODULE_DIR$/tubesync/common/templates" />
-      </list>
-    </option>
-  </component>
-</module>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="" vcs="Git" />
-  </component>
-</project>

docs/other-database-backends.md

@@ -79,6 +79,14 @@ entry in the container or stdout logs:
 If you see a line similar to the above and the web interface loads, congratulations,
 you are now using an external database server for your TubeSync data!
 
+## Database Compression (For MariaDB)
+With a lot of media files the `sync_media` table grows in size quickly.
+You can save space using column compression using the following steps while using MariaDB:
+
+ 1. Stop tubesync 
+ 2. Execute `ALTER TABLE sync_media MODIFY metadata LONGTEXT COMPRESSED;` on database tubesync
+ 3. Start tunesync and confirm the connection still works.
+
 ## Docker Compose
 
 If you're using Docker Compose and simply want to connect to another container with
@@ -118,6 +126,7 @@ database before it can be written to. This file should contain:
 CREATE DATABASE tubesync;
 ```
 
+
 Then it must be mapped to `/docker-entrypoint-initdb.d/init.sql` for it
 to be executed on first startup of the container. See the `tubesync-db`
 volume mapping above for how to do this.

tubesync/sync/migrations/0021_add_delete_files_on_disk.py

@@ -0,0 +1,17 @@
+# Generated by pac
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0020_auto_20231024_1825'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='delete_files_on_disk',
+            field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'),
+        ),
+    ]

tubesync/sync/migrations/0021_add_lightweight_metadata.py

@@ -1,22 +0,0 @@
-# Generated by pac
-
-from django.db import migrations, models
-from sync.models import Source
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ('sync', '0020_auto_20231024_1825'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='source',
-            name='lightweight_metadata',
-            field=models.CharField(max_length=20,
-                                   default=Source.LIGHTWEIGHT_METADATA_TYPE_RAW,
-                                   choices=Source.LIGHTWEIGHT_METADATA_TYPE_CHOICES,
-                                   help_text='Lightweight metadata',
-                                   verbose_name='lightweight metadata'),
-        ),
-    ]

tubesync/sync/models.py

@@ -107,7 +107,6 @@ class Source(models.Model):
     EXTENSION_MKV = 'mkv'
     EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
 
-
     # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
     SPONSORBLOCK_CATEGORIES_CHOICES = (
         ('sponsor', 'Sponsor'),
@@ -121,15 +120,14 @@ class Source(models.Model):
     )
 
     sponsorblock_categories = CommaSepChoiceField(
-            _(''),
+        _(''),
-            possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
+        possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
-            all_choice="all",
+        all_choice='all',
-            allow_all=True,
+        allow_all=True,
-            all_label="(all options)",
+        all_label='(all options)',
-            default="all",
+        default='all',
-            help_text=_("Select the sponsorblocks you want to enforce")
+        help_text=_('Select the sponsorblocks you want to enforce')
-        )
+    )
-
     embed_metadata = models.BooleanField(
         _('embed metadata'),
         default=False,
@@ -140,14 +138,12 @@ class Source(models.Model):
         default=False,
         help_text=_('Embed thumbnail into the file')
     )
-
     enable_sponsorblock = models.BooleanField(
         _('enable sponsorblock'),
         default=True,
         help_text=_('Use SponsorBlock?')
     )
 
-
     # Fontawesome icons used for the source on the front end
     ICONS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
@@ -300,6 +296,11 @@ class Source(models.Model):
         default=False,
         help_text=_('Delete media that is no longer on this playlist')
     )
+    delete_files_on_disk = models.BooleanField(
+        _('delete files on disk'),
+        default=False,
+        help_text=_('Delete files on disk when they are removed from TubeSync')
+    )
     source_resolution = models.CharField(
         _('source resolution'),
         max_length=8,
@@ -387,24 +388,6 @@ class Source(models.Model):
         ]
     )
 
-    LIGHTWEIGHT_METADATA_TYPE_RAW = 'RAW'
-    LIGHTWEIGHT_METADATA_TYPE_UNNECESSARY = 'UNNECESSARY'
-    LIGHTWEIGHT_METADATA_TYPE_FEATHER = 'FEATHER'
-    LIGHTWEIGHT_METADATA_TYPES = (LIGHTWEIGHT_METADATA_TYPE_RAW, LIGHTWEIGHT_METADATA_TYPE_UNNECESSARY, LIGHTWEIGHT_METADATA_TYPE_FEATHER)
-    LIGHTWEIGHT_METADATA_TYPE_CHOICES = (
-        (LIGHTWEIGHT_METADATA_TYPE_RAW, _("(LARGE) Save raw metadata")),
-        (LIGHTWEIGHT_METADATA_TYPE_UNNECESSARY, _("(MEDIUM) Treeshake unnecessary metadata json keys")),
-        (LIGHTWEIGHT_METADATA_TYPE_FEATHER, _("(TINY) if the capacity is large, Treeshake it event if it is in use")),
-    )
-
-    lightweight_metadata = models.CharField(
-        _('lightweight metadata'),
-        max_length=20,
-        default=LIGHTWEIGHT_METADATA_TYPE_RAW,
-        choices=LIGHTWEIGHT_METADATA_TYPE_CHOICES,
-        help_text=_('Lightweight metadata')
-    )
-
     def __str__(self):
         return self.name
 
@@ -1254,7 +1237,6 @@ class Media(models.Model):
             acodec = self.downloaded_audio_codec
             if acodec is None:
                 raise TypeError() # nothing here.
-            
             acodec = acodec.lower()
             if acodec == "mp4a":
                 return "audio/mp4"
@@ -1263,7 +1245,6 @@ class Media(models.Model):
             else:
                 # fall-fall-back.
                 return 'audio/ogg'
-
         vcodec = vcodec.lower()
         if vcodec == 'vp9':
             return 'video/webm'
@@ -1287,6 +1268,22 @@ class Media(models.Model):
         showtitle.text = str(self.source.name).strip()
         showtitle.tail = '\n  '
         nfo.append(showtitle)
+        # season = upload date year
+        season = nfo.makeelement('season', {})
+        if self.source.source_type == Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
+            # If it's a playlist, set season to 1
+            season.text = '1'
+        else:
+            # If it's not a playlist, set season to upload date year
+            season.text = str(self.upload_date.year) if self.upload_date else ''
+        season.tail = '\n  '
+        nfo.append(season)
+        # episode = number of video in the year
+        episode = nfo.makeelement('episode', {})
+        episode_number = self.calculate_episode_number()
+        episode.text = str(episode_number) if episode_number else ''
+        episode.tail = '\n  '
+        nfo.append(episode)
         # ratings = media metadata youtube rating
         value = nfo.makeelement('value', {})
         value.text = str(self.rating)
@@ -1395,7 +1392,7 @@ class Media(models.Model):
         # Download the media with youtube-dl
         download_youtube_media(self.url, format_str, self.source.extension,
                                str(self.filepath), self.source.write_json,
-                               self.source.sponsorblock_categories, self.source.embed_thumbnail,
+                               self.source.sponsorblock_categories.selected_choices, self.source.embed_thumbnail,
                                self.source.embed_metadata, self.source.enable_sponsorblock,
                               self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs )
         # Return the download paramaters
@@ -1411,6 +1408,19 @@ class Media(models.Model):
                             f'has no indexer')
         return indexer(self.url)
 
+    def calculate_episode_number(self):
+        if self.source.source_type == Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
+            sorted_media = Media.objects.filter(source=self.source)
+        else:
+            self_year = self.upload_date.year if self.upload_date else self.created.year
+            filtered_media = Media.objects.filter(source=self.source, published__year=self_year)
+            sorted_media = sorted(filtered_media, key=lambda x: (x.upload_date, x.key))
+        position_counter = 1
+        for media in sorted_media:
+            if media == self:
+                return position_counter
+            position_counter += 1
+
 
 class MediaServer(models.Model):
     '''

tubesync/sync/signals.py

@@ -1,4 +1,5 @@
 import os
+import glob
 from django.conf import settings
 from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
 from django.dispatch import receiver
@@ -74,6 +75,7 @@ def source_pre_delete(sender, instance, **kwargs):
         media.delete()
 
 
+
 @receiver(post_delete, sender=Source)
 def source_post_delete(sender, instance, **kwargs):
     # Triggered after a source is deleted
@@ -222,6 +224,16 @@ def media_pre_delete(sender, instance, **kwargs):
     if thumbnail_url:
         delete_task_by_media('sync.tasks.download_media_thumbnail',
                              (str(instance.pk), thumbnail_url))
+    if instance.source.delete_files_on_disk and (instance.media_file or instance.thumb):
+        # Delete all media files if it contains filename
+        filepath = instance.media_file.path if instance.media_file else instance.thumb.path
+        barefilepath, fileext = os.path.splitext(filepath)
+        # Get all files that start with the bare file path
+        all_related_files = glob.glob(f'{barefilepath}.*')
+        for file in all_related_files:
+            log.info(f'Deleting file for: {instance} path: {file}')
+            delete_file(file)
+
 
 
 @receiver(post_delete, sender=Media)

tubesync/sync/tasks.py

@@ -236,12 +236,6 @@ def download_media_metadata(media_id):
         return
     source = media.source
     metadata = media.index_metadata()
-    if source.lightweight_metadata == Source.LIGHTWEIGHT_METADATA_TYPE_FEATHER:
-        del metadata["formats"]
-        del metadata["thumbnails"]
-        del metadata["automatic_captions"]
-        del metadata["requested_formats"]
-        del metadata["heatmap"]
     media.metadata = json.dumps(metadata, default=json_serial)
     upload_date = media.upload_date
     # Media must have a valid upload date

tubesync/sync/templates/sync/media-item.html

@@ -132,8 +132,6 @@
         <td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if media.can_download %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
       {% endif %}
-
-    {%  if media.source.lightweight_metadata == "RAW" %}
       <tr title="The available media formats">
         <td class="hide-on-small-only">Available formats</td>
         <td><span class="hide-on-med-and-up">Available formats<br></span>
@@ -157,10 +155,7 @@
           Video: <strong>{% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}
         </strong></td>
       </tr>
-    {% endif %}
     </table>
-  <p>{{ media.source.lightweight_metadata }}</p>
-  <p>{{ media.source }}</p>
   </div>
 </div>
 {% if media.downloaded %}

tubesync/sync/templates/sync/source-delete.html

@@ -9,8 +9,8 @@
     <p>
       Are you sure you want to delete this source? Deleting a source is permanent.
       By default, deleting a source does not delete any saved media files. You can
-      tick the &quot;also delete downloaded media&quot; checkbox to also remove save
+      <strong>tick the &quot;also delete downloaded media&quot; checkbox to also remove directory {{ source.directory_path }}
-      media when you delete the source. Deleting a source cannot be undone.
+	  </strong>when you delete the source. Deleting a source cannot be undone.
     </p>
   </div>
 </div>

tubesync/sync/templates/sync/source.html

@@ -122,6 +122,10 @@
       <tr title="Delete media that is no longer on this playlist?">
         <td class="hide-on-small-only">Delete removed media</td>
         <td><span class="hide-on-med-and-up">Delete removed media<br></span><strong>{% if source.delete_removed_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
+        <tr title="Delete files on disk when they are removed from TubeSync?">
+        <td class="hide-on-small-only">Delete files on disk</td>
+        <td><span class="hide-on-med-and-up">Delete files on disk<br></span><strong>{% if source.delete_files_on_disk %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
       {% if source.delete_old_media and source.days_to_keep > 0 %}
       <tr title="Days after which your media from this source will be locally deleted">
@@ -187,16 +191,6 @@
       </tr>
       {% endif %}
       
-      {% if source.lightweight_metadata %}
-      <tr title="{{ _('Are auto subs accepted?') }}">
-        <td class="hide-on-small-only">{{ _("Auto-generated subtitles?") }}:</td>
-        <td><span class="hide-on-med-and-up">{{ _("Auto-generated subtitles?") }}:</span><strong><i class="fas {% if source.auto_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
-      </tr>
-      <tr title="{{ _('Subs langs?') }}">
-        <td class="hide-on-small-only">{{ _("Subs langs?") }}:</td>
-        <td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
-      </tr>
-      {% endif %}
     </table>
   </div>
 </div>

tubesync/sync/tests.py

@@ -661,6 +661,8 @@ class MediaTestCase(TestCase):
             '<episodedetails>',
             '  <title>no fancy stuff title</title>',
             '  <showtitle>testname</showtitle>',
+            '  <season>2017</season>',
+            '  <episode></episode>',
             '  <ratings>',
             '    <rating default="True" max="5" name="youtube">',
             '      <value>1.2345</value>',

tubesync/sync/views.py

@@ -1,7 +1,9 @@
+import glob
 import os
 import json
 from base64 import b64decode
 import pathlib
+import shutil
 import sys
 from django.conf import settings
 from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect
@@ -59,7 +61,7 @@ class DashboardView(TemplateView):
         # Disk usage
         disk_usage = Media.objects.filter(
             downloaded=True, downloaded_filesize__isnull=False
-        ).aggregate(Sum('downloaded_filesize'))
+        ).defer('metadata').aggregate(Sum('downloaded_filesize'))
         data['disk_usage_bytes'] = disk_usage['downloaded_filesize__sum']
         if not data['disk_usage_bytes']:
             data['disk_usage_bytes'] = 0
@@ -71,11 +73,11 @@ class DashboardView(TemplateView):
         # Latest downloads
         data['latest_downloads'] = Media.objects.filter(
             downloaded=True, downloaded_filesize__isnull=False
-        ).order_by('-download_date')[:10]
+        ).defer('metadata').order_by('-download_date')[:10]
         # Largest downloads
         data['largest_downloads'] = Media.objects.filter(
             downloaded=True, downloaded_filesize__isnull=False
-        ).order_by('-downloaded_filesize')[:10]
+        ).defer('metadata').order_by('-downloaded_filesize')[:10]
         # UID and GID
         data['uid'] = os.getuid()
         data['gid'] = os.getgid()
@@ -296,11 +298,11 @@ class EditSourceMixin:
     model = Source
     fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format',
               'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
-              'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
+              'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution',
-              'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails',
+              'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback',
-              'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
+              'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
               'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
-              'auto_subtitles', 'sub_langs', 'lightweight_metadata')
+              'auto_subtitles', 'sub_langs')
     errors = {
         'invalid_media_format': _('Invalid media format, the media format contains '
                                   'errors or is empty. Check the table at the end of '
@@ -404,7 +406,7 @@ class SourceView(DetailView):
             error_message = get_error_message(error)
             setattr(error, 'error_message', error_message)
             data['errors'].append(error)
-        data['media'] = Media.objects.filter(source=self.object).order_by('-published')
+        data['media'] = Media.objects.filter(source=self.object).order_by('-published').defer('metadata')
         return data
 
 
@@ -435,14 +437,13 @@ class DeleteSourceView(DeleteView, FormMixin):
             source = self.get_object()
             for media in Media.objects.filter(source=source):
                 if media.media_file:
-                    # Delete the media file
+                    file_path = media.media_file.path
-                    delete_file(media.media_file.path)
+                    matching_files = glob.glob(os.path.splitext(file_path)[0] + '.*')
-                    # Delete thumbnail copy if it exists
+                    for file in matching_files:
-                    delete_file(media.thumbpath)
+                        delete_file(file)
-                    # Delete NFO file if it exists
+            directory_path = source.directory_path
-                    delete_file(media.nfopath)
+            if os.path.exists(directory_path):
-                    # Delete JSON file if it exists
+                shutil.rmtree(directory_path, True)
-                    delete_file(media.jsonpath)
         return super().post(request, *args, **kwargs)
 
     def get_success_url(self):
@@ -653,12 +654,13 @@ class MediaSkipView(FormView, SingleObjectMixin):
         delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),))
         # If the media file exists on disk, delete it
         if self.object.media_file_exists:
-            delete_file(self.object.media_file.path)
+            # Delete all files which contains filename
-            self.object.media_file = None
+            filepath = self.object.media_file.path
-            # If the media has an associated thumbnail copied, also delete it
+            barefilepath, fileext = os.path.splitext(filepath)
-            delete_file(self.object.thumbpath)
+            # Get all files that start with the bare file path
-            # If the media has an associated NFO file with it, also delete it
+            all_related_files = glob.glob(f'{barefilepath}.*')
-            delete_file(self.object.nfopath)
+            for file in all_related_files:
+                delete_file(file)
         # Reset all download data
         self.object.metadata = None
         self.object.downloaded = False

tubesync/sync/youtube.py

@@ -1,5 +1,5 @@
 '''
-    Wrapper for the youtube-dl library. Used so if there are any library interface
+    Wrapper for the yt-dlp library. Used so if there are any library interface
     updates we only need to udpate them in one place.
 '''
 
@@ -65,7 +65,7 @@ def get_media_info(url):
 
 
 def download_media(url, media_format, extension, output_file, info_json,
-                   sponsor_categories="all", 
+                   sponsor_categories=None,
                    embed_thumbnail=False, embed_metadata=False, skip_sponsors=True,
                    write_subtitles=False, auto_subtitles=False, sub_langs='en'):
     '''
@@ -106,8 +106,8 @@ def download_media(url, media_format, extension, output_file, info_json,
                      f'{total_size_str} in {elapsed_str}')
         else:
             log.warn(f'[youtube-dl] unknown event: {str(event)}')
-    hook.download_progress = 0
 
+    hook.download_progress = 0
     ytopts = {
         'format': media_format,
         'merge_output_format': extension,
@@ -120,27 +120,23 @@ def download_media(url, media_format, extension, output_file, info_json,
         'writeautomaticsub': auto_subtitles,
         'subtitleslangs': sub_langs.split(','),
     }
-    
+    if not sponsor_categories:
+        sponsor_categories = []
     sbopt = {
         'key': 'SponsorBlock',
-        'categories': [sponsor_categories]
+        'categories': sponsor_categories
     }
     ffmdopt = {
         'key': 'FFmpegMetadata',
-        'add_chapters': True,
+        'add_chapters': embed_metadata,
-        'add_metadata': True
+        'add_metadata': embed_metadata
     }
-
     opts = get_yt_opts()
     if embed_thumbnail:
         ytopts['postprocessors'].append({'key': 'EmbedThumbnail'})
-    if embed_metadata:
-        ffmdopt["add_metadata"] = True
     if skip_sponsors:
         ytopts['postprocessors'].append(sbopt)
-    
     ytopts['postprocessors'].append(ffmdopt)
-        
     opts.update(ytopts)
 
     with yt_dlp.YoutubeDL(opts) as y:

tubesync/tubesync/local_settings.py.container

@@ -25,9 +25,6 @@ DEBUG = True if os.getenv('TUBESYNC_DEBUG', False) else False
 FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX)
 
 
-TIME_ZONE = os.getenv('TZ', 'UTC')
-
-
 database_dict = {}
 database_connection_env = os.getenv('DATABASE_CONNECTION', '')
 if database_connection_env:

tubesync/tubesync/settings.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 
 
@@ -96,7 +97,7 @@ AUTH_PASSWORD_VALIDATORS = [
 
 
 LANGUAGE_CODE = 'en-us'
-TIME_ZONE = 'UTC'
+TIME_ZONE = os.getenv('TZ', 'UTC')
 USE_I18N = True
 USE_L10N = True
 USE_TZ = True