Compare commits

..

25 Commits

Author SHA1 Message Date
meeb c6acd5378c move TIME_ZONE set by env var from local_settings to settings, resolves #462 2024-02-02 05:51:20 +11:00
meeb e7788eb8fb
Merge pull request #450 from InterN0te/main-delete-files-on-disk
Following Delete files on disk #426
2024-01-17 22:31:09 +11:00
meeb e4e0b48c0b
Merge pull request #460 from skayred/main
Optimized source page and dashboard loading
2024-01-17 22:29:59 +11:00
Dmitrii Savchenko 3573c1187f
Optimized source page and dashboard loading 2024-01-16 11:57:00 +02:00
meeb b11b667aff
Merge pull request #452 from ShaneBridges1234/patch-1
Update other-database-backends.md
2024-01-03 16:06:17 +11:00
Yottatron 1b581aa4ba
Update other-database-backends.md
Correct table new in SQL for MariaDB column compression.
2024-01-01 09:16:04 -08:00
meeb 7384c00713 fix typo in sponsorblock categories, remove dupe metadata flags, related to #362 2023-12-12 16:28:26 +11:00
meeb 4fdd172b05 tidy up and pass a serialised list through to youtube.download_media, may help with #362 2023-12-12 14:40:23 +11:00
Someone 9c18115032
Merge branch 'meeb:main' into main-delete-files-on-disk 2023-12-11 15:41:50 +01:00
meeb 6853c1fa76 fix tests 2023-12-11 13:42:33 +11:00
administrator ed07073cf4 Revert "Removed non-pertinent source option"
This reverts commit 46ba2593a2.

Restore option
2023-12-11 03:38:37 +01:00
administrator af94b37ee6 Revert "Removed non-pertinent source option"
This reverts commit ad1d49a835.

t cherry-pick 46a43b9

Restore option
2023-12-11 03:38:14 +01:00
administrator ad1d49a835 Removed non-pertinent source option 2023-12-11 03:25:17 +01:00
administrator 46ba2593a2 Removed non-pertinent source option 2023-12-11 03:19:43 +01:00
administrator 46a43b968a Rework delete method to delete all files matching filename
Remove Source folder if checkbox 'remove media' is checked
2023-12-11 02:29:57 +01:00
administrator 805a0eefbd Merge branch 'delete-files-on-disk' of https://github.com/sparklesmcfadden/tubesync into main-delete-files-on-disk
Merge from sparklesmcfadden:delete-files-on-disk
2023-12-11 00:19:37 +01:00
meeb 3a87b5779e
Merge pull request #448 from InterN0te/main-addNfoInfo
Added season and episode tags in NFO to get Jellyfin displaying correctly
2023-12-10 17:58:04 +11:00
Someone f86e72aa92
Optimization of episode calculation 2023-12-09 23:13:28 +01:00
Someone f550e32b5e
Fix secondary sorting on video key 2023-12-09 19:33:59 +01:00
Someone 034d877d6a
Add season and episode tags in NFO test 2023-12-09 17:56:32 +01:00
Someone b9b702ab85
Add season and episode tags in NFO 2023-12-09 17:55:17 +01:00
meeb c159c24d15
Merge pull request #443 from depuhitv/patch-1
compress sync_media table for mariadb
2023-12-04 18:05:02 +11:00
depuhitv 6c9772d573 fixed grammar 2023-12-04 16:26:46 +11:00
depuhitv 45b8b3f65b
compress sync_media table for mariadb
Added steps to compress sync_media table for mariadb.
For 1,608 records, I am seeing the size reduced from 642.8 MB to 55.8 MB
2023-12-04 16:18:10 +11:00
sparklesmcfadden 43cf532903 Adds option to delete files on disk when removing a media item from TubeSync 2023-10-21 20:54:21 -05:00
19 changed files with 136 additions and 177 deletions

8
.idea/.gitignore vendored
View File

@ -1,8 +0,0 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

View File

@ -1,6 +0,0 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

View File

@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/tubesync.iml" filepath="$PROJECT_DIR$/.idea/tubesync.iml" />
</modules>
</component>
</project>

View File

@ -1,20 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
</component>
<component name="TemplatesService">
<option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
<option name="TEMPLATE_FOLDERS">
<list>
<option value="$MODULE_DIR$/tubesync/common/templates" />
</list>
</option>
</component>
</module>

View File

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

View File

@ -79,6 +79,14 @@ entry in the container or stdout logs:
If you see a line similar to the above and the web interface loads, congratulations, If you see a line similar to the above and the web interface loads, congratulations,
you are now using an external database server for your TubeSync data! you are now using an external database server for your TubeSync data!
## Database Compression (For MariaDB)
With a lot of media files the `sync_media` table grows in size quickly.
You can save space using column compression using the following steps while using MariaDB:
1. Stop tubesync
2. Execute `ALTER TABLE sync_media MODIFY metadata LONGTEXT COMPRESSED;` on database tubesync
3. Start tunesync and confirm the connection still works.
## Docker Compose ## Docker Compose
If you're using Docker Compose and simply want to connect to another container with If you're using Docker Compose and simply want to connect to another container with
@ -118,6 +126,7 @@ database before it can be written to. This file should contain:
CREATE DATABASE tubesync; CREATE DATABASE tubesync;
``` ```
Then it must be mapped to `/docker-entrypoint-initdb.d/init.sql` for it Then it must be mapped to `/docker-entrypoint-initdb.d/init.sql` for it
to be executed on first startup of the container. See the `tubesync-db` to be executed on first startup of the container. See the `tubesync-db`
volume mapping above for how to do this. volume mapping above for how to do this.

View File

@ -0,0 +1,17 @@
# Generated by pac
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sync', '0020_auto_20231024_1825'),
]
operations = [
migrations.AddField(
model_name='source',
name='delete_files_on_disk',
field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'),
),
]

View File

@ -1,22 +0,0 @@
# Generated by pac
from django.db import migrations, models
from sync.models import Source
class Migration(migrations.Migration):
dependencies = [
('sync', '0020_auto_20231024_1825'),
]
operations = [
migrations.AddField(
model_name='source',
name='lightweight_metadata',
field=models.CharField(max_length=20,
default=Source.LIGHTWEIGHT_METADATA_TYPE_RAW,
choices=Source.LIGHTWEIGHT_METADATA_TYPE_CHOICES,
help_text='Lightweight metadata',
verbose_name='lightweight metadata'),
),
]

View File

@ -18,7 +18,7 @@ from common.utils import clean_filename
from .youtube import (get_media_info as get_youtube_media_info, from .youtube import (get_media_info as get_youtube_media_info,
download_media as download_youtube_media) download_media as download_youtube_media)
from .utils import seconds_to_timestr, parse_media_format from .utils import seconds_to_timestr, parse_media_format
from .matching import (get_best_combined_format, get_best_audio_format, from .matching import (get_best_combined_format, get_best_audio_format,
get_best_video_format) get_best_video_format)
from .mediaservers import PlexMediaServer from .mediaservers import PlexMediaServer
from .fields import CommaSepChoiceField from .fields import CommaSepChoiceField
@ -107,7 +107,6 @@ class Source(models.Model):
EXTENSION_MKV = 'mkv' EXTENSION_MKV = 'mkv'
EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV) EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
# as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
SPONSORBLOCK_CATEGORIES_CHOICES = ( SPONSORBLOCK_CATEGORIES_CHOICES = (
('sponsor', 'Sponsor'), ('sponsor', 'Sponsor'),
@ -119,17 +118,16 @@ class Source(models.Model):
('interaction', 'Interaction Reminder'), ('interaction', 'Interaction Reminder'),
('music_offtopic', 'Non-Music Section'), ('music_offtopic', 'Non-Music Section'),
) )
sponsorblock_categories = CommaSepChoiceField(
_(''),
possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
all_choice="all",
allow_all=True,
all_label="(all options)",
default="all",
help_text=_("Select the sponsorblocks you want to enforce")
)
sponsorblock_categories = CommaSepChoiceField(
_(''),
possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
all_choice='all',
allow_all=True,
all_label='(all options)',
default='all',
help_text=_('Select the sponsorblocks you want to enforce')
)
embed_metadata = models.BooleanField( embed_metadata = models.BooleanField(
_('embed metadata'), _('embed metadata'),
default=False, default=False,
@ -140,14 +138,12 @@ class Source(models.Model):
default=False, default=False,
help_text=_('Embed thumbnail into the file') help_text=_('Embed thumbnail into the file')
) )
enable_sponsorblock = models.BooleanField( enable_sponsorblock = models.BooleanField(
_('enable sponsorblock'), _('enable sponsorblock'),
default=True, default=True,
help_text=_('Use SponsorBlock?') help_text=_('Use SponsorBlock?')
) )
# Fontawesome icons used for the source on the front end # Fontawesome icons used for the source on the front end
ICONS = { ICONS = {
SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>', SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
@ -300,6 +296,11 @@ class Source(models.Model):
default=False, default=False,
help_text=_('Delete media that is no longer on this playlist') help_text=_('Delete media that is no longer on this playlist')
) )
delete_files_on_disk = models.BooleanField(
_('delete files on disk'),
default=False,
help_text=_('Delete files on disk when they are removed from TubeSync')
)
source_resolution = models.CharField( source_resolution = models.CharField(
_('source resolution'), _('source resolution'),
max_length=8, max_length=8,
@ -387,24 +388,6 @@ class Source(models.Model):
] ]
) )
LIGHTWEIGHT_METADATA_TYPE_RAW = 'RAW'
LIGHTWEIGHT_METADATA_TYPE_UNNECESSARY = 'UNNECESSARY'
LIGHTWEIGHT_METADATA_TYPE_FEATHER = 'FEATHER'
LIGHTWEIGHT_METADATA_TYPES = (LIGHTWEIGHT_METADATA_TYPE_RAW, LIGHTWEIGHT_METADATA_TYPE_UNNECESSARY, LIGHTWEIGHT_METADATA_TYPE_FEATHER)
LIGHTWEIGHT_METADATA_TYPE_CHOICES = (
(LIGHTWEIGHT_METADATA_TYPE_RAW, _("(LARGE) Save raw metadata")),
(LIGHTWEIGHT_METADATA_TYPE_UNNECESSARY, _("(MEDIUM) Treeshake unnecessary metadata json keys")),
(LIGHTWEIGHT_METADATA_TYPE_FEATHER, _("(TINY) if the capacity is large, Treeshake it event if it is in use")),
)
lightweight_metadata = models.CharField(
_('lightweight metadata'),
max_length=20,
default=LIGHTWEIGHT_METADATA_TYPE_RAW,
choices=LIGHTWEIGHT_METADATA_TYPE_CHOICES,
help_text=_('Lightweight metadata')
)
def __str__(self): def __str__(self):
return self.name return self.name
@ -568,7 +551,7 @@ class Source(models.Model):
if not self.filter_text: if not self.filter_text:
return True return True
return bool(re.search(self.filter_text, media_item_title)) return bool(re.search(self.filter_text, media_item_title))
def index_media(self): def index_media(self):
''' '''
Index the media source returning a list of media metadata as dicts. Index the media source returning a list of media metadata as dicts.
@ -887,7 +870,7 @@ class Media(models.Model):
def get_best_video_format(self): def get_best_video_format(self):
return get_best_video_format(self) return get_best_video_format(self)
def get_format_str(self): def get_format_str(self):
''' '''
Returns a youtube-dl compatible format string for the best matches Returns a youtube-dl compatible format string for the best matches
@ -912,7 +895,7 @@ class Media(models.Model):
else: else:
return False return False
return False return False
def get_display_format(self, format_str): def get_display_format(self, format_str):
''' '''
Returns a tuple used in the format component of the output filename. This Returns a tuple used in the format component of the output filename. This
@ -1203,7 +1186,7 @@ class Media(models.Model):
filename = self.filename filename = self.filename
prefix, ext = os.path.splitext(filename) prefix, ext = os.path.splitext(filename)
return f'{prefix}.nfo' return f'{prefix}.nfo'
@property @property
def nfopath(self): def nfopath(self):
return self.source.directory_path / self.nfoname return self.source.directory_path / self.nfoname
@ -1216,7 +1199,7 @@ class Media(models.Model):
filename = self.filename filename = self.filename
prefix, ext = os.path.splitext(filename) prefix, ext = os.path.splitext(filename)
return f'{prefix}.info.json' return f'{prefix}.info.json'
@property @property
def jsonpath(self): def jsonpath(self):
return self.source.directory_path / self.jsonname return self.source.directory_path / self.jsonname
@ -1254,7 +1237,6 @@ class Media(models.Model):
acodec = self.downloaded_audio_codec acodec = self.downloaded_audio_codec
if acodec is None: if acodec is None:
raise TypeError() # nothing here. raise TypeError() # nothing here.
acodec = acodec.lower() acodec = acodec.lower()
if acodec == "mp4a": if acodec == "mp4a":
return "audio/mp4" return "audio/mp4"
@ -1263,7 +1245,6 @@ class Media(models.Model):
else: else:
# fall-fall-back. # fall-fall-back.
return 'audio/ogg' return 'audio/ogg'
vcodec = vcodec.lower() vcodec = vcodec.lower()
if vcodec == 'vp9': if vcodec == 'vp9':
return 'video/webm' return 'video/webm'
@ -1287,6 +1268,22 @@ class Media(models.Model):
showtitle.text = str(self.source.name).strip() showtitle.text = str(self.source.name).strip()
showtitle.tail = '\n ' showtitle.tail = '\n '
nfo.append(showtitle) nfo.append(showtitle)
# season = upload date year
season = nfo.makeelement('season', {})
if self.source.source_type == Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
# If it's a playlist, set season to 1
season.text = '1'
else:
# If it's not a playlist, set season to upload date year
season.text = str(self.upload_date.year) if self.upload_date else ''
season.tail = '\n '
nfo.append(season)
# episode = number of video in the year
episode = nfo.makeelement('episode', {})
episode_number = self.calculate_episode_number()
episode.text = str(episode_number) if episode_number else ''
episode.tail = '\n '
nfo.append(episode)
# ratings = media metadata youtube rating # ratings = media metadata youtube rating
value = nfo.makeelement('value', {}) value = nfo.makeelement('value', {})
value.text = str(self.rating) value.text = str(self.rating)
@ -1394,8 +1391,8 @@ class Media(models.Model):
f'no valid format available') f'no valid format available')
# Download the media with youtube-dl # Download the media with youtube-dl
download_youtube_media(self.url, format_str, self.source.extension, download_youtube_media(self.url, format_str, self.source.extension,
str(self.filepath), self.source.write_json, str(self.filepath), self.source.write_json,
self.source.sponsorblock_categories, self.source.embed_thumbnail, self.source.sponsorblock_categories.selected_choices, self.source.embed_thumbnail,
self.source.embed_metadata, self.source.enable_sponsorblock, self.source.embed_metadata, self.source.enable_sponsorblock,
self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs ) self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs )
# Return the download paramaters # Return the download paramaters
@ -1411,6 +1408,19 @@ class Media(models.Model):
f'has no indexer') f'has no indexer')
return indexer(self.url) return indexer(self.url)
def calculate_episode_number(self):
if self.source.source_type == Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
sorted_media = Media.objects.filter(source=self.source)
else:
self_year = self.upload_date.year if self.upload_date else self.created.year
filtered_media = Media.objects.filter(source=self.source, published__year=self_year)
sorted_media = sorted(filtered_media, key=lambda x: (x.upload_date, x.key))
position_counter = 1
for media in sorted_media:
if media == self:
return position_counter
position_counter += 1
class MediaServer(models.Model): class MediaServer(models.Model):
''' '''

View File

@ -1,4 +1,5 @@
import os import os
import glob
from django.conf import settings from django.conf import settings
from django.db.models.signals import pre_save, post_save, pre_delete, post_delete from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
from django.dispatch import receiver from django.dispatch import receiver
@ -74,6 +75,7 @@ def source_pre_delete(sender, instance, **kwargs):
media.delete() media.delete()
@receiver(post_delete, sender=Source) @receiver(post_delete, sender=Source)
def source_post_delete(sender, instance, **kwargs): def source_post_delete(sender, instance, **kwargs):
# Triggered after a source is deleted # Triggered after a source is deleted
@ -222,6 +224,16 @@ def media_pre_delete(sender, instance, **kwargs):
if thumbnail_url: if thumbnail_url:
delete_task_by_media('sync.tasks.download_media_thumbnail', delete_task_by_media('sync.tasks.download_media_thumbnail',
(str(instance.pk), thumbnail_url)) (str(instance.pk), thumbnail_url))
if instance.source.delete_files_on_disk and (instance.media_file or instance.thumb):
# Delete all media files if it contains filename
filepath = instance.media_file.path if instance.media_file else instance.thumb.path
barefilepath, fileext = os.path.splitext(filepath)
# Get all files that start with the bare file path
all_related_files = glob.glob(f'{barefilepath}.*')
for file in all_related_files:
log.info(f'Deleting file for: {instance} path: {file}')
delete_file(file)
@receiver(post_delete, sender=Media) @receiver(post_delete, sender=Media)

View File

@ -236,12 +236,6 @@ def download_media_metadata(media_id):
return return
source = media.source source = media.source
metadata = media.index_metadata() metadata = media.index_metadata()
if source.lightweight_metadata == Source.LIGHTWEIGHT_METADATA_TYPE_FEATHER:
del metadata["formats"]
del metadata["thumbnails"]
del metadata["automatic_captions"]
del metadata["requested_formats"]
del metadata["heatmap"]
media.metadata = json.dumps(metadata, default=json_serial) media.metadata = json.dumps(metadata, default=json_serial)
upload_date = media.upload_date upload_date = media.upload_date
# Media must have a valid upload date # Media must have a valid upload date

View File

@ -132,8 +132,6 @@
<td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if media.can_download %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td> <td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if media.can_download %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr> </tr>
{% endif %} {% endif %}
{% if media.source.lightweight_metadata == "RAW" %}
<tr title="The available media formats"> <tr title="The available media formats">
<td class="hide-on-small-only">Available formats</td> <td class="hide-on-small-only">Available formats</td>
<td><span class="hide-on-med-and-up">Available formats<br></span> <td><span class="hide-on-med-and-up">Available formats<br></span>
@ -157,10 +155,7 @@
Video: <strong>{% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %} Video: <strong>{% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}
</strong></td> </strong></td>
</tr> </tr>
{% endif %}
</table> </table>
<p>{{ media.source.lightweight_metadata }}</p>
<p>{{ media.source }}</p>
</div> </div>
</div> </div>
{% if media.downloaded %} {% if media.downloaded %}

View File

@ -9,8 +9,8 @@
<p> <p>
Are you sure you want to delete this source? Deleting a source is permanent. Are you sure you want to delete this source? Deleting a source is permanent.
By default, deleting a source does not delete any saved media files. You can By default, deleting a source does not delete any saved media files. You can
tick the &quot;also delete downloaded media&quot; checkbox to also remove save <strong>tick the &quot;also delete downloaded media&quot; checkbox to also remove directory {{ source.directory_path }}
media when you delete the source. Deleting a source cannot be undone. </strong>when you delete the source. Deleting a source cannot be undone.
</p> </p>
</div> </div>
</div> </div>

View File

@ -122,6 +122,10 @@
<tr title="Delete media that is no longer on this playlist?"> <tr title="Delete media that is no longer on this playlist?">
<td class="hide-on-small-only">Delete removed media</td> <td class="hide-on-small-only">Delete removed media</td>
<td><span class="hide-on-med-and-up">Delete removed media<br></span><strong>{% if source.delete_removed_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td> <td><span class="hide-on-med-and-up">Delete removed media<br></span><strong>{% if source.delete_removed_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr>
<tr title="Delete files on disk when they are removed from TubeSync?">
<td class="hide-on-small-only">Delete files on disk</td>
<td><span class="hide-on-med-and-up">Delete files on disk<br></span><strong>{% if source.delete_files_on_disk %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr> </tr>
{% if source.delete_old_media and source.days_to_keep > 0 %} {% if source.delete_old_media and source.days_to_keep > 0 %}
<tr title="Days after which your media from this source will be locally deleted"> <tr title="Days after which your media from this source will be locally deleted">
@ -186,17 +190,7 @@
<td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td> <td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
</tr> </tr>
{% endif %} {% endif %}
{% if source.lightweight_metadata %}
<tr title="{{ _('Are auto subs accepted?') }}">
<td class="hide-on-small-only">{{ _("Auto-generated subtitles?") }}:</td>
<td><span class="hide-on-med-and-up">{{ _("Auto-generated subtitles?") }}:</span><strong><i class="fas {% if source.auto_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
</tr>
<tr title="{{ _('Subs langs?') }}">
<td class="hide-on-small-only">{{ _("Subs langs?") }}:</td>
<td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
</tr>
{% endif %}
</table> </table>
</div> </div>
</div> </div>

View File

@ -661,6 +661,8 @@ class MediaTestCase(TestCase):
'<episodedetails>', '<episodedetails>',
' <title>no fancy stuff title</title>', ' <title>no fancy stuff title</title>',
' <showtitle>testname</showtitle>', ' <showtitle>testname</showtitle>',
' <season>2017</season>',
' <episode></episode>',
' <ratings>', ' <ratings>',
' <rating default="True" max="5" name="youtube">', ' <rating default="True" max="5" name="youtube">',
' <value>1.2345</value>', ' <value>1.2345</value>',

View File

@ -1,7 +1,9 @@
import glob
import os import os
import json import json
from base64 import b64decode from base64 import b64decode
import pathlib import pathlib
import shutil
import sys import sys
from django.conf import settings from django.conf import settings
from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect
@ -59,7 +61,7 @@ class DashboardView(TemplateView):
# Disk usage # Disk usage
disk_usage = Media.objects.filter( disk_usage = Media.objects.filter(
downloaded=True, downloaded_filesize__isnull=False downloaded=True, downloaded_filesize__isnull=False
).aggregate(Sum('downloaded_filesize')) ).defer('metadata').aggregate(Sum('downloaded_filesize'))
data['disk_usage_bytes'] = disk_usage['downloaded_filesize__sum'] data['disk_usage_bytes'] = disk_usage['downloaded_filesize__sum']
if not data['disk_usage_bytes']: if not data['disk_usage_bytes']:
data['disk_usage_bytes'] = 0 data['disk_usage_bytes'] = 0
@ -71,11 +73,11 @@ class DashboardView(TemplateView):
# Latest downloads # Latest downloads
data['latest_downloads'] = Media.objects.filter( data['latest_downloads'] = Media.objects.filter(
downloaded=True, downloaded_filesize__isnull=False downloaded=True, downloaded_filesize__isnull=False
).order_by('-download_date')[:10] ).defer('metadata').order_by('-download_date')[:10]
# Largest downloads # Largest downloads
data['largest_downloads'] = Media.objects.filter( data['largest_downloads'] = Media.objects.filter(
downloaded=True, downloaded_filesize__isnull=False downloaded=True, downloaded_filesize__isnull=False
).order_by('-downloaded_filesize')[:10] ).defer('metadata').order_by('-downloaded_filesize')[:10]
# UID and GID # UID and GID
data['uid'] = os.getuid() data['uid'] = os.getuid()
data['gid'] = os.getgid() data['gid'] = os.getgid()
@ -296,11 +298,11 @@ class EditSourceMixin:
model = Source model = Source
fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format', fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format',
'index_schedule', 'download_media', 'download_cap', 'delete_old_media', 'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution',
'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback',
'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', 'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', 'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
'auto_subtitles', 'sub_langs', 'lightweight_metadata') 'auto_subtitles', 'sub_langs')
errors = { errors = {
'invalid_media_format': _('Invalid media format, the media format contains ' 'invalid_media_format': _('Invalid media format, the media format contains '
'errors or is empty. Check the table at the end of ' 'errors or is empty. Check the table at the end of '
@ -404,7 +406,7 @@ class SourceView(DetailView):
error_message = get_error_message(error) error_message = get_error_message(error)
setattr(error, 'error_message', error_message) setattr(error, 'error_message', error_message)
data['errors'].append(error) data['errors'].append(error)
data['media'] = Media.objects.filter(source=self.object).order_by('-published') data['media'] = Media.objects.filter(source=self.object).order_by('-published').defer('metadata')
return data return data
@ -435,14 +437,13 @@ class DeleteSourceView(DeleteView, FormMixin):
source = self.get_object() source = self.get_object()
for media in Media.objects.filter(source=source): for media in Media.objects.filter(source=source):
if media.media_file: if media.media_file:
# Delete the media file file_path = media.media_file.path
delete_file(media.media_file.path) matching_files = glob.glob(os.path.splitext(file_path)[0] + '.*')
# Delete thumbnail copy if it exists for file in matching_files:
delete_file(media.thumbpath) delete_file(file)
# Delete NFO file if it exists directory_path = source.directory_path
delete_file(media.nfopath) if os.path.exists(directory_path):
# Delete JSON file if it exists shutil.rmtree(directory_path, True)
delete_file(media.jsonpath)
return super().post(request, *args, **kwargs) return super().post(request, *args, **kwargs)
def get_success_url(self): def get_success_url(self):
@ -653,12 +654,13 @@ class MediaSkipView(FormView, SingleObjectMixin):
delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),)) delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),))
# If the media file exists on disk, delete it # If the media file exists on disk, delete it
if self.object.media_file_exists: if self.object.media_file_exists:
delete_file(self.object.media_file.path) # Delete all files which contains filename
self.object.media_file = None filepath = self.object.media_file.path
# If the media has an associated thumbnail copied, also delete it barefilepath, fileext = os.path.splitext(filepath)
delete_file(self.object.thumbpath) # Get all files that start with the bare file path
# If the media has an associated NFO file with it, also delete it all_related_files = glob.glob(f'{barefilepath}.*')
delete_file(self.object.nfopath) for file in all_related_files:
delete_file(file)
# Reset all download data # Reset all download data
self.object.metadata = None self.object.metadata = None
self.object.downloaded = False self.object.downloaded = False

View File

@ -1,5 +1,5 @@
''' '''
Wrapper for the youtube-dl library. Used so if there are any library interface Wrapper for the yt-dlp library. Used so if there are any library interface
updates we only need to udpate them in one place. updates we only need to udpate them in one place.
''' '''
@ -64,9 +64,9 @@ def get_media_info(url):
return response return response
def download_media(url, media_format, extension, output_file, info_json, def download_media(url, media_format, extension, output_file, info_json,
sponsor_categories="all", sponsor_categories=None,
embed_thumbnail=False, embed_metadata=False, skip_sponsors=True, embed_thumbnail=False, embed_metadata=False, skip_sponsors=True,
write_subtitles=False, auto_subtitles=False, sub_langs='en'): write_subtitles=False, auto_subtitles=False, sub_langs='en'):
''' '''
Downloads a YouTube URL to a file on disk. Downloads a YouTube URL to a file on disk.
@ -74,7 +74,7 @@ def download_media(url, media_format, extension, output_file, info_json,
def hook(event): def hook(event):
filename = os.path.basename(event['filename']) filename = os.path.basename(event['filename'])
if event.get('downloaded_bytes') is None or event.get('total_bytes') is None: if event.get('downloaded_bytes') is None or event.get('total_bytes') is None:
return None return None
@ -106,8 +106,8 @@ def download_media(url, media_format, extension, output_file, info_json,
f'{total_size_str} in {elapsed_str}') f'{total_size_str} in {elapsed_str}')
else: else:
log.warn(f'[youtube-dl] unknown event: {str(event)}') log.warn(f'[youtube-dl] unknown event: {str(event)}')
hook.download_progress = 0
hook.download_progress = 0
ytopts = { ytopts = {
'format': media_format, 'format': media_format,
'merge_output_format': extension, 'merge_output_format': extension,
@ -120,29 +120,25 @@ def download_media(url, media_format, extension, output_file, info_json,
'writeautomaticsub': auto_subtitles, 'writeautomaticsub': auto_subtitles,
'subtitleslangs': sub_langs.split(','), 'subtitleslangs': sub_langs.split(','),
} }
if not sponsor_categories:
sponsor_categories = []
sbopt = { sbopt = {
'key': 'SponsorBlock', 'key': 'SponsorBlock',
'categories': [sponsor_categories] 'categories': sponsor_categories
} }
ffmdopt = { ffmdopt = {
'key': 'FFmpegMetadata', 'key': 'FFmpegMetadata',
'add_chapters': True, 'add_chapters': embed_metadata,
'add_metadata': True 'add_metadata': embed_metadata
} }
opts = get_yt_opts() opts = get_yt_opts()
if embed_thumbnail: if embed_thumbnail:
ytopts['postprocessors'].append({'key': 'EmbedThumbnail'}) ytopts['postprocessors'].append({'key': 'EmbedThumbnail'})
if embed_metadata:
ffmdopt["add_metadata"] = True
if skip_sponsors: if skip_sponsors:
ytopts['postprocessors'].append(sbopt) ytopts['postprocessors'].append(sbopt)
ytopts['postprocessors'].append(ffmdopt) ytopts['postprocessors'].append(ffmdopt)
opts.update(ytopts) opts.update(ytopts)
with yt_dlp.YoutubeDL(opts) as y: with yt_dlp.YoutubeDL(opts) as y:
try: try:
return y.download([url]) return y.download([url])

View File

@ -25,9 +25,6 @@ DEBUG = True if os.getenv('TUBESYNC_DEBUG', False) else False
FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX) FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX)
TIME_ZONE = os.getenv('TZ', 'UTC')
database_dict = {} database_dict = {}
database_connection_env = os.getenv('DATABASE_CONNECTION', '') database_connection_env = os.getenv('DATABASE_CONNECTION', '')
if database_connection_env: if database_connection_env:

View File

@ -1,3 +1,4 @@
import os
from pathlib import Path from pathlib import Path
@ -96,7 +97,7 @@ AUTH_PASSWORD_VALIDATORS = [
LANGUAGE_CODE = 'en-us' LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC' TIME_ZONE = os.getenv('TZ', 'UTC')
USE_I18N = True USE_I18N = True
USE_L10N = True USE_L10N = True
USE_TZ = True USE_TZ = True