Compare commits

..

1 Commits

Author SHA1 Message Date
woozu-shin 0b13065c9d [NO-ISSUE] Add lightweight option 2024-02-21 12:53:53 +09:00
19 changed files with 177 additions and 136 deletions

8
.idea/.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

View File

@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

8
.idea/modules.xml Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/tubesync.iml" filepath="$PROJECT_DIR$/.idea/tubesync.iml" />
</modules>
</component>
</project>

20
.idea/tubesync.iml Normal file
View File

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
</component>
<component name="TemplatesService">
<option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
<option name="TEMPLATE_FOLDERS">
<list>
<option value="$MODULE_DIR$/tubesync/common/templates" />
</list>
</option>
</component>
</module>

6
.idea/vcs.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

View File

@ -79,14 +79,6 @@ entry in the container or stdout logs:
If you see a line similar to the above and the web interface loads, congratulations, If you see a line similar to the above and the web interface loads, congratulations,
you are now using an external database server for your TubeSync data! you are now using an external database server for your TubeSync data!
## Database Compression (For MariaDB)
With a lot of media files the `sync_media` table grows in size quickly.
You can save space using column compression using the following steps while using MariaDB:
1. Stop tubesync
2. Execute `ALTER TABLE sync_media MODIFY metadata LONGTEXT COMPRESSED;` on database tubesync
3. Start tunesync and confirm the connection still works.
## Docker Compose ## Docker Compose
If you're using Docker Compose and simply want to connect to another container with If you're using Docker Compose and simply want to connect to another container with
@ -126,7 +118,6 @@ database before it can be written to. This file should contain:
CREATE DATABASE tubesync; CREATE DATABASE tubesync;
``` ```
Then it must be mapped to `/docker-entrypoint-initdb.d/init.sql` for it Then it must be mapped to `/docker-entrypoint-initdb.d/init.sql` for it
to be executed on first startup of the container. See the `tubesync-db` to be executed on first startup of the container. See the `tubesync-db`
volume mapping above for how to do this. volume mapping above for how to do this.

View File

@ -1,17 +0,0 @@
# Generated by pac
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sync', '0020_auto_20231024_1825'),
]
operations = [
migrations.AddField(
model_name='source',
name='delete_files_on_disk',
field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'),
),
]

View File

@ -0,0 +1,22 @@
# Generated by pac
from django.db import migrations, models
from sync.models import Source
class Migration(migrations.Migration):
dependencies = [
('sync', '0020_auto_20231024_1825'),
]
operations = [
migrations.AddField(
model_name='source',
name='lightweight_metadata',
field=models.CharField(max_length=20,
default=Source.LIGHTWEIGHT_METADATA_TYPE_RAW,
choices=Source.LIGHTWEIGHT_METADATA_TYPE_CHOICES,
help_text='Lightweight metadata',
verbose_name='lightweight metadata'),
),
]

View File

@ -18,7 +18,7 @@ from common.utils import clean_filename
from .youtube import (get_media_info as get_youtube_media_info, from .youtube import (get_media_info as get_youtube_media_info,
download_media as download_youtube_media) download_media as download_youtube_media)
from .utils import seconds_to_timestr, parse_media_format from .utils import seconds_to_timestr, parse_media_format
from .matching import (get_best_combined_format, get_best_audio_format, from .matching import (get_best_combined_format, get_best_audio_format,
get_best_video_format) get_best_video_format)
from .mediaservers import PlexMediaServer from .mediaservers import PlexMediaServer
from .fields import CommaSepChoiceField from .fields import CommaSepChoiceField
@ -107,6 +107,7 @@ class Source(models.Model):
EXTENSION_MKV = 'mkv' EXTENSION_MKV = 'mkv'
EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV) EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
# as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
SPONSORBLOCK_CATEGORIES_CHOICES = ( SPONSORBLOCK_CATEGORIES_CHOICES = (
('sponsor', 'Sponsor'), ('sponsor', 'Sponsor'),
@ -118,16 +119,17 @@ class Source(models.Model):
('interaction', 'Interaction Reminder'), ('interaction', 'Interaction Reminder'),
('music_offtopic', 'Non-Music Section'), ('music_offtopic', 'Non-Music Section'),
) )
sponsorblock_categories = CommaSepChoiceField( sponsorblock_categories = CommaSepChoiceField(
_(''), _(''),
possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES, possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
all_choice='all', all_choice="all",
allow_all=True, allow_all=True,
all_label='(all options)', all_label="(all options)",
default='all', default="all",
help_text=_('Select the sponsorblocks you want to enforce') help_text=_("Select the sponsorblocks you want to enforce")
) )
embed_metadata = models.BooleanField( embed_metadata = models.BooleanField(
_('embed metadata'), _('embed metadata'),
default=False, default=False,
@ -138,12 +140,14 @@ class Source(models.Model):
default=False, default=False,
help_text=_('Embed thumbnail into the file') help_text=_('Embed thumbnail into the file')
) )
enable_sponsorblock = models.BooleanField( enable_sponsorblock = models.BooleanField(
_('enable sponsorblock'), _('enable sponsorblock'),
default=True, default=True,
help_text=_('Use SponsorBlock?') help_text=_('Use SponsorBlock?')
) )
# Fontawesome icons used for the source on the front end # Fontawesome icons used for the source on the front end
ICONS = { ICONS = {
SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>', SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
@ -296,11 +300,6 @@ class Source(models.Model):
default=False, default=False,
help_text=_('Delete media that is no longer on this playlist') help_text=_('Delete media that is no longer on this playlist')
) )
delete_files_on_disk = models.BooleanField(
_('delete files on disk'),
default=False,
help_text=_('Delete files on disk when they are removed from TubeSync')
)
source_resolution = models.CharField( source_resolution = models.CharField(
_('source resolution'), _('source resolution'),
max_length=8, max_length=8,
@ -388,6 +387,24 @@ class Source(models.Model):
] ]
) )
LIGHTWEIGHT_METADATA_TYPE_RAW = 'RAW'
LIGHTWEIGHT_METADATA_TYPE_UNNECESSARY = 'UNNECESSARY'
LIGHTWEIGHT_METADATA_TYPE_FEATHER = 'FEATHER'
LIGHTWEIGHT_METADATA_TYPES = (LIGHTWEIGHT_METADATA_TYPE_RAW, LIGHTWEIGHT_METADATA_TYPE_UNNECESSARY, LIGHTWEIGHT_METADATA_TYPE_FEATHER)
LIGHTWEIGHT_METADATA_TYPE_CHOICES = (
(LIGHTWEIGHT_METADATA_TYPE_RAW, _("(LARGE) Save raw metadata")),
(LIGHTWEIGHT_METADATA_TYPE_UNNECESSARY, _("(MEDIUM) Treeshake unnecessary metadata json keys")),
(LIGHTWEIGHT_METADATA_TYPE_FEATHER, _("(TINY) if the capacity is large, Treeshake it event if it is in use")),
)
lightweight_metadata = models.CharField(
_('lightweight metadata'),
max_length=20,
default=LIGHTWEIGHT_METADATA_TYPE_RAW,
choices=LIGHTWEIGHT_METADATA_TYPE_CHOICES,
help_text=_('Lightweight metadata')
)
def __str__(self): def __str__(self):
return self.name return self.name
@ -551,7 +568,7 @@ class Source(models.Model):
if not self.filter_text: if not self.filter_text:
return True return True
return bool(re.search(self.filter_text, media_item_title)) return bool(re.search(self.filter_text, media_item_title))
def index_media(self): def index_media(self):
''' '''
Index the media source returning a list of media metadata as dicts. Index the media source returning a list of media metadata as dicts.
@ -870,7 +887,7 @@ class Media(models.Model):
def get_best_video_format(self): def get_best_video_format(self):
return get_best_video_format(self) return get_best_video_format(self)
def get_format_str(self): def get_format_str(self):
''' '''
Returns a youtube-dl compatible format string for the best matches Returns a youtube-dl compatible format string for the best matches
@ -895,7 +912,7 @@ class Media(models.Model):
else: else:
return False return False
return False return False
def get_display_format(self, format_str): def get_display_format(self, format_str):
''' '''
Returns a tuple used in the format component of the output filename. This Returns a tuple used in the format component of the output filename. This
@ -1186,7 +1203,7 @@ class Media(models.Model):
filename = self.filename filename = self.filename
prefix, ext = os.path.splitext(filename) prefix, ext = os.path.splitext(filename)
return f'{prefix}.nfo' return f'{prefix}.nfo'
@property @property
def nfopath(self): def nfopath(self):
return self.source.directory_path / self.nfoname return self.source.directory_path / self.nfoname
@ -1199,7 +1216,7 @@ class Media(models.Model):
filename = self.filename filename = self.filename
prefix, ext = os.path.splitext(filename) prefix, ext = os.path.splitext(filename)
return f'{prefix}.info.json' return f'{prefix}.info.json'
@property @property
def jsonpath(self): def jsonpath(self):
return self.source.directory_path / self.jsonname return self.source.directory_path / self.jsonname
@ -1237,6 +1254,7 @@ class Media(models.Model):
acodec = self.downloaded_audio_codec acodec = self.downloaded_audio_codec
if acodec is None: if acodec is None:
raise TypeError() # nothing here. raise TypeError() # nothing here.
acodec = acodec.lower() acodec = acodec.lower()
if acodec == "mp4a": if acodec == "mp4a":
return "audio/mp4" return "audio/mp4"
@ -1245,6 +1263,7 @@ class Media(models.Model):
else: else:
# fall-fall-back. # fall-fall-back.
return 'audio/ogg' return 'audio/ogg'
vcodec = vcodec.lower() vcodec = vcodec.lower()
if vcodec == 'vp9': if vcodec == 'vp9':
return 'video/webm' return 'video/webm'
@ -1268,22 +1287,6 @@ class Media(models.Model):
showtitle.text = str(self.source.name).strip() showtitle.text = str(self.source.name).strip()
showtitle.tail = '\n ' showtitle.tail = '\n '
nfo.append(showtitle) nfo.append(showtitle)
# season = upload date year
season = nfo.makeelement('season', {})
if self.source.source_type == Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
# If it's a playlist, set season to 1
season.text = '1'
else:
# If it's not a playlist, set season to upload date year
season.text = str(self.upload_date.year) if self.upload_date else ''
season.tail = '\n '
nfo.append(season)
# episode = number of video in the year
episode = nfo.makeelement('episode', {})
episode_number = self.calculate_episode_number()
episode.text = str(episode_number) if episode_number else ''
episode.tail = '\n '
nfo.append(episode)
# ratings = media metadata youtube rating # ratings = media metadata youtube rating
value = nfo.makeelement('value', {}) value = nfo.makeelement('value', {})
value.text = str(self.rating) value.text = str(self.rating)
@ -1391,8 +1394,8 @@ class Media(models.Model):
f'no valid format available') f'no valid format available')
# Download the media with youtube-dl # Download the media with youtube-dl
download_youtube_media(self.url, format_str, self.source.extension, download_youtube_media(self.url, format_str, self.source.extension,
str(self.filepath), self.source.write_json, str(self.filepath), self.source.write_json,
self.source.sponsorblock_categories.selected_choices, self.source.embed_thumbnail, self.source.sponsorblock_categories, self.source.embed_thumbnail,
self.source.embed_metadata, self.source.enable_sponsorblock, self.source.embed_metadata, self.source.enable_sponsorblock,
self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs ) self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs )
# Return the download paramaters # Return the download paramaters
@ -1408,19 +1411,6 @@ class Media(models.Model):
f'has no indexer') f'has no indexer')
return indexer(self.url) return indexer(self.url)
def calculate_episode_number(self):
if self.source.source_type == Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
sorted_media = Media.objects.filter(source=self.source)
else:
self_year = self.upload_date.year if self.upload_date else self.created.year
filtered_media = Media.objects.filter(source=self.source, published__year=self_year)
sorted_media = sorted(filtered_media, key=lambda x: (x.upload_date, x.key))
position_counter = 1
for media in sorted_media:
if media == self:
return position_counter
position_counter += 1
class MediaServer(models.Model): class MediaServer(models.Model):
''' '''

View File

@ -1,5 +1,4 @@
import os import os
import glob
from django.conf import settings from django.conf import settings
from django.db.models.signals import pre_save, post_save, pre_delete, post_delete from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
from django.dispatch import receiver from django.dispatch import receiver
@ -75,7 +74,6 @@ def source_pre_delete(sender, instance, **kwargs):
media.delete() media.delete()
@receiver(post_delete, sender=Source) @receiver(post_delete, sender=Source)
def source_post_delete(sender, instance, **kwargs): def source_post_delete(sender, instance, **kwargs):
# Triggered after a source is deleted # Triggered after a source is deleted
@ -224,16 +222,6 @@ def media_pre_delete(sender, instance, **kwargs):
if thumbnail_url: if thumbnail_url:
delete_task_by_media('sync.tasks.download_media_thumbnail', delete_task_by_media('sync.tasks.download_media_thumbnail',
(str(instance.pk), thumbnail_url)) (str(instance.pk), thumbnail_url))
if instance.source.delete_files_on_disk and (instance.media_file or instance.thumb):
# Delete all media files if it contains filename
filepath = instance.media_file.path if instance.media_file else instance.thumb.path
barefilepath, fileext = os.path.splitext(filepath)
# Get all files that start with the bare file path
all_related_files = glob.glob(f'{barefilepath}.*')
for file in all_related_files:
log.info(f'Deleting file for: {instance} path: {file}')
delete_file(file)
@receiver(post_delete, sender=Media) @receiver(post_delete, sender=Media)

View File

@ -236,6 +236,12 @@ def download_media_metadata(media_id):
return return
source = media.source source = media.source
metadata = media.index_metadata() metadata = media.index_metadata()
if source.lightweight_metadata == Source.LIGHTWEIGHT_METADATA_TYPE_FEATHER:
del metadata["formats"]
del metadata["thumbnails"]
del metadata["automatic_captions"]
del metadata["requested_formats"]
del metadata["heatmap"]
media.metadata = json.dumps(metadata, default=json_serial) media.metadata = json.dumps(metadata, default=json_serial)
upload_date = media.upload_date upload_date = media.upload_date
# Media must have a valid upload date # Media must have a valid upload date

View File

@ -132,6 +132,8 @@
<td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if media.can_download %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td> <td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if media.can_download %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr> </tr>
{% endif %} {% endif %}
{% if media.source.lightweight_metadata == "RAW" %}
<tr title="The available media formats"> <tr title="The available media formats">
<td class="hide-on-small-only">Available formats</td> <td class="hide-on-small-only">Available formats</td>
<td><span class="hide-on-med-and-up">Available formats<br></span> <td><span class="hide-on-med-and-up">Available formats<br></span>
@ -155,7 +157,10 @@
Video: <strong>{% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %} Video: <strong>{% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}
</strong></td> </strong></td>
</tr> </tr>
{% endif %}
</table> </table>
<p>{{ media.source.lightweight_metadata }}</p>
<p>{{ media.source }}</p>
</div> </div>
</div> </div>
{% if media.downloaded %} {% if media.downloaded %}

View File

@ -9,8 +9,8 @@
<p> <p>
Are you sure you want to delete this source? Deleting a source is permanent. Are you sure you want to delete this source? Deleting a source is permanent.
By default, deleting a source does not delete any saved media files. You can By default, deleting a source does not delete any saved media files. You can
<strong>tick the &quot;also delete downloaded media&quot; checkbox to also remove directory {{ source.directory_path }} tick the &quot;also delete downloaded media&quot; checkbox to also remove save
</strong>when you delete the source. Deleting a source cannot be undone. media when you delete the source. Deleting a source cannot be undone.
</p> </p>
</div> </div>
</div> </div>

View File

@ -122,10 +122,6 @@
<tr title="Delete media that is no longer on this playlist?"> <tr title="Delete media that is no longer on this playlist?">
<td class="hide-on-small-only">Delete removed media</td> <td class="hide-on-small-only">Delete removed media</td>
<td><span class="hide-on-med-and-up">Delete removed media<br></span><strong>{% if source.delete_removed_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td> <td><span class="hide-on-med-and-up">Delete removed media<br></span><strong>{% if source.delete_removed_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr>
<tr title="Delete files on disk when they are removed from TubeSync?">
<td class="hide-on-small-only">Delete files on disk</td>
<td><span class="hide-on-med-and-up">Delete files on disk<br></span><strong>{% if source.delete_files_on_disk %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr> </tr>
{% if source.delete_old_media and source.days_to_keep > 0 %} {% if source.delete_old_media and source.days_to_keep > 0 %}
<tr title="Days after which your media from this source will be locally deleted"> <tr title="Days after which your media from this source will be locally deleted">
@ -190,7 +186,17 @@
<td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td> <td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
</tr> </tr>
{% endif %} {% endif %}
{% if source.lightweight_metadata %}
<tr title="{{ _('Are auto subs accepted?') }}">
<td class="hide-on-small-only">{{ _("Auto-generated subtitles?") }}:</td>
<td><span class="hide-on-med-and-up">{{ _("Auto-generated subtitles?") }}:</span><strong><i class="fas {% if source.auto_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
</tr>
<tr title="{{ _('Subs langs?') }}">
<td class="hide-on-small-only">{{ _("Subs langs?") }}:</td>
<td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
</tr>
{% endif %}
</table> </table>
</div> </div>
</div> </div>

View File

@ -661,8 +661,6 @@ class MediaTestCase(TestCase):
'<episodedetails>', '<episodedetails>',
' <title>no fancy stuff title</title>', ' <title>no fancy stuff title</title>',
' <showtitle>testname</showtitle>', ' <showtitle>testname</showtitle>',
' <season>2017</season>',
' <episode></episode>',
' <ratings>', ' <ratings>',
' <rating default="True" max="5" name="youtube">', ' <rating default="True" max="5" name="youtube">',
' <value>1.2345</value>', ' <value>1.2345</value>',

View File

@ -1,9 +1,7 @@
import glob
import os import os
import json import json
from base64 import b64decode from base64 import b64decode
import pathlib import pathlib
import shutil
import sys import sys
from django.conf import settings from django.conf import settings
from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect
@ -61,7 +59,7 @@ class DashboardView(TemplateView):
# Disk usage # Disk usage
disk_usage = Media.objects.filter( disk_usage = Media.objects.filter(
downloaded=True, downloaded_filesize__isnull=False downloaded=True, downloaded_filesize__isnull=False
).defer('metadata').aggregate(Sum('downloaded_filesize')) ).aggregate(Sum('downloaded_filesize'))
data['disk_usage_bytes'] = disk_usage['downloaded_filesize__sum'] data['disk_usage_bytes'] = disk_usage['downloaded_filesize__sum']
if not data['disk_usage_bytes']: if not data['disk_usage_bytes']:
data['disk_usage_bytes'] = 0 data['disk_usage_bytes'] = 0
@ -73,11 +71,11 @@ class DashboardView(TemplateView):
# Latest downloads # Latest downloads
data['latest_downloads'] = Media.objects.filter( data['latest_downloads'] = Media.objects.filter(
downloaded=True, downloaded_filesize__isnull=False downloaded=True, downloaded_filesize__isnull=False
).defer('metadata').order_by('-download_date')[:10] ).order_by('-download_date')[:10]
# Largest downloads # Largest downloads
data['largest_downloads'] = Media.objects.filter( data['largest_downloads'] = Media.objects.filter(
downloaded=True, downloaded_filesize__isnull=False downloaded=True, downloaded_filesize__isnull=False
).defer('metadata').order_by('-downloaded_filesize')[:10] ).order_by('-downloaded_filesize')[:10]
# UID and GID # UID and GID
data['uid'] = os.getuid() data['uid'] = os.getuid()
data['gid'] = os.getgid() data['gid'] = os.getgid()
@ -298,11 +296,11 @@ class EditSourceMixin:
model = Source model = Source
fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format', fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format',
'index_schedule', 'download_media', 'download_cap', 'delete_old_media', 'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution', 'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails',
'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', 'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
'auto_subtitles', 'sub_langs') 'auto_subtitles', 'sub_langs', 'lightweight_metadata')
errors = { errors = {
'invalid_media_format': _('Invalid media format, the media format contains ' 'invalid_media_format': _('Invalid media format, the media format contains '
'errors or is empty. Check the table at the end of ' 'errors or is empty. Check the table at the end of '
@ -406,7 +404,7 @@ class SourceView(DetailView):
error_message = get_error_message(error) error_message = get_error_message(error)
setattr(error, 'error_message', error_message) setattr(error, 'error_message', error_message)
data['errors'].append(error) data['errors'].append(error)
data['media'] = Media.objects.filter(source=self.object).order_by('-published').defer('metadata') data['media'] = Media.objects.filter(source=self.object).order_by('-published')
return data return data
@ -437,13 +435,14 @@ class DeleteSourceView(DeleteView, FormMixin):
source = self.get_object() source = self.get_object()
for media in Media.objects.filter(source=source): for media in Media.objects.filter(source=source):
if media.media_file: if media.media_file:
file_path = media.media_file.path # Delete the media file
matching_files = glob.glob(os.path.splitext(file_path)[0] + '.*') delete_file(media.media_file.path)
for file in matching_files: # Delete thumbnail copy if it exists
delete_file(file) delete_file(media.thumbpath)
directory_path = source.directory_path # Delete NFO file if it exists
if os.path.exists(directory_path): delete_file(media.nfopath)
shutil.rmtree(directory_path, True) # Delete JSON file if it exists
delete_file(media.jsonpath)
return super().post(request, *args, **kwargs) return super().post(request, *args, **kwargs)
def get_success_url(self): def get_success_url(self):
@ -654,13 +653,12 @@ class MediaSkipView(FormView, SingleObjectMixin):
delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),)) delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),))
# If the media file exists on disk, delete it # If the media file exists on disk, delete it
if self.object.media_file_exists: if self.object.media_file_exists:
# Delete all files which contains filename delete_file(self.object.media_file.path)
filepath = self.object.media_file.path self.object.media_file = None
barefilepath, fileext = os.path.splitext(filepath) # If the media has an associated thumbnail copied, also delete it
# Get all files that start with the bare file path delete_file(self.object.thumbpath)
all_related_files = glob.glob(f'{barefilepath}.*') # If the media has an associated NFO file with it, also delete it
for file in all_related_files: delete_file(self.object.nfopath)
delete_file(file)
# Reset all download data # Reset all download data
self.object.metadata = None self.object.metadata = None
self.object.downloaded = False self.object.downloaded = False

View File

@ -1,5 +1,5 @@
''' '''
Wrapper for the yt-dlp library. Used so if there are any library interface Wrapper for the youtube-dl library. Used so if there are any library interface
updates we only need to udpate them in one place. updates we only need to udpate them in one place.
''' '''
@ -64,9 +64,9 @@ def get_media_info(url):
return response return response
def download_media(url, media_format, extension, output_file, info_json, def download_media(url, media_format, extension, output_file, info_json,
sponsor_categories=None, sponsor_categories="all",
embed_thumbnail=False, embed_metadata=False, skip_sponsors=True, embed_thumbnail=False, embed_metadata=False, skip_sponsors=True,
write_subtitles=False, auto_subtitles=False, sub_langs='en'): write_subtitles=False, auto_subtitles=False, sub_langs='en'):
''' '''
Downloads a YouTube URL to a file on disk. Downloads a YouTube URL to a file on disk.
@ -74,7 +74,7 @@ def download_media(url, media_format, extension, output_file, info_json,
def hook(event): def hook(event):
filename = os.path.basename(event['filename']) filename = os.path.basename(event['filename'])
if event.get('downloaded_bytes') is None or event.get('total_bytes') is None: if event.get('downloaded_bytes') is None or event.get('total_bytes') is None:
return None return None
@ -106,8 +106,8 @@ def download_media(url, media_format, extension, output_file, info_json,
f'{total_size_str} in {elapsed_str}') f'{total_size_str} in {elapsed_str}')
else: else:
log.warn(f'[youtube-dl] unknown event: {str(event)}') log.warn(f'[youtube-dl] unknown event: {str(event)}')
hook.download_progress = 0 hook.download_progress = 0
ytopts = { ytopts = {
'format': media_format, 'format': media_format,
'merge_output_format': extension, 'merge_output_format': extension,
@ -120,25 +120,29 @@ def download_media(url, media_format, extension, output_file, info_json,
'writeautomaticsub': auto_subtitles, 'writeautomaticsub': auto_subtitles,
'subtitleslangs': sub_langs.split(','), 'subtitleslangs': sub_langs.split(','),
} }
if not sponsor_categories:
sponsor_categories = []
sbopt = { sbopt = {
'key': 'SponsorBlock', 'key': 'SponsorBlock',
'categories': sponsor_categories 'categories': [sponsor_categories]
} }
ffmdopt = { ffmdopt = {
'key': 'FFmpegMetadata', 'key': 'FFmpegMetadata',
'add_chapters': embed_metadata, 'add_chapters': True,
'add_metadata': embed_metadata 'add_metadata': True
} }
opts = get_yt_opts() opts = get_yt_opts()
if embed_thumbnail: if embed_thumbnail:
ytopts['postprocessors'].append({'key': 'EmbedThumbnail'}) ytopts['postprocessors'].append({'key': 'EmbedThumbnail'})
if embed_metadata:
ffmdopt["add_metadata"] = True
if skip_sponsors: if skip_sponsors:
ytopts['postprocessors'].append(sbopt) ytopts['postprocessors'].append(sbopt)
ytopts['postprocessors'].append(ffmdopt) ytopts['postprocessors'].append(ffmdopt)
opts.update(ytopts) opts.update(ytopts)
with yt_dlp.YoutubeDL(opts) as y: with yt_dlp.YoutubeDL(opts) as y:
try: try:
return y.download([url]) return y.download([url])

View File

@ -25,6 +25,9 @@ DEBUG = True if os.getenv('TUBESYNC_DEBUG', False) else False
FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX) FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX)
TIME_ZONE = os.getenv('TZ', 'UTC')
database_dict = {} database_dict = {}
database_connection_env = os.getenv('DATABASE_CONNECTION', '') database_connection_env = os.getenv('DATABASE_CONNECTION', '')
if database_connection_env: if database_connection_env:

View File

@ -1,4 +1,3 @@
import os
from pathlib import Path from pathlib import Path
@ -97,7 +96,7 @@ AUTH_PASSWORD_VALIDATORS = [
LANGUAGE_CODE = 'en-us' LANGUAGE_CODE = 'en-us'
TIME_ZONE = os.getenv('TZ', 'UTC') TIME_ZONE = 'UTC'
USE_I18N = True USE_I18N = True
USE_L10N = True USE_L10N = True
USE_TZ = True USE_TZ = True