Compare commits
25 Commits
feature/wo
...
main
Author | SHA1 | Date |
---|---|---|
meeb | c6acd5378c | |
meeb | e7788eb8fb | |
meeb | e4e0b48c0b | |
Dmitrii Savchenko | 3573c1187f | |
meeb | b11b667aff | |
Yottatron | 1b581aa4ba | |
meeb | 7384c00713 | |
meeb | 4fdd172b05 | |
Someone | 9c18115032 | |
meeb | 6853c1fa76 | |
administrator | ed07073cf4 | |
administrator | af94b37ee6 | |
administrator | ad1d49a835 | |
administrator | 46ba2593a2 | |
administrator | 46a43b968a | |
administrator | 805a0eefbd | |
meeb | 3a87b5779e | |
Someone | f86e72aa92 | |
Someone | f550e32b5e | |
Someone | 034d877d6a | |
Someone | b9b702ab85 | |
meeb | c159c24d15 | |
depuhitv | 6c9772d573 | |
depuhitv | 45b8b3f65b | |
sparklesmcfadden | 43cf532903 |
|
@ -1,8 +0,0 @@
|
||||||
# Default ignored files
|
|
||||||
/shelf/
|
|
||||||
/workspace.xml
|
|
||||||
# Editor-based HTTP Client requests
|
|
||||||
/httpRequests/
|
|
||||||
# Datasource local storage ignored files
|
|
||||||
/dataSources/
|
|
||||||
/dataSources.local.xml
|
|
|
@ -1,6 +0,0 @@
|
||||||
<component name="InspectionProjectProfileManager">
|
|
||||||
<settings>
|
|
||||||
<option name="USE_PROJECT_PROFILE" value="false" />
|
|
||||||
<version value="1.0" />
|
|
||||||
</settings>
|
|
||||||
</component>
|
|
|
@ -1,8 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project version="4">
|
|
||||||
<component name="ProjectModuleManager">
|
|
||||||
<modules>
|
|
||||||
<module fileurl="file://$PROJECT_DIR$/.idea/tubesync.iml" filepath="$PROJECT_DIR$/.idea/tubesync.iml" />
|
|
||||||
</modules>
|
|
||||||
</component>
|
|
||||||
</project>
|
|
|
@ -1,20 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<module type="PYTHON_MODULE" version="4">
|
|
||||||
<component name="NewModuleRootManager">
|
|
||||||
<content url="file://$MODULE_DIR$" />
|
|
||||||
<orderEntry type="inheritedJdk" />
|
|
||||||
<orderEntry type="sourceFolder" forTests="false" />
|
|
||||||
</component>
|
|
||||||
<component name="PyDocumentationSettings">
|
|
||||||
<option name="format" value="PLAIN" />
|
|
||||||
<option name="myDocStringFormat" value="Plain" />
|
|
||||||
</component>
|
|
||||||
<component name="TemplatesService">
|
|
||||||
<option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
|
|
||||||
<option name="TEMPLATE_FOLDERS">
|
|
||||||
<list>
|
|
||||||
<option value="$MODULE_DIR$/tubesync/common/templates" />
|
|
||||||
</list>
|
|
||||||
</option>
|
|
||||||
</component>
|
|
||||||
</module>
|
|
|
@ -1,6 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project version="4">
|
|
||||||
<component name="VcsDirectoryMappings">
|
|
||||||
<mapping directory="" vcs="Git" />
|
|
||||||
</component>
|
|
||||||
</project>
|
|
|
@ -79,6 +79,14 @@ entry in the container or stdout logs:
|
||||||
If you see a line similar to the above and the web interface loads, congratulations,
|
If you see a line similar to the above and the web interface loads, congratulations,
|
||||||
you are now using an external database server for your TubeSync data!
|
you are now using an external database server for your TubeSync data!
|
||||||
|
|
||||||
|
## Database Compression (For MariaDB)
|
||||||
|
With a lot of media files the `sync_media` table grows in size quickly.
|
||||||
|
You can save space using column compression using the following steps while using MariaDB:
|
||||||
|
|
||||||
|
1. Stop tubesync
|
||||||
|
2. Execute `ALTER TABLE sync_media MODIFY metadata LONGTEXT COMPRESSED;` on database tubesync
|
||||||
|
3. Start tunesync and confirm the connection still works.
|
||||||
|
|
||||||
## Docker Compose
|
## Docker Compose
|
||||||
|
|
||||||
If you're using Docker Compose and simply want to connect to another container with
|
If you're using Docker Compose and simply want to connect to another container with
|
||||||
|
@ -118,6 +126,7 @@ database before it can be written to. This file should contain:
|
||||||
CREATE DATABASE tubesync;
|
CREATE DATABASE tubesync;
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
Then it must be mapped to `/docker-entrypoint-initdb.d/init.sql` for it
|
Then it must be mapped to `/docker-entrypoint-initdb.d/init.sql` for it
|
||||||
to be executed on first startup of the container. See the `tubesync-db`
|
to be executed on first startup of the container. See the `tubesync-db`
|
||||||
volume mapping above for how to do this.
|
volume mapping above for how to do this.
|
||||||
|
|
|
@ -0,0 +1,17 @@
|
||||||
|
# Generated by pac
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('sync', '0020_auto_20231024_1825'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='source',
|
||||||
|
name='delete_files_on_disk',
|
||||||
|
field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'),
|
||||||
|
),
|
||||||
|
]
|
|
@ -1,22 +0,0 @@
|
||||||
# Generated by pac
|
|
||||||
|
|
||||||
from django.db import migrations, models
|
|
||||||
from sync.models import Source
|
|
||||||
|
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
|
||||||
dependencies = [
|
|
||||||
('sync', '0020_auto_20231024_1825'),
|
|
||||||
]
|
|
||||||
|
|
||||||
operations = [
|
|
||||||
migrations.AddField(
|
|
||||||
model_name='source',
|
|
||||||
name='lightweight_metadata',
|
|
||||||
field=models.CharField(max_length=20,
|
|
||||||
default=Source.LIGHTWEIGHT_METADATA_TYPE_RAW,
|
|
||||||
choices=Source.LIGHTWEIGHT_METADATA_TYPE_CHOICES,
|
|
||||||
help_text='Lightweight metadata',
|
|
||||||
verbose_name='lightweight metadata'),
|
|
||||||
),
|
|
||||||
]
|
|
|
@ -18,7 +18,7 @@ from common.utils import clean_filename
|
||||||
from .youtube import (get_media_info as get_youtube_media_info,
|
from .youtube import (get_media_info as get_youtube_media_info,
|
||||||
download_media as download_youtube_media)
|
download_media as download_youtube_media)
|
||||||
from .utils import seconds_to_timestr, parse_media_format
|
from .utils import seconds_to_timestr, parse_media_format
|
||||||
from .matching import (get_best_combined_format, get_best_audio_format,
|
from .matching import (get_best_combined_format, get_best_audio_format,
|
||||||
get_best_video_format)
|
get_best_video_format)
|
||||||
from .mediaservers import PlexMediaServer
|
from .mediaservers import PlexMediaServer
|
||||||
from .fields import CommaSepChoiceField
|
from .fields import CommaSepChoiceField
|
||||||
|
@ -107,7 +107,6 @@ class Source(models.Model):
|
||||||
EXTENSION_MKV = 'mkv'
|
EXTENSION_MKV = 'mkv'
|
||||||
EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
|
EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
|
||||||
|
|
||||||
|
|
||||||
# as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
|
# as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
|
||||||
SPONSORBLOCK_CATEGORIES_CHOICES = (
|
SPONSORBLOCK_CATEGORIES_CHOICES = (
|
||||||
('sponsor', 'Sponsor'),
|
('sponsor', 'Sponsor'),
|
||||||
|
@ -119,17 +118,16 @@ class Source(models.Model):
|
||||||
('interaction', 'Interaction Reminder'),
|
('interaction', 'Interaction Reminder'),
|
||||||
('music_offtopic', 'Non-Music Section'),
|
('music_offtopic', 'Non-Music Section'),
|
||||||
)
|
)
|
||||||
|
|
||||||
sponsorblock_categories = CommaSepChoiceField(
|
|
||||||
_(''),
|
|
||||||
possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
|
|
||||||
all_choice="all",
|
|
||||||
allow_all=True,
|
|
||||||
all_label="(all options)",
|
|
||||||
default="all",
|
|
||||||
help_text=_("Select the sponsorblocks you want to enforce")
|
|
||||||
)
|
|
||||||
|
|
||||||
|
sponsorblock_categories = CommaSepChoiceField(
|
||||||
|
_(''),
|
||||||
|
possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
|
||||||
|
all_choice='all',
|
||||||
|
allow_all=True,
|
||||||
|
all_label='(all options)',
|
||||||
|
default='all',
|
||||||
|
help_text=_('Select the sponsorblocks you want to enforce')
|
||||||
|
)
|
||||||
embed_metadata = models.BooleanField(
|
embed_metadata = models.BooleanField(
|
||||||
_('embed metadata'),
|
_('embed metadata'),
|
||||||
default=False,
|
default=False,
|
||||||
|
@ -140,14 +138,12 @@ class Source(models.Model):
|
||||||
default=False,
|
default=False,
|
||||||
help_text=_('Embed thumbnail into the file')
|
help_text=_('Embed thumbnail into the file')
|
||||||
)
|
)
|
||||||
|
|
||||||
enable_sponsorblock = models.BooleanField(
|
enable_sponsorblock = models.BooleanField(
|
||||||
_('enable sponsorblock'),
|
_('enable sponsorblock'),
|
||||||
default=True,
|
default=True,
|
||||||
help_text=_('Use SponsorBlock?')
|
help_text=_('Use SponsorBlock?')
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# Fontawesome icons used for the source on the front end
|
# Fontawesome icons used for the source on the front end
|
||||||
ICONS = {
|
ICONS = {
|
||||||
SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
|
SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
|
||||||
|
@ -300,6 +296,11 @@ class Source(models.Model):
|
||||||
default=False,
|
default=False,
|
||||||
help_text=_('Delete media that is no longer on this playlist')
|
help_text=_('Delete media that is no longer on this playlist')
|
||||||
)
|
)
|
||||||
|
delete_files_on_disk = models.BooleanField(
|
||||||
|
_('delete files on disk'),
|
||||||
|
default=False,
|
||||||
|
help_text=_('Delete files on disk when they are removed from TubeSync')
|
||||||
|
)
|
||||||
source_resolution = models.CharField(
|
source_resolution = models.CharField(
|
||||||
_('source resolution'),
|
_('source resolution'),
|
||||||
max_length=8,
|
max_length=8,
|
||||||
|
@ -387,24 +388,6 @@ class Source(models.Model):
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
LIGHTWEIGHT_METADATA_TYPE_RAW = 'RAW'
|
|
||||||
LIGHTWEIGHT_METADATA_TYPE_UNNECESSARY = 'UNNECESSARY'
|
|
||||||
LIGHTWEIGHT_METADATA_TYPE_FEATHER = 'FEATHER'
|
|
||||||
LIGHTWEIGHT_METADATA_TYPES = (LIGHTWEIGHT_METADATA_TYPE_RAW, LIGHTWEIGHT_METADATA_TYPE_UNNECESSARY, LIGHTWEIGHT_METADATA_TYPE_FEATHER)
|
|
||||||
LIGHTWEIGHT_METADATA_TYPE_CHOICES = (
|
|
||||||
(LIGHTWEIGHT_METADATA_TYPE_RAW, _("(LARGE) Save raw metadata")),
|
|
||||||
(LIGHTWEIGHT_METADATA_TYPE_UNNECESSARY, _("(MEDIUM) Treeshake unnecessary metadata json keys")),
|
|
||||||
(LIGHTWEIGHT_METADATA_TYPE_FEATHER, _("(TINY) if the capacity is large, Treeshake it event if it is in use")),
|
|
||||||
)
|
|
||||||
|
|
||||||
lightweight_metadata = models.CharField(
|
|
||||||
_('lightweight metadata'),
|
|
||||||
max_length=20,
|
|
||||||
default=LIGHTWEIGHT_METADATA_TYPE_RAW,
|
|
||||||
choices=LIGHTWEIGHT_METADATA_TYPE_CHOICES,
|
|
||||||
help_text=_('Lightweight metadata')
|
|
||||||
)
|
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return self.name
|
return self.name
|
||||||
|
|
||||||
|
@ -568,7 +551,7 @@ class Source(models.Model):
|
||||||
if not self.filter_text:
|
if not self.filter_text:
|
||||||
return True
|
return True
|
||||||
return bool(re.search(self.filter_text, media_item_title))
|
return bool(re.search(self.filter_text, media_item_title))
|
||||||
|
|
||||||
def index_media(self):
|
def index_media(self):
|
||||||
'''
|
'''
|
||||||
Index the media source returning a list of media metadata as dicts.
|
Index the media source returning a list of media metadata as dicts.
|
||||||
|
@ -887,7 +870,7 @@ class Media(models.Model):
|
||||||
|
|
||||||
def get_best_video_format(self):
|
def get_best_video_format(self):
|
||||||
return get_best_video_format(self)
|
return get_best_video_format(self)
|
||||||
|
|
||||||
def get_format_str(self):
|
def get_format_str(self):
|
||||||
'''
|
'''
|
||||||
Returns a youtube-dl compatible format string for the best matches
|
Returns a youtube-dl compatible format string for the best matches
|
||||||
|
@ -912,7 +895,7 @@ class Media(models.Model):
|
||||||
else:
|
else:
|
||||||
return False
|
return False
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def get_display_format(self, format_str):
|
def get_display_format(self, format_str):
|
||||||
'''
|
'''
|
||||||
Returns a tuple used in the format component of the output filename. This
|
Returns a tuple used in the format component of the output filename. This
|
||||||
|
@ -1203,7 +1186,7 @@ class Media(models.Model):
|
||||||
filename = self.filename
|
filename = self.filename
|
||||||
prefix, ext = os.path.splitext(filename)
|
prefix, ext = os.path.splitext(filename)
|
||||||
return f'{prefix}.nfo'
|
return f'{prefix}.nfo'
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def nfopath(self):
|
def nfopath(self):
|
||||||
return self.source.directory_path / self.nfoname
|
return self.source.directory_path / self.nfoname
|
||||||
|
@ -1216,7 +1199,7 @@ class Media(models.Model):
|
||||||
filename = self.filename
|
filename = self.filename
|
||||||
prefix, ext = os.path.splitext(filename)
|
prefix, ext = os.path.splitext(filename)
|
||||||
return f'{prefix}.info.json'
|
return f'{prefix}.info.json'
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def jsonpath(self):
|
def jsonpath(self):
|
||||||
return self.source.directory_path / self.jsonname
|
return self.source.directory_path / self.jsonname
|
||||||
|
@ -1254,7 +1237,6 @@ class Media(models.Model):
|
||||||
acodec = self.downloaded_audio_codec
|
acodec = self.downloaded_audio_codec
|
||||||
if acodec is None:
|
if acodec is None:
|
||||||
raise TypeError() # nothing here.
|
raise TypeError() # nothing here.
|
||||||
|
|
||||||
acodec = acodec.lower()
|
acodec = acodec.lower()
|
||||||
if acodec == "mp4a":
|
if acodec == "mp4a":
|
||||||
return "audio/mp4"
|
return "audio/mp4"
|
||||||
|
@ -1263,7 +1245,6 @@ class Media(models.Model):
|
||||||
else:
|
else:
|
||||||
# fall-fall-back.
|
# fall-fall-back.
|
||||||
return 'audio/ogg'
|
return 'audio/ogg'
|
||||||
|
|
||||||
vcodec = vcodec.lower()
|
vcodec = vcodec.lower()
|
||||||
if vcodec == 'vp9':
|
if vcodec == 'vp9':
|
||||||
return 'video/webm'
|
return 'video/webm'
|
||||||
|
@ -1287,6 +1268,22 @@ class Media(models.Model):
|
||||||
showtitle.text = str(self.source.name).strip()
|
showtitle.text = str(self.source.name).strip()
|
||||||
showtitle.tail = '\n '
|
showtitle.tail = '\n '
|
||||||
nfo.append(showtitle)
|
nfo.append(showtitle)
|
||||||
|
# season = upload date year
|
||||||
|
season = nfo.makeelement('season', {})
|
||||||
|
if self.source.source_type == Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
|
||||||
|
# If it's a playlist, set season to 1
|
||||||
|
season.text = '1'
|
||||||
|
else:
|
||||||
|
# If it's not a playlist, set season to upload date year
|
||||||
|
season.text = str(self.upload_date.year) if self.upload_date else ''
|
||||||
|
season.tail = '\n '
|
||||||
|
nfo.append(season)
|
||||||
|
# episode = number of video in the year
|
||||||
|
episode = nfo.makeelement('episode', {})
|
||||||
|
episode_number = self.calculate_episode_number()
|
||||||
|
episode.text = str(episode_number) if episode_number else ''
|
||||||
|
episode.tail = '\n '
|
||||||
|
nfo.append(episode)
|
||||||
# ratings = media metadata youtube rating
|
# ratings = media metadata youtube rating
|
||||||
value = nfo.makeelement('value', {})
|
value = nfo.makeelement('value', {})
|
||||||
value.text = str(self.rating)
|
value.text = str(self.rating)
|
||||||
|
@ -1394,8 +1391,8 @@ class Media(models.Model):
|
||||||
f'no valid format available')
|
f'no valid format available')
|
||||||
# Download the media with youtube-dl
|
# Download the media with youtube-dl
|
||||||
download_youtube_media(self.url, format_str, self.source.extension,
|
download_youtube_media(self.url, format_str, self.source.extension,
|
||||||
str(self.filepath), self.source.write_json,
|
str(self.filepath), self.source.write_json,
|
||||||
self.source.sponsorblock_categories, self.source.embed_thumbnail,
|
self.source.sponsorblock_categories.selected_choices, self.source.embed_thumbnail,
|
||||||
self.source.embed_metadata, self.source.enable_sponsorblock,
|
self.source.embed_metadata, self.source.enable_sponsorblock,
|
||||||
self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs )
|
self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs )
|
||||||
# Return the download paramaters
|
# Return the download paramaters
|
||||||
|
@ -1411,6 +1408,19 @@ class Media(models.Model):
|
||||||
f'has no indexer')
|
f'has no indexer')
|
||||||
return indexer(self.url)
|
return indexer(self.url)
|
||||||
|
|
||||||
|
def calculate_episode_number(self):
|
||||||
|
if self.source.source_type == Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
|
||||||
|
sorted_media = Media.objects.filter(source=self.source)
|
||||||
|
else:
|
||||||
|
self_year = self.upload_date.year if self.upload_date else self.created.year
|
||||||
|
filtered_media = Media.objects.filter(source=self.source, published__year=self_year)
|
||||||
|
sorted_media = sorted(filtered_media, key=lambda x: (x.upload_date, x.key))
|
||||||
|
position_counter = 1
|
||||||
|
for media in sorted_media:
|
||||||
|
if media == self:
|
||||||
|
return position_counter
|
||||||
|
position_counter += 1
|
||||||
|
|
||||||
|
|
||||||
class MediaServer(models.Model):
|
class MediaServer(models.Model):
|
||||||
'''
|
'''
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import os
|
import os
|
||||||
|
import glob
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
|
from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
|
||||||
from django.dispatch import receiver
|
from django.dispatch import receiver
|
||||||
|
@ -74,6 +75,7 @@ def source_pre_delete(sender, instance, **kwargs):
|
||||||
media.delete()
|
media.delete()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@receiver(post_delete, sender=Source)
|
@receiver(post_delete, sender=Source)
|
||||||
def source_post_delete(sender, instance, **kwargs):
|
def source_post_delete(sender, instance, **kwargs):
|
||||||
# Triggered after a source is deleted
|
# Triggered after a source is deleted
|
||||||
|
@ -222,6 +224,16 @@ def media_pre_delete(sender, instance, **kwargs):
|
||||||
if thumbnail_url:
|
if thumbnail_url:
|
||||||
delete_task_by_media('sync.tasks.download_media_thumbnail',
|
delete_task_by_media('sync.tasks.download_media_thumbnail',
|
||||||
(str(instance.pk), thumbnail_url))
|
(str(instance.pk), thumbnail_url))
|
||||||
|
if instance.source.delete_files_on_disk and (instance.media_file or instance.thumb):
|
||||||
|
# Delete all media files if it contains filename
|
||||||
|
filepath = instance.media_file.path if instance.media_file else instance.thumb.path
|
||||||
|
barefilepath, fileext = os.path.splitext(filepath)
|
||||||
|
# Get all files that start with the bare file path
|
||||||
|
all_related_files = glob.glob(f'{barefilepath}.*')
|
||||||
|
for file in all_related_files:
|
||||||
|
log.info(f'Deleting file for: {instance} path: {file}')
|
||||||
|
delete_file(file)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@receiver(post_delete, sender=Media)
|
@receiver(post_delete, sender=Media)
|
||||||
|
|
|
@ -236,12 +236,6 @@ def download_media_metadata(media_id):
|
||||||
return
|
return
|
||||||
source = media.source
|
source = media.source
|
||||||
metadata = media.index_metadata()
|
metadata = media.index_metadata()
|
||||||
if source.lightweight_metadata == Source.LIGHTWEIGHT_METADATA_TYPE_FEATHER:
|
|
||||||
del metadata["formats"]
|
|
||||||
del metadata["thumbnails"]
|
|
||||||
del metadata["automatic_captions"]
|
|
||||||
del metadata["requested_formats"]
|
|
||||||
del metadata["heatmap"]
|
|
||||||
media.metadata = json.dumps(metadata, default=json_serial)
|
media.metadata = json.dumps(metadata, default=json_serial)
|
||||||
upload_date = media.upload_date
|
upload_date = media.upload_date
|
||||||
# Media must have a valid upload date
|
# Media must have a valid upload date
|
||||||
|
|
|
@ -132,8 +132,6 @@
|
||||||
<td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if media.can_download %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
<td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if media.can_download %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
||||||
</tr>
|
</tr>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
{% if media.source.lightweight_metadata == "RAW" %}
|
|
||||||
<tr title="The available media formats">
|
<tr title="The available media formats">
|
||||||
<td class="hide-on-small-only">Available formats</td>
|
<td class="hide-on-small-only">Available formats</td>
|
||||||
<td><span class="hide-on-med-and-up">Available formats<br></span>
|
<td><span class="hide-on-med-and-up">Available formats<br></span>
|
||||||
|
@ -157,10 +155,7 @@
|
||||||
Video: <strong>{% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}
|
Video: <strong>{% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}
|
||||||
</strong></td>
|
</strong></td>
|
||||||
</tr>
|
</tr>
|
||||||
{% endif %}
|
|
||||||
</table>
|
</table>
|
||||||
<p>{{ media.source.lightweight_metadata }}</p>
|
|
||||||
<p>{{ media.source }}</p>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{% if media.downloaded %}
|
{% if media.downloaded %}
|
||||||
|
|
|
@ -9,8 +9,8 @@
|
||||||
<p>
|
<p>
|
||||||
Are you sure you want to delete this source? Deleting a source is permanent.
|
Are you sure you want to delete this source? Deleting a source is permanent.
|
||||||
By default, deleting a source does not delete any saved media files. You can
|
By default, deleting a source does not delete any saved media files. You can
|
||||||
tick the "also delete downloaded media" checkbox to also remove save
|
<strong>tick the "also delete downloaded media" checkbox to also remove directory {{ source.directory_path }}
|
||||||
media when you delete the source. Deleting a source cannot be undone.
|
</strong>when you delete the source. Deleting a source cannot be undone.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -122,6 +122,10 @@
|
||||||
<tr title="Delete media that is no longer on this playlist?">
|
<tr title="Delete media that is no longer on this playlist?">
|
||||||
<td class="hide-on-small-only">Delete removed media</td>
|
<td class="hide-on-small-only">Delete removed media</td>
|
||||||
<td><span class="hide-on-med-and-up">Delete removed media<br></span><strong>{% if source.delete_removed_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
<td><span class="hide-on-med-and-up">Delete removed media<br></span><strong>{% if source.delete_removed_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
||||||
|
</tr>
|
||||||
|
<tr title="Delete files on disk when they are removed from TubeSync?">
|
||||||
|
<td class="hide-on-small-only">Delete files on disk</td>
|
||||||
|
<td><span class="hide-on-med-and-up">Delete files on disk<br></span><strong>{% if source.delete_files_on_disk %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
||||||
</tr>
|
</tr>
|
||||||
{% if source.delete_old_media and source.days_to_keep > 0 %}
|
{% if source.delete_old_media and source.days_to_keep > 0 %}
|
||||||
<tr title="Days after which your media from this source will be locally deleted">
|
<tr title="Days after which your media from this source will be locally deleted">
|
||||||
|
@ -186,17 +190,7 @@
|
||||||
<td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
|
<td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
|
||||||
</tr>
|
</tr>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
{% if source.lightweight_metadata %}
|
|
||||||
<tr title="{{ _('Are auto subs accepted?') }}">
|
|
||||||
<td class="hide-on-small-only">{{ _("Auto-generated subtitles?") }}:</td>
|
|
||||||
<td><span class="hide-on-med-and-up">{{ _("Auto-generated subtitles?") }}:</span><strong><i class="fas {% if source.auto_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
|
|
||||||
</tr>
|
|
||||||
<tr title="{{ _('Subs langs?') }}">
|
|
||||||
<td class="hide-on-small-only">{{ _("Subs langs?") }}:</td>
|
|
||||||
<td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
|
|
||||||
</tr>
|
|
||||||
{% endif %}
|
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -661,6 +661,8 @@ class MediaTestCase(TestCase):
|
||||||
'<episodedetails>',
|
'<episodedetails>',
|
||||||
' <title>no fancy stuff title</title>',
|
' <title>no fancy stuff title</title>',
|
||||||
' <showtitle>testname</showtitle>',
|
' <showtitle>testname</showtitle>',
|
||||||
|
' <season>2017</season>',
|
||||||
|
' <episode></episode>',
|
||||||
' <ratings>',
|
' <ratings>',
|
||||||
' <rating default="True" max="5" name="youtube">',
|
' <rating default="True" max="5" name="youtube">',
|
||||||
' <value>1.2345</value>',
|
' <value>1.2345</value>',
|
||||||
|
|
|
@ -1,7 +1,9 @@
|
||||||
|
import glob
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
from base64 import b64decode
|
from base64 import b64decode
|
||||||
import pathlib
|
import pathlib
|
||||||
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect
|
from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect
|
||||||
|
@ -59,7 +61,7 @@ class DashboardView(TemplateView):
|
||||||
# Disk usage
|
# Disk usage
|
||||||
disk_usage = Media.objects.filter(
|
disk_usage = Media.objects.filter(
|
||||||
downloaded=True, downloaded_filesize__isnull=False
|
downloaded=True, downloaded_filesize__isnull=False
|
||||||
).aggregate(Sum('downloaded_filesize'))
|
).defer('metadata').aggregate(Sum('downloaded_filesize'))
|
||||||
data['disk_usage_bytes'] = disk_usage['downloaded_filesize__sum']
|
data['disk_usage_bytes'] = disk_usage['downloaded_filesize__sum']
|
||||||
if not data['disk_usage_bytes']:
|
if not data['disk_usage_bytes']:
|
||||||
data['disk_usage_bytes'] = 0
|
data['disk_usage_bytes'] = 0
|
||||||
|
@ -71,11 +73,11 @@ class DashboardView(TemplateView):
|
||||||
# Latest downloads
|
# Latest downloads
|
||||||
data['latest_downloads'] = Media.objects.filter(
|
data['latest_downloads'] = Media.objects.filter(
|
||||||
downloaded=True, downloaded_filesize__isnull=False
|
downloaded=True, downloaded_filesize__isnull=False
|
||||||
).order_by('-download_date')[:10]
|
).defer('metadata').order_by('-download_date')[:10]
|
||||||
# Largest downloads
|
# Largest downloads
|
||||||
data['largest_downloads'] = Media.objects.filter(
|
data['largest_downloads'] = Media.objects.filter(
|
||||||
downloaded=True, downloaded_filesize__isnull=False
|
downloaded=True, downloaded_filesize__isnull=False
|
||||||
).order_by('-downloaded_filesize')[:10]
|
).defer('metadata').order_by('-downloaded_filesize')[:10]
|
||||||
# UID and GID
|
# UID and GID
|
||||||
data['uid'] = os.getuid()
|
data['uid'] = os.getuid()
|
||||||
data['gid'] = os.getgid()
|
data['gid'] = os.getgid()
|
||||||
|
@ -296,11 +298,11 @@ class EditSourceMixin:
|
||||||
model = Source
|
model = Source
|
||||||
fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format',
|
fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format',
|
||||||
'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
|
'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
|
||||||
'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec',
|
'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution',
|
||||||
'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails',
|
'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback',
|
||||||
'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
|
'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
|
||||||
'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
|
'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
|
||||||
'auto_subtitles', 'sub_langs', 'lightweight_metadata')
|
'auto_subtitles', 'sub_langs')
|
||||||
errors = {
|
errors = {
|
||||||
'invalid_media_format': _('Invalid media format, the media format contains '
|
'invalid_media_format': _('Invalid media format, the media format contains '
|
||||||
'errors or is empty. Check the table at the end of '
|
'errors or is empty. Check the table at the end of '
|
||||||
|
@ -404,7 +406,7 @@ class SourceView(DetailView):
|
||||||
error_message = get_error_message(error)
|
error_message = get_error_message(error)
|
||||||
setattr(error, 'error_message', error_message)
|
setattr(error, 'error_message', error_message)
|
||||||
data['errors'].append(error)
|
data['errors'].append(error)
|
||||||
data['media'] = Media.objects.filter(source=self.object).order_by('-published')
|
data['media'] = Media.objects.filter(source=self.object).order_by('-published').defer('metadata')
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
@ -435,14 +437,13 @@ class DeleteSourceView(DeleteView, FormMixin):
|
||||||
source = self.get_object()
|
source = self.get_object()
|
||||||
for media in Media.objects.filter(source=source):
|
for media in Media.objects.filter(source=source):
|
||||||
if media.media_file:
|
if media.media_file:
|
||||||
# Delete the media file
|
file_path = media.media_file.path
|
||||||
delete_file(media.media_file.path)
|
matching_files = glob.glob(os.path.splitext(file_path)[0] + '.*')
|
||||||
# Delete thumbnail copy if it exists
|
for file in matching_files:
|
||||||
delete_file(media.thumbpath)
|
delete_file(file)
|
||||||
# Delete NFO file if it exists
|
directory_path = source.directory_path
|
||||||
delete_file(media.nfopath)
|
if os.path.exists(directory_path):
|
||||||
# Delete JSON file if it exists
|
shutil.rmtree(directory_path, True)
|
||||||
delete_file(media.jsonpath)
|
|
||||||
return super().post(request, *args, **kwargs)
|
return super().post(request, *args, **kwargs)
|
||||||
|
|
||||||
def get_success_url(self):
|
def get_success_url(self):
|
||||||
|
@ -653,12 +654,13 @@ class MediaSkipView(FormView, SingleObjectMixin):
|
||||||
delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),))
|
delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),))
|
||||||
# If the media file exists on disk, delete it
|
# If the media file exists on disk, delete it
|
||||||
if self.object.media_file_exists:
|
if self.object.media_file_exists:
|
||||||
delete_file(self.object.media_file.path)
|
# Delete all files which contains filename
|
||||||
self.object.media_file = None
|
filepath = self.object.media_file.path
|
||||||
# If the media has an associated thumbnail copied, also delete it
|
barefilepath, fileext = os.path.splitext(filepath)
|
||||||
delete_file(self.object.thumbpath)
|
# Get all files that start with the bare file path
|
||||||
# If the media has an associated NFO file with it, also delete it
|
all_related_files = glob.glob(f'{barefilepath}.*')
|
||||||
delete_file(self.object.nfopath)
|
for file in all_related_files:
|
||||||
|
delete_file(file)
|
||||||
# Reset all download data
|
# Reset all download data
|
||||||
self.object.metadata = None
|
self.object.metadata = None
|
||||||
self.object.downloaded = False
|
self.object.downloaded = False
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
'''
|
'''
|
||||||
Wrapper for the youtube-dl library. Used so if there are any library interface
|
Wrapper for the yt-dlp library. Used so if there are any library interface
|
||||||
updates we only need to udpate them in one place.
|
updates we only need to udpate them in one place.
|
||||||
'''
|
'''
|
||||||
|
|
||||||
|
@ -64,9 +64,9 @@ def get_media_info(url):
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
def download_media(url, media_format, extension, output_file, info_json,
|
def download_media(url, media_format, extension, output_file, info_json,
|
||||||
sponsor_categories="all",
|
sponsor_categories=None,
|
||||||
embed_thumbnail=False, embed_metadata=False, skip_sponsors=True,
|
embed_thumbnail=False, embed_metadata=False, skip_sponsors=True,
|
||||||
write_subtitles=False, auto_subtitles=False, sub_langs='en'):
|
write_subtitles=False, auto_subtitles=False, sub_langs='en'):
|
||||||
'''
|
'''
|
||||||
Downloads a YouTube URL to a file on disk.
|
Downloads a YouTube URL to a file on disk.
|
||||||
|
@ -74,7 +74,7 @@ def download_media(url, media_format, extension, output_file, info_json,
|
||||||
|
|
||||||
def hook(event):
|
def hook(event):
|
||||||
filename = os.path.basename(event['filename'])
|
filename = os.path.basename(event['filename'])
|
||||||
|
|
||||||
if event.get('downloaded_bytes') is None or event.get('total_bytes') is None:
|
if event.get('downloaded_bytes') is None or event.get('total_bytes') is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@ -106,8 +106,8 @@ def download_media(url, media_format, extension, output_file, info_json,
|
||||||
f'{total_size_str} in {elapsed_str}')
|
f'{total_size_str} in {elapsed_str}')
|
||||||
else:
|
else:
|
||||||
log.warn(f'[youtube-dl] unknown event: {str(event)}')
|
log.warn(f'[youtube-dl] unknown event: {str(event)}')
|
||||||
hook.download_progress = 0
|
|
||||||
|
|
||||||
|
hook.download_progress = 0
|
||||||
ytopts = {
|
ytopts = {
|
||||||
'format': media_format,
|
'format': media_format,
|
||||||
'merge_output_format': extension,
|
'merge_output_format': extension,
|
||||||
|
@ -120,29 +120,25 @@ def download_media(url, media_format, extension, output_file, info_json,
|
||||||
'writeautomaticsub': auto_subtitles,
|
'writeautomaticsub': auto_subtitles,
|
||||||
'subtitleslangs': sub_langs.split(','),
|
'subtitleslangs': sub_langs.split(','),
|
||||||
}
|
}
|
||||||
|
if not sponsor_categories:
|
||||||
|
sponsor_categories = []
|
||||||
sbopt = {
|
sbopt = {
|
||||||
'key': 'SponsorBlock',
|
'key': 'SponsorBlock',
|
||||||
'categories': [sponsor_categories]
|
'categories': sponsor_categories
|
||||||
}
|
}
|
||||||
ffmdopt = {
|
ffmdopt = {
|
||||||
'key': 'FFmpegMetadata',
|
'key': 'FFmpegMetadata',
|
||||||
'add_chapters': True,
|
'add_chapters': embed_metadata,
|
||||||
'add_metadata': True
|
'add_metadata': embed_metadata
|
||||||
}
|
}
|
||||||
|
|
||||||
opts = get_yt_opts()
|
opts = get_yt_opts()
|
||||||
if embed_thumbnail:
|
if embed_thumbnail:
|
||||||
ytopts['postprocessors'].append({'key': 'EmbedThumbnail'})
|
ytopts['postprocessors'].append({'key': 'EmbedThumbnail'})
|
||||||
if embed_metadata:
|
|
||||||
ffmdopt["add_metadata"] = True
|
|
||||||
if skip_sponsors:
|
if skip_sponsors:
|
||||||
ytopts['postprocessors'].append(sbopt)
|
ytopts['postprocessors'].append(sbopt)
|
||||||
|
|
||||||
ytopts['postprocessors'].append(ffmdopt)
|
ytopts['postprocessors'].append(ffmdopt)
|
||||||
|
|
||||||
opts.update(ytopts)
|
opts.update(ytopts)
|
||||||
|
|
||||||
with yt_dlp.YoutubeDL(opts) as y:
|
with yt_dlp.YoutubeDL(opts) as y:
|
||||||
try:
|
try:
|
||||||
return y.download([url])
|
return y.download([url])
|
||||||
|
|
|
@ -25,9 +25,6 @@ DEBUG = True if os.getenv('TUBESYNC_DEBUG', False) else False
|
||||||
FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX)
|
FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX)
|
||||||
|
|
||||||
|
|
||||||
TIME_ZONE = os.getenv('TZ', 'UTC')
|
|
||||||
|
|
||||||
|
|
||||||
database_dict = {}
|
database_dict = {}
|
||||||
database_connection_env = os.getenv('DATABASE_CONNECTION', '')
|
database_connection_env = os.getenv('DATABASE_CONNECTION', '')
|
||||||
if database_connection_env:
|
if database_connection_env:
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
@ -96,7 +97,7 @@ AUTH_PASSWORD_VALIDATORS = [
|
||||||
|
|
||||||
|
|
||||||
LANGUAGE_CODE = 'en-us'
|
LANGUAGE_CODE = 'en-us'
|
||||||
TIME_ZONE = 'UTC'
|
TIME_ZONE = os.getenv('TZ', 'UTC')
|
||||||
USE_I18N = True
|
USE_I18N = True
|
||||||
USE_L10N = True
|
USE_L10N = True
|
||||||
USE_TZ = True
|
USE_TZ = True
|
||||||
|
|
Loading…
Reference in New Issue