PublicSnapshot
/
tubesync
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add XML NFO file writing support, rework media cleanup deletion, resolves #11

This commit is contained in:
meeb 2020-12-19 16:00:37 +11:00
parent 8f4b09f346
commit 410906ad8e
10 changed files with 327 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
tubesync/sync/migrations/0005_auto_20201219_0312.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 3.1.4 on 2020-12-19 03:12
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sync', '0004_source_media_format'),
]
operations = [
migrations.AlterField(
model_name='source',
name='source_type',
field=models.CharField(choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type'),
),
]

18
tubesync/sync/migrations/0006_source_write_nfo.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 3.1.4 on 2020-12-19 03:12
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sync', '0005_auto_20201219_0312'),
]
operations = [
migrations.AddField(
model_name='source',
name='write_nfo',
field=models.BooleanField(default=False, help_text='Write an NFO file with the media, these may be detected and used by some media servers', verbose_name='write nfo'),
),
]

179
tubesync/sync/models.py
View File

@ -1,6 +1,7 @@
import os
import uuid
import json
from xml.etree import ElementTree
from datetime import datetime
from pathlib import Path
from django.conf import settings
@ -253,6 +254,11 @@ class Source(models.Model):
default=False,
help_text=_('Copy thumbnails with the media, these may be detected and used by some media servers')
)
write_nfo = models.BooleanField(
_('write nfo'),
default=False,
help_text=_('Write an NFO file in XML with the media info, these may be detected and used by some media servers')
)
has_failed = models.BooleanField(
_('has failed'),
default=False,
@ -476,7 +482,38 @@ class Media(models.Model):
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'formats',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'formats',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats',
}
},
'categories': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'categories',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'categories',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'categories',
},
'rating': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'average_rating',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'average_rating',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'average_rating',
},
'age_limit': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'age_limit',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'age_limit',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'age_limit',
},
'uploader': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'uploader',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'uploader',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'uploader',
},
'upvotes': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'like_count',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'like_count',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'like_count',
},
'downvotes': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'dislike_count',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count',
},
}
STATE_UNKNOWN = 'unknown'
STATE_SCHEDULED = 'scheduled'
@ -883,6 +920,34 @@ class Media(models.Model):
return seconds_to_timestr(duration)
return '??:??:??'
@property
def categories(self):
field = self.get_metadata_field('categories')
return self.loaded_metadata.get(field, [])
@property
def rating(self):
field = self.get_metadata_field('rating')
return self.loaded_metadata.get(field, 0)
@property
def votes(self):
field = self.get_metadata_field('upvotes')
upvotes = self.loaded_metadata.get(field, 0)
field = self.get_metadata_field('downvotes')
downvotes = self.loaded_metadata.get(field, 0)
return upvotes + downvotes
@property
def age_limit(self):
field = self.get_metadata_field('age_limit')
return self.loaded_metadata.get(field, 0)
@property
def uploader(self):
field = self.get_metadata_field('uploader')
return self.loaded_metadata.get(field, '')
@property
def formats(self):
field = self.get_metadata_field('formats')
@ -898,6 +963,26 @@ class Media(models.Model):
media_details = self.format_dict
return media_format.format(**media_details)
@property
def thumbname(self):
filename = self.filename
prefix, ext = os.path.splitext(filename)
return f'{prefix}.jpg'
@property
def thumbpath(self):
return self.source.directory_path / self.thumbname
@property
def nfoname(self):
filename = self.filename
prefix, ext = os.path.splitext(filename)
return f'{prefix}.nfo'
@property
def nfopath(self):
return self.source.directory_path / self.nfoname
@property
def directory_path(self):
# If a media_file has been downloaded use its existing directory
@ -925,6 +1010,98 @@ class Media(models.Model):
return False
return os.path.exists(self.media_file.path)
@property
def nfoxml(self):
'''
Returns an NFO formatted (prettified) XML string.
'''
nfo = ElementTree.Element('episodedetails')
nfo.text = '\n '
# title = media metadata title
title = nfo.makeelement('title', {})
title.text = str(self.name).strip()
title.tail = '\n '
nfo.append(title)
# showtitle = source name
showtitle = nfo.makeelement('showtitle', {})
showtitle.text = str(self.source.name).strip()
showtitle.tail = '\n '
nfo.append(showtitle)
# ratings = media metadata youtube rating
value = nfo.makeelement('value', {})
value.text = str(self.rating)
value.tail = '\n '
votes = nfo.makeelement('votes', {})
votes.text = str(self.votes)
votes.tail = '\n '
rating_attrs = {'name': 'youtube', 'max': '5', 'default': 'True'}
rating = nfo.makeelement('rating', rating_attrs)
rating.text = '\n '
rating.append(value)
rating.append(votes)
rating.tail = '\n '
ratings = nfo.makeelement('ratings', {})
ratings.text = '\n '
ratings.append(rating)
ratings.tail = '\n '
nfo.append(ratings)
# plot = media metadata description
plot = nfo.makeelement('plot', {})
plot.text = str(self.description).strip()
plot.tail = '\n '
nfo.append(plot)
# thumb = local path to media thumbnail
thumb = nfo.makeelement('thumb', {})
thumb.text = self.thumbname if self.source.copy_thumbnails else ''
thumb.tail = '\n '
nfo.append(thumb)
# mpaa = media metadata age requirement
mpaa = nfo.makeelement('mpaa', {})
mpaa.text = str(self.age_limit)
mpaa.tail = '\n '
nfo.append(mpaa)
# runtime = media metadata duration in seconds
runtime = nfo.makeelement('runtime', {})
runtime.text = str(self.duration)
runtime.tail = '\n '
nfo.append(runtime)
# id = media key
idn = nfo.makeelement('id', {})
idn.text = str(self.key).strip()
idn.tail = '\n '
nfo.append(idn)
# uniqueid = media key
uniqueid_attrs = {'type': 'youtube', 'default': 'True'}
uniqueid = nfo.makeelement('uniqueid', uniqueid_attrs)
uniqueid.text = str(self.key).strip()
uniqueid.tail = '\n '
nfo.append(uniqueid)
# studio = media metadata uploader
studio = nfo.makeelement('studio', {})
studio.text = str(self.uploader).strip()
studio.tail = '\n '
nfo.append(studio)
# aired = media metadata uploaded date
aired = nfo.makeelement('aired', {})
upload_date = self.upload_date
aired.text = upload_date.strftime('%Y-%m-%d') if upload_date else ''
aired.tail = '\n '
nfo.append(aired)
# dateadded = date and time media was created in tubesync
dateadded = nfo.makeelement('dateadded', {})
dateadded.text = self.created.strftime('%Y-%m-%d %H:%M:%S')
dateadded.tail = '\n '
nfo.append(dateadded)
# genre = any media metadata categories if they exist
for category_str in self.categories:
genre = nfo.makeelement('genre', {})
genre.text = str(category_str).strip()
genre.tail = '\n '
nfo.append(genre)
nfo[-1].tail = '\n'
# Return XML tree as a prettified string
return ElementTree.tostring(nfo, encoding='utf8', method='xml').decode('utf8')
def get_download_state(self, task=None):
if self.downloaded:
return self.STATE_DOWNLOADED

14
tubesync/sync/signals.py
View File

@ -145,20 +145,6 @@ def media_pre_delete(sender, instance, **kwargs):
if thumbnail_url:
delete_task_by_media('sync.tasks.download_media_thumbnail',
(str(instance.pk), thumbnail_url))
# Delete media thumbnail if it exists
if instance.thumb:
log.info(f'Deleting thumbnail for: {instance} path: {instance.thumb.path}')
delete_file(instance.thumb.path)
# Delete the media file if it exists
if instance.media_file:
filepath = instance.media_file.path
log.info(f'Deleting media for: {instance} path: {filepath}')
delete_file(filepath)
# Delete thumbnail copy if it exists
barefilepath, fileext = os.path.splitext(filepath)
thumbpath = f'{barefilepath}.jpg'
log.info(f'Deleting thumbnail for: {instance} path: {thumbpath}')
delete_file(thumbpath)
@receiver(post_delete, sender=Media)

13
tubesync/sync/tasks.py
View File

@ -23,7 +23,8 @@ from background_task.models import Task, CompletedTask
from common.logger import log
from common.errors import NoMediaException, DownloadFailedException
from .models import Source, Media, MediaServer
from .utils import get_remote_image, resize_image_to_height, delete_file
from .utils import (get_remote_image, resize_image_to_height, delete_file,
write_text_file)
def get_hash(task_name, pk):
@ -317,11 +318,13 @@ def download_media(media_id):
media.save()
# If selected, copy the thumbnail over as well
if media.source.copy_thumbnails and media.thumb:
barefilepath, fileext = os.path.splitext(filepath)
thumbpath = f'{barefilepath}.jpg'
log.info(f'Copying media thumbnail from: {media.thumb.path} '
f'to: {thumbpath}')
copyfile(media.thumb.path, thumbpath)
f'to: {media.thumbpath}')
copyfile(media.thumb.path, media.thumbpath)
# If selected, write an NFO file
if media.source.write_nfo:
log.info(f'Writing media NFO file to: to: {media.nfopath}')
write_text_file(media.nfopath, media.nfoxml)
# Schedule a task to update media servers
for mediaserver in MediaServer.objects.all():
log.info(f'Scheduling media server updates')

4
tubesync/sync/templates/sync/source.html
View File

@ -97,6 +97,10 @@
<td class="hide-on-small-only">Copy thumbnails?</td>
<td><span class="hide-on-med-and-up">Copy thumbnails?<br></span><strong>{% if source.copy_thumbnails %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr>
<tr title="Should an NFO file be written with the media?">
<td class="hide-on-small-only">Write NFO?</td>
<td><span class="hide-on-med-and-up">Write NFO?<br></span><strong>{% if source.write_nfo %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr>
{% if source.delete_old_media and source.days_to_keep > 0 %}
<tr title="Days after which your media from this source will be locally deleted">
<td class="hide-on-small-only">Delete old media</td>

15
tubesync/sync/testdata/metadata.json vendored
View File

@ -3,15 +3,22 @@
"upload_date":"20170911",
"license":null,
"creator":null,
"title":"no fancy stuff",
"title":"no fancy stuff title",
"alt_title":null,
"description":"no fancy stuff",
"categories":[],
"description":"no fancy stuff desc",
"average_rating": 1.2345,
"dislike_count": 123,
"like_count": 456,
"uploader": "test uploader",
"categories":[
"test category 1",
"test category 2"
],
"tags":[],
"subtitles":{},
"automatic_captions":{},
"duration":401.0,
"age_limit":0,
"age_limit":50,
"annotations":null,
"chapters":null,
"formats":[

68
tubesync/sync/tests.py
View File

@ -6,6 +6,7 @@
import logging
from datetime import datetime
from urllib.parse import urlsplit
from django.conf import settings
from django.test import TestCase, Client
@ -430,7 +431,6 @@ class FrontEndTestCase(TestCase):
response = c.get('/tasks-completed')
self.assertEqual(response.status_code, 200)
def test_mediasevrers(self):
# Media servers overview page
c = Client()
@ -588,7 +588,71 @@ class FilepathTestCase(TestCase):
self.source.media_format = ('{title}_{key}_{resolution}-{height}x{width}-'
'{acodec}-{vcodec}-{fps}fps-{hdr}.{ext}')
self.assertEqual(test_media.filename,
'no-fancy-stuff_test_720p-720x1280-opus-vp9-30fps-hdr.mkv')
('no-fancy-stuff-title_test_720p-720x1280-opus'
'-vp9-30fps-hdr.mkv'))
class MediaTestCase(TestCase):
def setUp(self):
# Disable general logging for test case
logging.disable(logging.CRITICAL)
# Add a test source
self.source = Source.objects.create(
source_type=Source.SOURCE_TYPE_YOUTUBE_CHANNEL,
key='testkey',
name='testname',
directory='testdirectory',
media_format=settings.MEDIA_FORMATSTR_DEFAULT,
index_schedule=3600,
delete_old_media=False,
days_to_keep=14,
source_resolution=Source.SOURCE_RESOLUTION_1080P,
source_vcodec=Source.SOURCE_VCODEC_VP9,
source_acodec=Source.SOURCE_ACODEC_OPUS,
prefer_60fps=False,
prefer_hdr=False,
fallback=Source.FALLBACK_FAIL
)
# Add some test media
self.media = Media.objects.create(
key='mediakey',
source=self.source,
metadata=metadata,
)
# Fix a created datetime for predictable testing
self.media.created = datetime(year=2020, month=1, day=1, hour=1,
minute=1, second=1)
def test_nfo(self):
expected_nfo = [
"<?xml version='1.0' encoding='utf8'?>",
'<episodedetails>',
' <title>no fancy stuff title</title>',
' <showtitle>testname</showtitle>',
' <ratings>',
' <rating default="True" max="5" name="youtube">',
' <value>1.2345</value>',
' <votes>579</votes>',
' </rating>',
' </ratings>',
' <plot>no fancy stuff desc</plot>',
' <thumb />', # media.thumbfile is empty without media existing
' <mpaa>50</mpaa>',
' <runtime>401</runtime>',
' <id>mediakey</id>',
' <uniqueid default="True" type="youtube">mediakey</uniqueid>',
' <studio>test uploader</studio>',
' <aired>2017-09-11</aired>',
' <dateadded>2020-01-01 01:01:01</dateadded>',
' <genre>test category 1</genre>',
' <genre>test category 2</genre>',
'</episodedetails>',
]
# Compare it line by line
test_nfo = self.media.nfoxml.split('\n')
for i, line in enumerate(test_nfo):
self.assertEqual(line, expected_nfo[i])
class FormatMatchingTestCase(TestCase):

8
tubesync/sync/utils.py
View File

@ -108,6 +108,14 @@ def file_is_editable(filepath):
return False
def write_text_file(filepath, filedata):
if not isinstance(filedata, str):
raise ValueError(f'filedata must be a str, got "{type(filedata)}"')
with open(filepath, 'wt') as f:
bytes_written = f.write(filedata)
return bytes_written
def delete_file(filepath):
if file_is_editable(filepath):
return os.remove(filepath)

29
tubesync/sync/views.py
View File

@ -276,7 +276,7 @@ class AddSourceView(CreateView):
fields = ('source_type', 'key', 'name', 'directory', 'media_format',
'index_schedule', 'delete_old_media', 'days_to_keep',
'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
'prefer_hdr', 'fallback', 'copy_thumbnails')
'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
errors = {
'invalid_media_format': _('Invalid media format, the media format contains '
'errors or is empty. Check the table at the end of '
@ -334,8 +334,8 @@ class SourceView(DetailView):
messages = {
'source-created': _('Your new source has been created. If you have added a '
'very large source such as a channel with hundreds of '
'videos it can take several minutes for media to start '
'to appear.'),
'videos it can take several minutes or up to an hour '
'for media to start to appear.'),
'source-updated': _('Your source has been updated.'),
}
@ -367,7 +367,7 @@ class UpdateSourceView(UpdateView):
fields = ('source_type', 'key', 'name', 'directory', 'media_format',
'index_schedule', 'delete_old_media', 'days_to_keep',
'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
'prefer_hdr', 'fallback', 'copy_thumbnails')
'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
errors = {
'invalid_media_format': _('Invalid media format, the media format contains '
'errors or is empty. Check the table at the end of '
@ -411,7 +411,12 @@ class DeleteSourceView(DeleteView, FormMixin):
source = self.get_object()
for media in Media.objects.filter(source=source):
if media.media_file:
# Delete the media file
delete_file(media.media_file.name)
# Delete thumbnail copy if it exists
delete_file(media.thumbpath)
# Delete NFO file if it exists
delete_file(media.nfopath)
return super().post(request, *args, **kwargs)
def get_success_url(self):
@ -556,13 +561,12 @@ class MediaRedownloadView(FormView, SingleObjectMixin):
self.object.thumb = None
# If the media file exists on disk, delete it
if self.object.media_file_exists:
filepath = self.object.media_file.path
delete_file(filepath)
delete_file(self.object.media_file.path)
self.object.media_file = None
# If the media has an associated thumbnail copied, also delete it
barefilepath, fileext = os.path.splitext(filepath)
thumbpath = f'{barefilepath}.jpg'
delete_file(thumbpath)
delete_file(self.object.thumbpath)
# If the media has an associated NFO file with it, also delete it
delete_file(self.object.nfopath)
# Reset all download data
self.object.downloaded = False
self.object.downloaded_audio_codec = None
@ -602,13 +606,12 @@ class MediaSkipView(FormView, SingleObjectMixin):
delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),))
# If the media file exists on disk, delete it
if self.object.media_file_exists:
filepath = self.object.media_file.path
delete_file(self.object.media_file.path)
self.object.media_file = None
# If the media has an associated thumbnail copied, also delete it
barefilepath, fileext = os.path.splitext(filepath)
thumbpath = f'{barefilepath}.jpg'
delete_file(thumbpath)
delete_file(self.object.thumbpath)
# If the media has an associated NFO file with it, also delete it
delete_file(self.object.nfopath)
# Reset all download data
self.object.downloaded = False
self.object.downloaded_audio_codec = None