diff --git a/tubesync/sync/migrations/0005_auto_20201219_0312.py b/tubesync/sync/migrations/0005_auto_20201219_0312.py new file mode 100644 index 0000000..11ee8e2 --- /dev/null +++ b/tubesync/sync/migrations/0005_auto_20201219_0312.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.4 on 2020-12-19 03:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0004_source_media_format'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='source_type', + field=models.CharField(choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type'), + ), + ] diff --git a/tubesync/sync/migrations/0006_source_write_nfo.py b/tubesync/sync/migrations/0006_source_write_nfo.py new file mode 100644 index 0000000..d5fe936 --- /dev/null +++ b/tubesync/sync/migrations/0006_source_write_nfo.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.4 on 2020-12-19 03:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0005_auto_20201219_0312'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='write_nfo', + field=models.BooleanField(default=False, help_text='Write an NFO file with the media, these may be detected and used by some media servers', verbose_name='write nfo'), + ), + ] diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 874bc7f..2b2ec4e 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -1,6 +1,7 @@ import os import uuid import json +from xml.etree import ElementTree from datetime import datetime from pathlib import Path from django.conf import settings @@ -253,6 +254,11 @@ class Source(models.Model): default=False, help_text=_('Copy thumbnails with the media, these may be detected and used by some media servers') ) + write_nfo = models.BooleanField( + _('write nfo'), + default=False, + help_text=_('Write an NFO file in XML with the media info, these may be detected and used by some media servers') + ) has_failed = models.BooleanField( _('has failed'), default=False, @@ -476,7 +482,38 @@ class Media(models.Model): Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'formats', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'formats', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats', - } + }, + 'categories': { + Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'categories', + Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'categories', + Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'categories', + }, + 'rating': { + Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'average_rating', + Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'average_rating', + Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'average_rating', + }, + 'age_limit': { + Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'age_limit', + Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'age_limit', + Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'age_limit', + }, + 'uploader': { + Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'uploader', + Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'uploader', + Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'uploader', + }, + 'upvotes': { + Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'like_count', + Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'like_count', + Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'like_count', + }, + 'downvotes': { + Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'dislike_count', + Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count', + Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count', + }, + } STATE_UNKNOWN = 'unknown' STATE_SCHEDULED = 'scheduled' @@ -883,6 +920,34 @@ class Media(models.Model): return seconds_to_timestr(duration) return '??:??:??' + @property + def categories(self): + field = self.get_metadata_field('categories') + return self.loaded_metadata.get(field, []) + + @property + def rating(self): + field = self.get_metadata_field('rating') + return self.loaded_metadata.get(field, 0) + + @property + def votes(self): + field = self.get_metadata_field('upvotes') + upvotes = self.loaded_metadata.get(field, 0) + field = self.get_metadata_field('downvotes') + downvotes = self.loaded_metadata.get(field, 0) + return upvotes + downvotes + + @property + def age_limit(self): + field = self.get_metadata_field('age_limit') + return self.loaded_metadata.get(field, 0) + + @property + def uploader(self): + field = self.get_metadata_field('uploader') + return self.loaded_metadata.get(field, '') + @property def formats(self): field = self.get_metadata_field('formats') @@ -898,6 +963,26 @@ class Media(models.Model): media_details = self.format_dict return media_format.format(**media_details) + @property + def thumbname(self): + filename = self.filename + prefix, ext = os.path.splitext(filename) + return f'{prefix}.jpg' + + @property + def thumbpath(self): + return self.source.directory_path / self.thumbname + + @property + def nfoname(self): + filename = self.filename + prefix, ext = os.path.splitext(filename) + return f'{prefix}.nfo' + + @property + def nfopath(self): + return self.source.directory_path / self.nfoname + @property def directory_path(self): # If a media_file has been downloaded use its existing directory @@ -925,6 +1010,98 @@ class Media(models.Model): return False return os.path.exists(self.media_file.path) + @property + def nfoxml(self): + ''' + Returns an NFO formatted (prettified) XML string. + ''' + nfo = ElementTree.Element('episodedetails') + nfo.text = '\n ' + # title = media metadata title + title = nfo.makeelement('title', {}) + title.text = str(self.name).strip() + title.tail = '\n ' + nfo.append(title) + # showtitle = source name + showtitle = nfo.makeelement('showtitle', {}) + showtitle.text = str(self.source.name).strip() + showtitle.tail = '\n ' + nfo.append(showtitle) + # ratings = media metadata youtube rating + value = nfo.makeelement('value', {}) + value.text = str(self.rating) + value.tail = '\n ' + votes = nfo.makeelement('votes', {}) + votes.text = str(self.votes) + votes.tail = '\n ' + rating_attrs = {'name': 'youtube', 'max': '5', 'default': 'True'} + rating = nfo.makeelement('rating', rating_attrs) + rating.text = '\n ' + rating.append(value) + rating.append(votes) + rating.tail = '\n ' + ratings = nfo.makeelement('ratings', {}) + ratings.text = '\n ' + ratings.append(rating) + ratings.tail = '\n ' + nfo.append(ratings) + # plot = media metadata description + plot = nfo.makeelement('plot', {}) + plot.text = str(self.description).strip() + plot.tail = '\n ' + nfo.append(plot) + # thumb = local path to media thumbnail + thumb = nfo.makeelement('thumb', {}) + thumb.text = self.thumbname if self.source.copy_thumbnails else '' + thumb.tail = '\n ' + nfo.append(thumb) + # mpaa = media metadata age requirement + mpaa = nfo.makeelement('mpaa', {}) + mpaa.text = str(self.age_limit) + mpaa.tail = '\n ' + nfo.append(mpaa) + # runtime = media metadata duration in seconds + runtime = nfo.makeelement('runtime', {}) + runtime.text = str(self.duration) + runtime.tail = '\n ' + nfo.append(runtime) + # id = media key + idn = nfo.makeelement('id', {}) + idn.text = str(self.key).strip() + idn.tail = '\n ' + nfo.append(idn) + # uniqueid = media key + uniqueid_attrs = {'type': 'youtube', 'default': 'True'} + uniqueid = nfo.makeelement('uniqueid', uniqueid_attrs) + uniqueid.text = str(self.key).strip() + uniqueid.tail = '\n ' + nfo.append(uniqueid) + # studio = media metadata uploader + studio = nfo.makeelement('studio', {}) + studio.text = str(self.uploader).strip() + studio.tail = '\n ' + nfo.append(studio) + # aired = media metadata uploaded date + aired = nfo.makeelement('aired', {}) + upload_date = self.upload_date + aired.text = upload_date.strftime('%Y-%m-%d') if upload_date else '' + aired.tail = '\n ' + nfo.append(aired) + # dateadded = date and time media was created in tubesync + dateadded = nfo.makeelement('dateadded', {}) + dateadded.text = self.created.strftime('%Y-%m-%d %H:%M:%S') + dateadded.tail = '\n ' + nfo.append(dateadded) + # genre = any media metadata categories if they exist + for category_str in self.categories: + genre = nfo.makeelement('genre', {}) + genre.text = str(category_str).strip() + genre.tail = '\n ' + nfo.append(genre) + nfo[-1].tail = '\n' + # Return XML tree as a prettified string + return ElementTree.tostring(nfo, encoding='utf8', method='xml').decode('utf8') + def get_download_state(self, task=None): if self.downloaded: return self.STATE_DOWNLOADED diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index bd45a8a..c5d3fcd 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -145,20 +145,6 @@ def media_pre_delete(sender, instance, **kwargs): if thumbnail_url: delete_task_by_media('sync.tasks.download_media_thumbnail', (str(instance.pk), thumbnail_url)) - # Delete media thumbnail if it exists - if instance.thumb: - log.info(f'Deleting thumbnail for: {instance} path: {instance.thumb.path}') - delete_file(instance.thumb.path) - # Delete the media file if it exists - if instance.media_file: - filepath = instance.media_file.path - log.info(f'Deleting media for: {instance} path: {filepath}') - delete_file(filepath) - # Delete thumbnail copy if it exists - barefilepath, fileext = os.path.splitext(filepath) - thumbpath = f'{barefilepath}.jpg' - log.info(f'Deleting thumbnail for: {instance} path: {thumbpath}') - delete_file(thumbpath) @receiver(post_delete, sender=Media) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 96b0266..28a8406 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -23,7 +23,8 @@ from background_task.models import Task, CompletedTask from common.logger import log from common.errors import NoMediaException, DownloadFailedException from .models import Source, Media, MediaServer -from .utils import get_remote_image, resize_image_to_height, delete_file +from .utils import (get_remote_image, resize_image_to_height, delete_file, + write_text_file) def get_hash(task_name, pk): @@ -317,11 +318,13 @@ def download_media(media_id): media.save() # If selected, copy the thumbnail over as well if media.source.copy_thumbnails and media.thumb: - barefilepath, fileext = os.path.splitext(filepath) - thumbpath = f'{barefilepath}.jpg' log.info(f'Copying media thumbnail from: {media.thumb.path} ' - f'to: {thumbpath}') - copyfile(media.thumb.path, thumbpath) + f'to: {media.thumbpath}') + copyfile(media.thumb.path, media.thumbpath) + # If selected, write an NFO file + if media.source.write_nfo: + log.info(f'Writing media NFO file to: to: {media.nfopath}') + write_text_file(media.nfopath, media.nfoxml) # Schedule a task to update media servers for mediaserver in MediaServer.objects.all(): log.info(f'Scheduling media server updates') diff --git a/tubesync/sync/templates/sync/source.html b/tubesync/sync/templates/sync/source.html index 221bde6..84c9957 100644 --- a/tubesync/sync/templates/sync/source.html +++ b/tubesync/sync/templates/sync/source.html @@ -97,6 +97,10 @@ Copy thumbnails? Copy thumbnails?
{% if source.copy_thumbnails %}{% else %}{% endif %} + + Write NFO? + Write NFO?
{% if source.write_nfo %}{% else %}{% endif %} + {% if source.delete_old_media and source.days_to_keep > 0 %} Delete old media diff --git a/tubesync/sync/testdata/metadata.json b/tubesync/sync/testdata/metadata.json index 220acb1..aa743e0 100644 --- a/tubesync/sync/testdata/metadata.json +++ b/tubesync/sync/testdata/metadata.json @@ -3,15 +3,22 @@ "upload_date":"20170911", "license":null, "creator":null, - "title":"no fancy stuff", + "title":"no fancy stuff title", "alt_title":null, - "description":"no fancy stuff", - "categories":[], + "description":"no fancy stuff desc", + "average_rating": 1.2345, + "dislike_count": 123, + "like_count": 456, + "uploader": "test uploader", + "categories":[ + "test category 1", + "test category 2" + ], "tags":[], "subtitles":{}, "automatic_captions":{}, "duration":401.0, - "age_limit":0, + "age_limit":50, "annotations":null, "chapters":null, "formats":[ diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index c2a725e..b08369f 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -6,6 +6,7 @@ import logging +from datetime import datetime from urllib.parse import urlsplit from django.conf import settings from django.test import TestCase, Client @@ -430,7 +431,6 @@ class FrontEndTestCase(TestCase): response = c.get('/tasks-completed') self.assertEqual(response.status_code, 200) - def test_mediasevrers(self): # Media servers overview page c = Client() @@ -588,7 +588,71 @@ class FilepathTestCase(TestCase): self.source.media_format = ('{title}_{key}_{resolution}-{height}x{width}-' '{acodec}-{vcodec}-{fps}fps-{hdr}.{ext}') self.assertEqual(test_media.filename, - 'no-fancy-stuff_test_720p-720x1280-opus-vp9-30fps-hdr.mkv') + ('no-fancy-stuff-title_test_720p-720x1280-opus' + '-vp9-30fps-hdr.mkv')) + + +class MediaTestCase(TestCase): + + def setUp(self): + # Disable general logging for test case + logging.disable(logging.CRITICAL) + # Add a test source + self.source = Source.objects.create( + source_type=Source.SOURCE_TYPE_YOUTUBE_CHANNEL, + key='testkey', + name='testname', + directory='testdirectory', + media_format=settings.MEDIA_FORMATSTR_DEFAULT, + index_schedule=3600, + delete_old_media=False, + days_to_keep=14, + source_resolution=Source.SOURCE_RESOLUTION_1080P, + source_vcodec=Source.SOURCE_VCODEC_VP9, + source_acodec=Source.SOURCE_ACODEC_OPUS, + prefer_60fps=False, + prefer_hdr=False, + fallback=Source.FALLBACK_FAIL + ) + # Add some test media + self.media = Media.objects.create( + key='mediakey', + source=self.source, + metadata=metadata, + ) + # Fix a created datetime for predictable testing + self.media.created = datetime(year=2020, month=1, day=1, hour=1, + minute=1, second=1) + + def test_nfo(self): + expected_nfo = [ + "", + '', + ' no fancy stuff title', + ' testname', + ' ', + ' ', + ' 1.2345', + ' 579', + ' ', + ' ', + ' no fancy stuff desc', + ' ', # media.thumbfile is empty without media existing + ' 50', + ' 401', + ' mediakey', + ' mediakey', + ' test uploader', + ' 2017-09-11', + ' 2020-01-01 01:01:01', + ' test category 1', + ' test category 2', + '', + ] + # Compare it line by line + test_nfo = self.media.nfoxml.split('\n') + for i, line in enumerate(test_nfo): + self.assertEqual(line, expected_nfo[i]) class FormatMatchingTestCase(TestCase): diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 4c87171..e20c418 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -108,6 +108,14 @@ def file_is_editable(filepath): return False +def write_text_file(filepath, filedata): + if not isinstance(filedata, str): + raise ValueError(f'filedata must be a str, got "{type(filedata)}"') + with open(filepath, 'wt') as f: + bytes_written = f.write(filedata) + return bytes_written + + def delete_file(filepath): if file_is_editable(filepath): return os.remove(filepath) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 8fa275a..da0e483 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -276,7 +276,7 @@ class AddSourceView(CreateView): fields = ('source_type', 'key', 'name', 'directory', 'media_format', 'index_schedule', 'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps', - 'prefer_hdr', 'fallback', 'copy_thumbnails') + 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo') errors = { 'invalid_media_format': _('Invalid media format, the media format contains ' 'errors or is empty. Check the table at the end of ' @@ -334,8 +334,8 @@ class SourceView(DetailView): messages = { 'source-created': _('Your new source has been created. If you have added a ' 'very large source such as a channel with hundreds of ' - 'videos it can take several minutes for media to start ' - 'to appear.'), + 'videos it can take several minutes or up to an hour ' + 'for media to start to appear.'), 'source-updated': _('Your source has been updated.'), } @@ -367,7 +367,7 @@ class UpdateSourceView(UpdateView): fields = ('source_type', 'key', 'name', 'directory', 'media_format', 'index_schedule', 'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps', - 'prefer_hdr', 'fallback', 'copy_thumbnails') + 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo') errors = { 'invalid_media_format': _('Invalid media format, the media format contains ' 'errors or is empty. Check the table at the end of ' @@ -411,7 +411,12 @@ class DeleteSourceView(DeleteView, FormMixin): source = self.get_object() for media in Media.objects.filter(source=source): if media.media_file: + # Delete the media file delete_file(media.media_file.name) + # Delete thumbnail copy if it exists + delete_file(media.thumbpath) + # Delete NFO file if it exists + delete_file(media.nfopath) return super().post(request, *args, **kwargs) def get_success_url(self): @@ -556,13 +561,12 @@ class MediaRedownloadView(FormView, SingleObjectMixin): self.object.thumb = None # If the media file exists on disk, delete it if self.object.media_file_exists: - filepath = self.object.media_file.path - delete_file(filepath) + delete_file(self.object.media_file.path) self.object.media_file = None # If the media has an associated thumbnail copied, also delete it - barefilepath, fileext = os.path.splitext(filepath) - thumbpath = f'{barefilepath}.jpg' - delete_file(thumbpath) + delete_file(self.object.thumbpath) + # If the media has an associated NFO file with it, also delete it + delete_file(self.object.nfopath) # Reset all download data self.object.downloaded = False self.object.downloaded_audio_codec = None @@ -602,13 +606,12 @@ class MediaSkipView(FormView, SingleObjectMixin): delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),)) # If the media file exists on disk, delete it if self.object.media_file_exists: - filepath = self.object.media_file.path delete_file(self.object.media_file.path) self.object.media_file = None # If the media has an associated thumbnail copied, also delete it - barefilepath, fileext = os.path.splitext(filepath) - thumbpath = f'{barefilepath}.jpg' - delete_file(thumbpath) + delete_file(self.object.thumbpath) + # If the media has an associated NFO file with it, also delete it + delete_file(self.object.nfopath) # Reset all download data self.object.downloaded = False self.object.downloaded_audio_codec = None