From 60a726bb7c04de0089da6d4b01627a000aa5fab8 Mon Sep 17 00:00:00 2001 From: meeb Date: Wed, 9 Dec 2020 01:31:45 +1100 Subject: [PATCH] refactoring, test suite for sync app --- app/sync/matching.py | 8 +- app/sync/signals.py | 19 +- app/sync/tasks.py | 4 +- app/sync/tests.py | 1735 +++++++++++++++++++++++++++++++++++++++++- app/sync/urls.py | 2 +- app/sync/utils.py | 5 + app/sync/views.py | 2 +- 7 files changed, 1756 insertions(+), 19 deletions(-) diff --git a/app/sync/matching.py b/app/sync/matching.py index 0ef10c1..c684891 100644 --- a/app/sync/matching.py +++ b/app/sync/matching.py @@ -51,7 +51,7 @@ def get_best_audio_format(media): audio_formats = [] for fmt in media.iter_formats(): # If the format has a video stream, skip it - if fmt['vcodec']: + if fmt['vcodec'] is not None: continue audio_formats.append(fmt) audio_formats = list(reversed(sorted(audio_formats, key=lambda k: k['abr']))) @@ -86,7 +86,7 @@ def get_best_video_format(media): video_formats = [] for fmt in media.iter_formats(): # If the format has an audio stream, skip it - if fmt['acodec']: + if fmt['acodec'] is not None: continue if media.source.source_resolution.strip().upper() == fmt['format']: video_formats.append(fmt) @@ -97,7 +97,7 @@ def get_best_video_format(media): # Find the next-best format matches by height for fmt in media.iter_formats(): # If the format has an audio stream, skip it - if fmt['acodec']: + if fmt['acodec'] is not None: continue if (fmt['height'] <= media.source.source_resolution_height and fmt['height'] >= min_height): @@ -106,8 +106,6 @@ def get_best_video_format(media): # Can't fallback return False, False video_formats = list(reversed(sorted(video_formats, key=lambda k: k['height']))) - print('height', media.source.source_resolution_height) - print('video_formats', video_formats) if not video_formats: # Still no matches return False, False diff --git a/app/sync/signals.py b/app/sync/signals.py index f40b018..6dabe8a 100644 --- a/app/sync/signals.py +++ b/app/sync/signals.py @@ -6,8 +6,8 @@ from background_task.signals import task_failed from background_task.models import Task from common.logger import log from .models import Source, Media -from .tasks import (delete_index_source_task, index_source_task, - download_media_thumbnail, map_task_to_instance) +from .tasks import (delete_task, index_source_task, download_media_thumbnail, + map_task_to_instance) from .utils import delete_file @@ -22,7 +22,7 @@ def source_pre_save(sender, instance, **kwargs): return if existing_source.index_schedule != instance.index_schedule: # Indexing schedule has changed, recreate the indexing task - delete_index_source_task(str(instance.pk)) + delete_task('sync.tasks.index_source_task', instance.pk) verbose_name = _('Index media from source "{}"') index_source_task( str(instance.pk), @@ -37,7 +37,7 @@ def source_post_save(sender, instance, created, **kwargs): # Triggered after a source is saved if created: # Create a new indexing task for newly created sources - delete_index_source_task(str(instance.pk)) + delete_task('sync.tasks.index_source_task', instance.pk) log.info(f'Scheduling media indexing for source: {instance.name}') verbose_name = _('Index media from source "{}"') index_source_task( @@ -61,7 +61,7 @@ def source_pre_delete(sender, instance, **kwargs): def source_post_delete(sender, instance, **kwargs): # Triggered after a source is deleted log.info(f'Deleting tasks for source: {instance.name}') - delete_index_source_task(str(instance.pk)) + delete_task('sync.tasks.index_source_task', instance.pk) @receiver(task_failed, sender=Task) @@ -93,9 +93,12 @@ def media_post_save(sender, instance, created, **kwargs): ) -@receiver(post_delete, sender=Media) -def media_post_delete(sender, instance, **kwargs): - # Triggered after media is deleted, delete media thumbnail +@receiver(pre_delete, sender=Media) +def media_pre_delete(sender, instance, **kwargs): + # Triggered before media is deleted, delete any scheduled tasks + log.info(f'Deleting tasks for media: {instance.name}') + delete_task('sync.tasks.download_media_thumbnail', instance.source.pk) + # Delete media thumbnail if it exists if instance.thumb: log.info(f'Deleting thumbnail for: {instance} path: {instance.thumb.path}') delete_file(instance.thumb.path) diff --git a/app/sync/tasks.py b/app/sync/tasks.py index 68e2585..1c174ce 100644 --- a/app/sync/tasks.py +++ b/app/sync/tasks.py @@ -110,8 +110,8 @@ def get_source_completed_tasks(source_id, only_errors=False): return CompletedTask.objects.filter(**q).order_by('-failed_at') -def delete_index_source_task(source_id): - Task.objects.drop_task('sync.tasks.index_source_task', args=(source_id,)) +def delete_task(task_name, source_id): + return Task.objects.filter(task_name=task_name, queue=str(source_id)).delete() def cleanup_completed_tasks(): diff --git a/app/sync/tests.py b/app/sync/tests.py index 7ce503c..9cd7607 100644 --- a/app/sync/tests.py +++ b/app/sync/tests.py @@ -1,3 +1,1734 @@ -from django.test import TestCase +''' + Note these tests do not test the scheduled tasks that perform live requests to + index media or download content. They only check for compliance of web + interface and validation code. +''' -# Create your tests here. + +import logging +from urllib.parse import urlsplit +from django.test import TestCase, Client +from django.utils import timezone +from background_task.models import Task +from .models import Source, Media + + +class FrontEndTestCase(TestCase): + + def setUp(self): + # Disable general logging for test case + logging.disable(logging.CRITICAL) + + def test_dashboard(self): + c = Client() + response = c.get('/') + self.assertEqual(response.status_code, 200) + + def test_validate_source(self): + test_source_types = { + 'youtube-channel': Source.SOURCE_TYPE_YOUTUBE_CHANNEL, + 'youtube-playlist': Source.SOURCE_TYPE_YOUTUBE_PLAYLIST, + } + test_sources = { + 'youtube-channel': { + 'valid': ( + 'https://www.youtube.com/testchannel', + 'https://www.youtube.com/c/testchannel', + ), + 'invalid_schema': ( + 'http://www.youtube.com/c/playlist', + 'ftp://www.youtube.com/c/playlist', + ), + 'invalid_domain': ( + 'https://www.test.com/c/testchannel', + 'https://www.example.com/c/testchannel', + ), + 'invalid_path': ( + 'https://www.youtube.com/test/invalid', + 'https://www.youtube.com/c/test/invalid', + ), + 'invalid_is_playlist': ( + 'https://www.youtube.com/c/playlist', + 'https://www.youtube.com/c/playlist', + ), + }, + 'youtube-playlist': { + 'valid': ( + 'https://www.youtube.com/playlist?list=testplaylist' + 'https://www.youtube.com/watch?v=testvideo&list=testplaylist' + ), + 'invalid_schema': ( + 'http://www.youtube.com/playlist?list=testplaylist', + 'ftp://www.youtube.com/playlist?list=testplaylist', + ), + 'invalid_domain': ( + 'https://www.test.com/playlist?list=testplaylist', + 'https://www.example.com/playlist?list=testplaylist', + ), + 'invalid_path': ( + 'https://www.youtube.com/notplaylist?list=testplaylist', + 'https://www.youtube.com/c/notplaylist?list=testplaylist', + ), + 'invalid_is_channel': ( + 'https://www.youtube.com/testchannel', + 'https://www.youtube.com/c/testchannel', + ), + } + } + c = Client() + for source_type in test_sources.keys(): + response = c.get(f'/source-validate/{source_type}') + self.assertEqual(response.status_code, 200) + response = c.get('/source-validate/invalid') + self.assertEqual(response.status_code, 404) + for (source_type, tests) in test_sources.items(): + for test, field in tests.items(): + source_type_char = test_source_types.get(source_type) + data = {'source_url': field, 'source_type': source_type_char} + response = c.post(f'/source-validate/{source_type}', data) + if test == 'valid': + # Valid source tests should bounce to /source-add + self.assertEqual(response.status_code, 302) + url_parts = urlsplit(response.url) + self.assertEqual(url_parts.path, '/source-add') + else: + # Invalid source tests should reload the page with an error message + self.assertEqual(response.status_code, 200) + self.assertIn('