From 837b6c310731dd9178423caa9de76357ab0cc0a0 Mon Sep 17 00:00:00 2001 From: meeb Date: Mon, 20 Sep 2021 16:49:03 +1000 Subject: [PATCH] switch to yt-dlp with required library updates, resolves #145 --- Pipfile | 1 - Pipfile.lock | 10 +--------- tubesync/common/context_processors.py | 4 ++-- tubesync/common/templates/base.html | 2 +- tubesync/common/third_party_versions.py | 4 ++-- tubesync/common/utils.py | 10 ++++++++++ tubesync/sync/tasks.py | 3 ++- tubesync/sync/youtube.py | 12 ++++++------ 8 files changed, 24 insertions(+), 22 deletions(-) diff --git a/Pipfile b/Pipfile index 5b934b0..9748e02 100644 --- a/Pipfile +++ b/Pipfile @@ -14,7 +14,6 @@ whitenoise = "*" gunicorn = "*" django-compressor = "*" httptools = "*" -youtube-dl = "*" django-background-tasks = "*" requests = "*" django-basicauth = "*" diff --git a/Pipfile.lock b/Pipfile.lock index e302f2a..b5aa9d1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d56605e509bb3a227050e182c835196c2889ba79c8e8a56cf0b61a6a57c5fe08" + "sha256": "ac12e45a1719945b2e19d4a12b03136225f1f5e81affd1adf44a7b3c8dd36b8a" }, "pipfile-spec": 6, "requires": { @@ -403,14 +403,6 @@ "index": "pypi", "version": "==5.3.0" }, - "youtube-dl": { - "hashes": [ - "sha256:263e04d53fb8ba3dfbd246ad09b7d388e896c132a20cc770c26ee7684de050ac", - "sha256:cb2d3ee002158ede783e97a82c95f3817594df54367ea6a77ce5ceea4772f0ab" - ], - "index": "pypi", - "version": "==2021.6.6" - }, "yt-dlp": { "hashes": [ "sha256:c97716a715261657345176ab8190a19efa51db0e5b174a6629956548750245e1", diff --git a/tubesync/common/context_processors.py b/tubesync/common/context_processors.py index 370e933..3970a23 100644 --- a/tubesync/common/context_processors.py +++ b/tubesync/common/context_processors.py @@ -1,10 +1,10 @@ from django.conf import settings -from .third_party_versions import youtube_dl_version, ffmpeg_version +from .third_party_versions import yt_dlp_version, ffmpeg_version def app_details(request): return { 'app_version': str(settings.VERSION), - 'youtube_dl_version': youtube_dl_version, + 'yt_dlp_version': yt_dlp_version, 'ffmpeg_version': ffmpeg_version, } diff --git a/tubesync/common/templates/base.html b/tubesync/common/templates/base.html index 6695b09..ed74923 100644 --- a/tubesync/common/templates/base.html +++ b/tubesync/common/templates/base.html @@ -57,7 +57,7 @@

TubeSync version {{ app_version }} with - youtube-dl version {{ youtube_dl_version }} and + yt-dlp version {{ yt_dlp_version }} and FFmpeg version {{ ffmpeg_version }}.

diff --git a/tubesync/common/third_party_versions.py b/tubesync/common/third_party_versions.py index 6622ce0..37d6420 100644 --- a/tubesync/common/third_party_versions.py +++ b/tubesync/common/third_party_versions.py @@ -1,7 +1,7 @@ -from youtube_dl import version as yt_version +from yt_dlp import version as yt_dlp_version -youtube_dl_version = str(yt_version.__version__) +yt_dlp_version = str(yt_dlp_version.__version__) ffmpeg_version = '(shared install)' diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index b078a9b..0b2dee8 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -1,4 +1,6 @@ +from datetime import datetime from urllib.parse import urlunsplit, urlencode, urlparse +from yt_dlp.utils import LazyList from .errors import DatabaseConnectionError @@ -113,3 +115,11 @@ def clean_filename(filename): filename = filename.replace(char, '') filename = ''.join([c for c in filename if ord(c) > 30]) return ' '.join(filename.split()) + + +def json_serial(obj): + if isinstance(obj, datetime): + return obj.isoformat() + if isinstance(obj, LazyList): + return list(obj) + raise TypeError(f'Type {type(obj)} is not json_serial()-able') diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 9b69f0a..9cca708 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -22,6 +22,7 @@ from background_task import background from background_task.models import Task, CompletedTask from common.logger import log from common.errors import NoMediaException, DownloadFailedException +from common.utils import json_serial from .models import Source, Media, MediaServer from .utils import (get_remote_image, resize_image_to_height, delete_file, write_text_file) @@ -224,7 +225,7 @@ def download_media_metadata(media_id): return source = media.source metadata = media.index_metadata() - media.metadata = json.dumps(metadata) + media.metadata = json.dumps(metadata, default=json_serial) upload_date = media.upload_date # Media must have a valid upload date if upload_date: diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 0c1afdc..ae6f38d 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -8,7 +8,7 @@ import os from django.conf import settings from copy import copy from common.logger import log -import youtube_dl +import yt_dlp _youtubedl_cachedir = getattr(settings, 'YOUTUBE_DL_CACHEDIR', None) @@ -19,7 +19,7 @@ if _youtubedl_cachedir: -class YouTubeError(youtube_dl.utils.DownloadError): +class YouTubeError(yt_dlp.utils.DownloadError): ''' Generic wrapped error for all errors that could be raised by youtube-dl. ''' @@ -41,10 +41,10 @@ def get_media_info(url): 'extract_flat': True, }) response = {} - with youtube_dl.YoutubeDL(opts) as y: + with yt_dlp.YoutubeDL(opts) as y: try: response = y.extract_info(url, download=False) - except youtube_dl.utils.DownloadError as e: + except yt_dlp.utils.DownloadError as e: raise YouTubeError(f'Failed to extract_info for "{url}": {e}') from e if not response: raise YouTubeError(f'Failed to extract_info for "{url}": No metadata was ' @@ -99,9 +99,9 @@ def download_media(url, media_format, extension, output_file): 'quiet': True, 'progress_hooks': [hook], }) - with youtube_dl.YoutubeDL(opts) as y: + with yt_dlp.YoutubeDL(opts) as y: try: return y.download([url]) - except youtube_dl.utils.DownloadError as e: + except yt_dlp.utils.DownloadError as e: raise YouTubeError(f'Failed to download for "{url}": {e}') from e return False