From 149d1357e6743dfc1cd7bfc180e5aa3c706fc323 Mon Sep 17 00:00:00 2001 From: meeb Date: Sun, 6 Dec 2020 12:22:16 +1100 Subject: [PATCH] media indexing and management --- Pipfile | 1 + Pipfile.lock | 44 +++- app/common/static/images/nothumb.jpg | Bin 0 -> 4644 bytes app/common/static/styles/_colours.scss | 3 + app/common/static/styles/_template.scss | 17 ++ app/common/templates/infobox.html | 2 +- .../migrations/0005_auto_20201205_0411.py | 23 ++ .../migrations/0006_auto_20201205_0502.py | 36 +++ .../migrations/0007_auto_20201205_0509.py | 18 ++ .../migrations/0008_auto_20201205_0512.py | 23 ++ .../migrations/0009_auto_20201205_0512.py | 18 ++ app/sync/models.py | 210 ++++++++++++++---- app/sync/signals.py | 38 ++++ app/sync/tasks.py | 84 +++++++ app/sync/templates/sync/media-item.html | 39 ++++ app/sync/templates/sync/media.html | 29 ++- app/sync/urls.py | 14 +- app/sync/utils.py | 12 + app/sync/views.py | 75 ++++++- app/sync/youtube.py | 6 +- app/tubesync/settings.py | 11 +- 21 files changed, 629 insertions(+), 74 deletions(-) create mode 100644 app/common/static/images/nothumb.jpg create mode 100644 app/sync/migrations/0005_auto_20201205_0411.py create mode 100644 app/sync/migrations/0006_auto_20201205_0502.py create mode 100644 app/sync/migrations/0007_auto_20201205_0509.py create mode 100644 app/sync/migrations/0008_auto_20201205_0512.py create mode 100644 app/sync/migrations/0009_auto_20201205_0512.py create mode 100644 app/sync/signals.py create mode 100644 app/sync/tasks.py create mode 100644 app/sync/templates/sync/media-item.html diff --git a/Pipfile b/Pipfile index 9d8e001..03e83ad 100644 --- a/Pipfile +++ b/Pipfile @@ -16,6 +16,7 @@ django-compressor = "*" httptools = "*" youtube-dl = "*" django-background-tasks = "*" +requests = "*" [requires] python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index 9a4b1c7..cf3534a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "229a3783dcfd9d030ac58856ecb5d52d806db6acfa40eb6346eb6b58d79e91ab" + "sha256": "a4bb556fc61ee4583f9588980450b071814298ee4d1a1023fad149c14d14aaba" }, "pipfile-spec": 6, "requires": { @@ -23,6 +23,20 @@ ], "version": "==3.3.1" }, + "certifi": { + "hashes": [ + "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", + "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4" + ], + "version": "==2020.11.8" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, "django": { "hashes": [ "sha256:5c866205f15e7a7123f1eec6ab939d22d5bde1416635cab259684af66d8e48a2", @@ -92,6 +106,13 @@ "index": "pypi", "version": "==0.1.1" }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "version": "==2.10" + }, "libsass": { "hashes": [ "sha256:1521d2a8d4b397c6ec90640a1f6b5529077035efc48ef1c2e53095544e713d1b", @@ -158,6 +179,14 @@ ], "version": "==1.0.6" }, + "requests": { + "hashes": [ + "sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8", + "sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998" + ], + "index": "pypi", + "version": "==2.25.0" + }, "rjsmin": { "hashes": [ "sha256:0ab825839125eaca57cc59581d72e596e58a7a56fbc0839996b7528f0343a0a8", @@ -190,6 +219,13 @@ ], "version": "==0.4.1" }, + "urllib3": { + "hashes": [ + "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", + "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" + ], + "version": "==1.26.2" + }, "whitenoise": { "hashes": [ "sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7", @@ -200,11 +236,11 @@ }, "youtube-dl": { "hashes": [ - "sha256:bc82acb0b59b25b822fad85bef0cbe78e5754ca532e3bd6899fe06386e2b8e7c", - "sha256:daa514c0d36af478fe249ea93ca63ae3aaa60ac5f82aa3d5514bce9ba7ed8451" + "sha256:c73c79ccaabb7eabc223b36889ad9d3fbe04433d933312e8752d6a2c2bdc028d", + "sha256:cd7a072314ee67472de75a0f770bffc1363be46921c28bfc47a786a5188c417f" ], "index": "pypi", - "version": "==2020.12.2" + "version": "==2020.12.5" } }, "develop": {} diff --git a/app/common/static/images/nothumb.jpg b/app/common/static/images/nothumb.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c186a3c043d3c73c0fca31bfe78409523a231952 GIT binary patch literal 4644 zcmcInc{o*F7eD7tC~+^MjMosC%rl8=PEsiIJcSpTk}0_=iQ-bEkcd#GZo<9EER|Bm z%rYl4nG@g9_ddDb_x<(#@vi+m=UHq2_CDwAz1Cjqw|BaCh5+5)hh7gQ8)erDfn|!LV@u-?CE& zU}%8_5Dy1201N}dG2l)EzyW{&3=S<}Pf#>41QLYrf>8f|fd~``N5W`!dI5Sk1ft<+ z001ezguUtii01gw!_4+retu@ODn}1-u;*jq>}Z(G7}d90ehB771&*vw0>I?vEzgfx ztH*2iV0w!j>@RriNDu%hb3Yy^sR~~CO!xyMoXi}vqk>?Kch9AI7j0@TuX*6g0kPnV z4{Yafe@4OJ+|9O)FXPI|3g_k@0m3*n&GDcVQ6)Kxt(bRzM!_2|1Mdkc!iUtfCW7vZ z2Foe+$8`TeAq5j=6LXJiXn#@Q=aB6o0u{ud7TvLYbfQdwE`1yMNFXp#7*{=5qMnT16g4TZ@|pLX=$CP`cuox= z{n@+VL>E@)XV?3}pxxOMu%@ehbUj_bO6j;TzQqdrM9EO4d``-cwp_70D(1sX16Eh1dmY+|OU=M{pEp#qGBy1Zn(aEkL z<6He?+`HPl464#oxV6Hxt8UmFnBhOR(H2S1YHBQ4F^=-Q>71V~Tsv=_*HNDG{lyNT zeL1&2QRwBhdFiux9)Y<}9O<17waz%T4JK-FuTv6}Dkp67hbLc6zRIO}=(!X=+JZ8!>b=MIIhjm3`IvsuxT>Tpde_Uq ze4~MbU1~ulTVB)x$=h5irk{iBZY{a`;nr|$sepu)RH07!wGUmheHcf9(@bs8{lCt3 zZ?4XSnbtd_!*9PTJMov56-+er!H8wg<1~f?A6j$yW4uUr@PCvzoBVtKDv;vU~%9Fl;xQw9Toa5z8Ol`Zhno+-drGlYa>tea z)cZ^OhGuAerw(HO3%$rmj3Y(3nUH5X;V6_#3|GlBILWD)qzcxnurcWWco&m@J^!M> z%}<|I!-pTRxX2bJ9l|k1`<{9DLNcyN+tz9;`Dm-xQBU!?>;qW5 zPZGA;mF+qWPTmXRZkg-3EIdYjzxn`inQ~DPecEAI(cjX6V2_61s2HfPV+0h7M>G3o z-%3_6hU|8O3LX^eM(nVvh>Q^33Gy;7;fsD=uYZmON*z!Xta#Yrm{a2|@#b*Cy;kw< zsH{SlHr20dHf`Q7WD-YS7|8vIz0v~0-~fgR&1=Yvl~U)!Lk|D#Z`iKCw_t5~SxQAEW)}HHZl`i1&hHcM zFApKvvBYuAS2%~qlnE{?{Ctrr;p`hK=V9`6gu8-Q!LCBB(24Cr7ps^3S{s2 ztl<*hh@!$}ZRW6*Xo+gn9BZ`DC(_%bprHpOw#VKDzT=c)PR?&$PN#_b)e7ww>s|+& z;mz>I;{02lcbrNOvtKU|=Haco>t7%=UYFqH<-j9;aP)u;YbM>=$xNNEgx@PuL<2AY zjzFPk=>AR;3J`w=1%SR^&=9j}sJEdHG~P-?Js;HfMO zcm0!hd(@X>g=&*u-J6$bqY#OR?L2B z!ig|ywYq;J^|4s=wa!Bd#RS)7DL!s2$xbX>qvlYB{1Icm`q5 z&rq;&4c6t6zVVzRzDZ|3$DzixBN?|&x)2)IZu_l{Qhz$7^L*DTC96WSXC#oPRCe8z zzNgt#c~;I8-|(#luVp1t4wBPDL$c_HbDHJOceOP?d$-+!VG+3v^!vGldfDavEcy^4 zTR9{W0}}dE6o&$u!=8Mn7^ORjCFp0heiNzMur+^2 zo#EoM){XY-zdp_=*K83R@XYf*?Xf%YOLhlkyV}}IL`^I1b$S{clC{p;N;x*6=dmKy zs>kM~v<3clohTa0+aQ8=FT+Dr2=bc)Ckv2@ z_og)9jI57wian`V zdd_K6dg(`#Ih~o2=`96s*0qIR1Kc+xzJ>$y1{{A<(bpC*TjG@sOgE5C6L53bckrM>g!x${jIJ~8G{FQ5ztL3$Vlk~ zX+J}XVF!LFbml|?LgUw^czvYCO(NHlqNfUSAY}He->wtmAA z*`lN7$Lk{Ux^-sm9ZHEgSCrJoD!bU^@{nhgVD2f)X1OkV`$^0PTN-xJC<0ktaH{%g z4t%ih+CX={ojp_U=8LL=d>``G__nm45t zDC@^2T@czJ(RIFOvDI8FX%i6kkj4j$lg@~~q?5K!I&hSG9P`F-lf_SLtkUs!{`i|+ z0=tC+BAEZIAUP_iJ^*(YxRbGG@6+?VLq&_Et-_X0?V%#p0N=Si6O;r`n(vvAX8a%2 zZ(rMgfC33MR5tu&7L^6SviF$hZhtnO8U?``YV*HzFq%N!gFI40ZDO9_sZCfof!h2h zbf_MPU?5TjMi zt4xd1RK*ZCCF$rwb_4e!L78H=!yO~iCo$7b-D?>JZQg2_hlg9G0}|s3s)W9uFO2Vf zWPL_&dF`>Nld?>vhn>C^aK^Ow3U*j~VB1eivfBTGYJcg-3&Dm2A2-p#!Pjw(qZz?9 z1aZyfm5~#P)5&-Ha`_+aWBo;M+tpB2zudpx7uM~NdJnk+NDeof1RtE8_K~d7Q=%%VL3~l$pzmWF`6Op|PAY=V`@Xr>^9H z7{t@A&s**#7rj~|9-SIF$!570m896Z=-Ob}@Rx#6nm&aeUB&PoU`Vr$G=FGue1ji& zg=wPO0hDD&e#rsSB9-N1c}CLh^lujj1rZkMLW~n$9Hr}#Vpg}V!MAovGauyFU0(Nx zGeh@Dj1*Mmv3PYONr=?^De~wopj0yTGwtp7=^C!`SC6D2BVy*`O2xsFDobKGF)~kU z|FMDRKU(C__F77D5~{IxyFwO{M2r~?YV-T5&K2cHouv%R!zHx!Y9l`edU@N?puGDp zt?-wlL0!a&tYZOo8USu+5>;V!6ELcC&GQa2bS#;D4JfJrfsXXs6(m8s~Og6uD;7y|~&(=W4wEgmT1N*iC^!BL!IQ z{=lXi)jEL2U4ZpUdyCi)1nf9&75zZb#!dg=gJ%UQa$93Xx*0H&`c@k!2lCW$`YND| zjW(HiD5IYhMVZ#(q$hmfH(VpyG=Y&m;0hCU{!s5hSkd<8-^eQ76su z93E$;u&HI)=b=t<;mL z>S>0XTiu&3*=gat#4*X2`}0)!2!-u8$L>QFuf7F=13^{M2bu7J0}m#anMF*%aH%`Il`E_ML1m zU~yS=PMP8#LD$L+kjT$2KYeT&z*3||_Ad^Ih2wFDQEllrJnL;ZiRu>kBJ$8-)Tl@N z)$4sCd-y)+B1Q0U-?(TB107v$pBB86#{YIv;Rm^k-TQj7Vn4~(^WTf2 b$>ucW=+YSyR_T`=J@2x1s+T7te5dDM8FP-j literal 0 HcmV?d00001 diff --git a/app/common/static/styles/_colours.scss b/app/common/static/styles/_colours.scss index b5f4b70..07cb185 100644 --- a/app/common/static/styles/_colours.scss +++ b/app/common/static/styles/_colours.scss @@ -40,6 +40,9 @@ $collection-no-items-text-colour: $colour-near-black; $collection-background-hover-colour: $colour-orange; $collection-text-hover-colour: $colour-near-white; +$mediacard-title-background-colour: $colour-white; +$mediacard-title-text-colour: $colour-black; + $box-error-background-colour: $colour-red; $box-error-text-colour: $colour-near-white; diff --git a/app/common/static/styles/_template.scss b/app/common/static/styles/_template.scss index 3a1c46f..fc5d181 100644 --- a/app/common/static/styles/_template.scss +++ b/app/common/static/styles/_template.scss @@ -104,6 +104,23 @@ main { } } + .mediacard { + .card-title { + padding: 3px 10px 3px 5px !important; + margin: 5px !important; + line-height: 1.5rem; + font-size: 1.5rem; + left: initial !important; + right: 0 !important; + max-width: 75% !important; + opacity: 80%; + background-color: $mediacard-title-background-colour !important; + color: $mediacard-title-text-colour !important; + span { + font-size: 1rem; + } + } + } .pagination { width: 100%; diff --git a/app/common/templates/infobox.html b/app/common/templates/infobox.html index fcb9b38..269dbd1 100644 --- a/app/common/templates/infobox.html +++ b/app/common/templates/infobox.html @@ -3,7 +3,7 @@
- {{ message }} + {{ message|safe }}
diff --git a/app/sync/migrations/0005_auto_20201205_0411.py b/app/sync/migrations/0005_auto_20201205_0411.py new file mode 100644 index 0000000..32511ae --- /dev/null +++ b/app/sync/migrations/0005_auto_20201205_0411.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.3 on 2020-12-05 04:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0004_remove_source_url'), + ] + + operations = [ + migrations.AlterField( + model_name='media', + name='thumb_height', + field=models.PositiveSmallIntegerField(blank=True, help_text='Height (Y) of the thumbnail', null=True, verbose_name='thumb height'), + ), + migrations.AlterField( + model_name='media', + name='thumb_width', + field=models.PositiveSmallIntegerField(blank=True, help_text='Width (X) of the thumbnail', null=True, verbose_name='thumb width'), + ), + ] diff --git a/app/sync/migrations/0006_auto_20201205_0502.py b/app/sync/migrations/0006_auto_20201205_0502.py new file mode 100644 index 0000000..4490d4e --- /dev/null +++ b/app/sync/migrations/0006_auto_20201205_0502.py @@ -0,0 +1,36 @@ +# Generated by Django 3.1.3 on 2020-12-05 05:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0005_auto_20201205_0411'), + ] + + operations = [ + migrations.RemoveField( + model_name='source', + name='output_format', + ), + migrations.RemoveField( + model_name='source', + name='source_profile', + ), + migrations.AddField( + model_name='source', + name='source_resolution', + field=models.CharField(choices=[('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('2160p', '2160p (4K)'), ('audio', 'Audio only')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution'), + ), + migrations.AddField( + model_name='source', + name='source_vcodec', + field=models.CharField(choices=[('M4A', 'M4A'), ('OPUS', 'OPUS'), ('AAC', 'AAC')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'), + ), + migrations.AlterField( + model_name='source', + name='prefer_60fps', + field=models.BooleanField(default=True, help_text='Where possible, prefer 60fps media for this source', verbose_name='prefer 60fps'), + ), + ] diff --git a/app/sync/migrations/0007_auto_20201205_0509.py b/app/sync/migrations/0007_auto_20201205_0509.py new file mode 100644 index 0000000..e3c6486 --- /dev/null +++ b/app/sync/migrations/0007_auto_20201205_0509.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.3 on 2020-12-05 05:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0006_auto_20201205_0502'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='source_vcodec', + field=models.CharField(choices=[('M4A', 'M4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'), + ), + ] diff --git a/app/sync/migrations/0008_auto_20201205_0512.py b/app/sync/migrations/0008_auto_20201205_0512.py new file mode 100644 index 0000000..aecc588 --- /dev/null +++ b/app/sync/migrations/0008_auto_20201205_0512.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.3 on 2020-12-05 05:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0007_auto_20201205_0509'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='source_acodec', + field=models.CharField(choices=[('M4A', 'M4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'), + ), + migrations.AlterField( + model_name='source', + name='source_vcodec', + field=models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download', max_length=8, verbose_name='source video codec'), + ), + ] diff --git a/app/sync/migrations/0009_auto_20201205_0512.py b/app/sync/migrations/0009_auto_20201205_0512.py new file mode 100644 index 0000000..ca84fee --- /dev/null +++ b/app/sync/migrations/0009_auto_20201205_0512.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.3 on 2020-12-05 05:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0008_auto_20201205_0512'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='source_vcodec', + field=models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec'), + ), + ] diff --git a/app/sync/models.py b/app/sync/models.py index befeb92..d4c4ae9 100644 --- a/app/sync/models.py +++ b/app/sync/models.py @@ -1,7 +1,12 @@ import uuid +import json +from datetime import datetime +from pathlib import Path from django.conf import settings from django.db import models +from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ +from .youtube import get_media_info as get_youtube_media_info class Source(models.Model): @@ -18,36 +23,40 @@ class Source(models.Model): (SOURCE_TYPE_YOUTUBE_PLAYLIST, _('YouTube playlist')), ) - SOURCE_PROFILE_360p = '360p' - SOURCE_PROFILE_480p = '480p' - SOURCE_PROFILE_720P = '720p' - SOURCE_PROFILE_1080P = '1080p' - SOURCE_PROFILE_2160P = '2160p' - SOURCE_PROFILE_AUDIO = 'audio' - SOURCE_PROFILES = (SOURCE_PROFILE_360p, SOURCE_PROFILE_480p, SOURCE_PROFILE_720P, - SOURCE_PROFILE_1080P, SOURCE_PROFILE_2160P, - SOURCE_PROFILE_AUDIO) - SOURCE_PROFILE_CHOICES = ( - (SOURCE_PROFILE_360p, _('360p (SD)')), - (SOURCE_PROFILE_480p, _('480p (SD)')), - (SOURCE_PROFILE_720P, _('720p (HD)')), - (SOURCE_PROFILE_1080P, _('1080p (Full HD)')), - (SOURCE_PROFILE_2160P, _('2160p (4K)')), - (SOURCE_PROFILE_AUDIO, _('Audio only')), + SOURCE_RESOLUTION_360p = '360p' + SOURCE_RESOLUTION_480p = '480p' + SOURCE_RESOLUTION_720P = '720p' + SOURCE_RESOLUTION_1080P = '1080p' + SOURCE_RESOLUTION_2160P = '2160p' + SOURCE_RESOLUTION_AUDIO = 'audio' + SOURCE_RESOLUTIONS = (SOURCE_RESOLUTION_360p, SOURCE_RESOLUTION_480p, + SOURCE_RESOLUTION_720P, SOURCE_RESOLUTION_1080P, + SOURCE_RESOLUTION_2160P, SOURCE_RESOLUTION_AUDIO) + SOURCE_RESOLUTION_CHOICES = ( + (SOURCE_RESOLUTION_360p, _('360p (SD)')), + (SOURCE_RESOLUTION_480p, _('480p (SD)')), + (SOURCE_RESOLUTION_720P, _('720p (HD)')), + (SOURCE_RESOLUTION_1080P, _('1080p (Full HD)')), + (SOURCE_RESOLUTION_2160P, _('2160p (4K)')), + (SOURCE_RESOLUTION_AUDIO, _('Audio only')), ) - OUTPUT_FORMAT_MP4 = 'mp4' - OUTPUT_FORMAT_MKV = 'mkv' - OUTPUT_FORMAT_M4A = 'm4a' - OUTPUT_FORMAT_OGG = 'ogg' - OUTPUT_FORMATS = (OUTPUT_FORMAT_MP4, OUTPUT_FORMAT_MKV, OUTPUT_FORMAT_M4A, - OUTPUT_FORMAT_OGG) - OUTPUT_FORMAT_CHOICES = ( - (OUTPUT_FORMAT_MP4, _('.mp4 container')), - (OUTPUT_FORMAT_MKV, _('.mkv container')), - (OUTPUT_FORMAT_MKV, _('.webm container')), - (OUTPUT_FORMAT_M4A, _('.m4a container (audio only)')), - (OUTPUT_FORMAT_OGG, _('.ogg container (audio only)')), + SOURCE_VCODEC_AVC1 = 'AVC1' + SOURCE_VCODEC_VP9 = 'VP9' + SOURCE_VCODECS = (SOURCE_VCODEC_AVC1, SOURCE_VCODEC_VP9) + SOURCE_VCODECS_PRIORITY = (SOURCE_VCODEC_VP9, SOURCE_VCODEC_AVC1) + SOURCE_VCODEC_CHOICES = ( + (SOURCE_VCODEC_AVC1, _('AVC1 (H.264)')), + (SOURCE_VCODEC_VP9, _('VP9')), + ) + + SOURCE_ACODEC_M4A = 'M4A' + SOURCE_ACODEC_OPUS = 'OPUS' + SOURCE_ACODECS = (SOURCE_ACODEC_M4A, SOURCE_ACODEC_OPUS) + SOURCE_ACODEC_PRIORITY = (SOURCE_ACODEC_OPUS, SOURCE_ACODEC_M4A) + SOURCE_ACODEC_CHOICES = ( + (SOURCE_ACODEC_M4A, _('M4A')), + (SOURCE_ACODEC_OPUS, _('OPUS')), ) FALLBACK_FAIL = 'f' @@ -56,8 +65,8 @@ class Source(models.Model): FALLBACKS = (FALLBACK_FAIL, FALLBACK_NEXT_SD, FALLBACK_NEXT_HD) FALLBACK_CHOICES = ( (FALLBACK_FAIL, _('Fail, do not download any media')), - (FALLBACK_NEXT_SD, _('Get next best SD media instead')), - (FALLBACK_NEXT_HD, _('Get next best HD media instead')), + (FALLBACK_NEXT_SD, _('Get next best SD media or codec instead')), + (FALLBACK_NEXT_HD, _('Get next best HD media or codec instead')), ) ICONS = { @@ -70,6 +79,16 @@ class Source(models.Model): SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}', } + INDEXERS = { + SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info, + SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info, + } + + KEY_FIELD = { # Field returned by indexing which contains a unique key + SOURCE_TYPE_YOUTUBE_CHANNEL: 'id', + SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id', + } + uuid = models.UUIDField( _('uuid'), primary_key=True, @@ -127,17 +146,33 @@ class Source(models.Model): help_text=_('If "delete old media" is ticked, the number of days after which ' 'to automatically delete media') ) - source_profile = models.CharField( - _('source profile'), + source_resolution = models.CharField( + _('source resolution'), max_length=8, db_index=True, - choices=SOURCE_PROFILE_CHOICES, - default=SOURCE_PROFILE_1080P, - help_text=_('Source profile, the quality to attempt to download media') + choices=SOURCE_RESOLUTION_CHOICES, + default=SOURCE_RESOLUTION_1080P, + help_text=_('Source resolution, desired video resolution to download') + ) + source_vcodec = models.CharField( + _('source video codec'), + max_length=8, + db_index=True, + choices=SOURCE_VCODEC_CHOICES, + default=SOURCE_VCODEC_VP9, + help_text=_('Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)') + ) + source_acodec = models.CharField( + _('source audio codec'), + max_length=8, + db_index=True, + choices=SOURCE_ACODEC_CHOICES, + default=SOURCE_ACODEC_OPUS, + help_text=_('Source audio codec, desired audio encoding format to download') ) prefer_60fps = models.BooleanField( _('prefer 60fps'), - default=False, + default=True, help_text=_('Where possible, prefer 60fps media for this source') ) prefer_hdr = models.BooleanField( @@ -145,21 +180,13 @@ class Source(models.Model): default=False, help_text=_('Where possible, prefer HDR media for this source') ) - output_format = models.CharField( - _('output format'), - max_length=8, - db_index=True, - choices=OUTPUT_FORMAT_CHOICES, - default=OUTPUT_FORMAT_MKV, - help_text=_('Output format, the file format container in which to save media') - ) fallback = models.CharField( _('fallback'), max_length=1, db_index=True, choices=FALLBACK_CHOICES, default=FALLBACK_FAIL, - help_text=_('What do do when media in your source profile is not available') + help_text=_('What do do when media in your source resolution and codecs is not available') ) def __str__(self): @@ -181,20 +208,48 @@ class Source(models.Model): @property def url(self): return Source.create_url(self.source_type, self.key) - + + @property + def format_summary(self): + if self.source_resolution == Source.SOURCE_RESOLUTION_AUDIO: + vc = 'none' + else: + vc = self.source_vcodec + ac = self.source_acodec + f = '60FPS' if self.prefer_60fps else '' + h = 'HDR' if self.prefer_hdr else '' + return f'{self.source_resolution} (video:{vc}, audio:{ac}) {f} {h}'.strip() + @property def directory_path(self): - if self.source_profile == self.SOURCE_PROFILE_AUDIO: + if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO: return settings.SYNC_AUDIO_ROOT / self.directory else: return settings.SYNC_VIDEO_ROOT / self.directory + @property + def key_field(self): + return self.KEY_FIELD.get(self.source_type, '') + + def index_media(self): + ''' + Index the media source returning a list of media metadata as dicts. + ''' + indexer = self.INDEXERS.get(self.source_type, None) + if not callable(indexer): + raise Exception(f'Source type f"{self.source_type}" has no indexer') + response = indexer(self.url) + return response.get('entries', []) + def get_media_thumb_path(instance, filename): fileid = str(instance.uuid) - filename = f'{fileid.lower()}.{instance.image_type.lower()}' + filename = f'{fileid.lower()}.jpg' prefix = fileid[:2] - return os.path.join('thumbs', prefix, filename) + return Path('thumbs') / prefix / filename + + +_metadata_cache = {} class Media(models.Model): @@ -203,6 +258,11 @@ class Media(models.Model): Source. ''' + URLS = { + Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/watch?v={key}', + Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}', + } + uuid = models.UUIDField( _('uuid'), primary_key=True, @@ -248,11 +308,13 @@ class Media(models.Model): thumb_width = models.PositiveSmallIntegerField( _('thumb width'), blank=True, + null=True, help_text=_('Width (X) of the thumbnail') ) thumb_height = models.PositiveSmallIntegerField( _('thumb height'), blank=True, + null=True, help_text=_('Height (Y) of the thumbnail') ) metadata = models.TextField( @@ -317,3 +379,53 @@ class Media(models.Model): class Meta: verbose_name = _('Media') verbose_name_plural = _('Media') + + @property + def loaded_metadata(self): + if self.pk in _metadata_cache: + return _metadata_cache[self.pk] + _metadata_cache[self.pk] = json.loads(self.metadata) + return _metadata_cache[self.pk] + + @property + def extension(self): + ''' + The extension is also used by youtube-dl to set the output container. As + it is possible to quite easily pick combinations of codecs and containers + which are invalid (e.g. OPUS audio in an MP4 container) just set this for + people. All video is set to mkv containers, audio-only is set to m4a or ogg + depending on audio codec. + ''' + if self.source.source_resolution == Source.SOURCE_RESOLUTION_AUDIO: + if self.source.source_acodec == Source.SOURCE_ACODEC_M4A: + return 'm4a' + elif self.source.source_acodec == Source.SOURCE_ACODEC_OPUS: + return 'ogg' + else: + raise ValueError('Unable to choose audio extension, uknown acodec') + else: + return 'mkv' + + @property + def url(self): + url = self.URLS.get(self.source.source_type, '') + return url.format(key=self.key) + + @property + def title(self): + return self.loaded_metadata.get('title', '').strip() + + @property + def upload_date(self): + upload_date_str = self.loaded_metadata.get('upload_date', '').strip() + try: + return datetime.strptime(upload_date_str, '%Y%m%d') + except ValueError as e: + return None + + @property + def filename(self): + upload_date = self.upload_date.strftime('%Y-%m-%d') + title = slugify(self.title.replace('&', 'and').replace('+', 'and')) + ext = self.extension + return f'{upload_date}_{title}.{ext}' diff --git a/app/sync/signals.py b/app/sync/signals.py new file mode 100644 index 0000000..bdbbcd3 --- /dev/null +++ b/app/sync/signals.py @@ -0,0 +1,38 @@ +from django.conf import settings +from django.db.models.signals import post_save, post_delete +from django.dispatch import receiver + +from .models import Source, Media +from .tasks import delete_index_source_task, index_source_task, download_media_thumbnail + + +@receiver(post_save, sender=Source) +def source_post_save(sender, instance, created, **kwargs): + # Triggered when a source is saved + if created: + # If the source is newly created schedule its indexing + index_source_task(str(instance.pk), repeat=settings.INDEX_SOURCE_EVERY) + + +@receiver(post_delete, sender=Source) +def source_post_delete(sender, instance, **kwargs): + # Triggered when a source is deleted + delete_index_source_task(str(instance.pk)) + + +@receiver(post_save, sender=Media) +def media_post_save(sender, instance, created, **kwargs): + # Triggered when media is saved + if created: + # If the media is newly created fire a task off to download its thumbnail + metadata = instance.loaded_metadata + thumbnail_url = metadata.get('thumbnail', '') + if thumbnail_url: + download_media_thumbnail(str(instance.pk), thumbnail_url) + + +@receiver(post_delete, sender=Media) +def media_post_delete(sender, instance, **kwargs): + # Triggered when media is deleted + pass + # TODO: delete thumbnail and media file from disk diff --git a/app/sync/tasks.py b/app/sync/tasks.py new file mode 100644 index 0000000..e51ceb1 --- /dev/null +++ b/app/sync/tasks.py @@ -0,0 +1,84 @@ +''' + Start, stop and manage scheduled tasks. These are generally triggered by Django + signals (see signals.py). +''' + + +import json +from io import BytesIO +from django.conf import settings +from django.core.files.uploadedfile import SimpleUploadedFile +from background_task import background +from background_task.models import Task +from .models import Source, Media +from .utils import get_remote_image + + +def delete_index_source_task(source_id): + task = None + try: + # get_task currently returns a QuerySet, but catch DoesNotExist just in case + task = Task.objects.get_task('sync.tasks.index_source_task', args=(source_id,)) + except Task.DoesNotExist: + pass + if task: + # A scheduled task exists for this Source, delete it + task.delete() + + +@background(schedule=0) +def index_source_task(source_id): + ''' + Indexes media available from a Source object. + ''' + try: + source = Source.objects.get(pk=source_id) + except Source.DoesNotExist: + # Task triggered but the Source has been deleted, delete the task + delete_index_source_task(source_id) + return + videos = source.index_media() + for video in videos: + # Create or update each video as a Media object + key = video.get(source.key_field, None) + if not key: + # Video has no unique key (ID), it can't be indexed + continue + try: + media = Media.objects.get(key=key) + except Media.DoesNotExist: + media = Media(key=key) + media.source = source + media.metadata = json.dumps(video) + media.save() + + +@background(schedule=0) +def download_media_thumbnail(media_id, url): + ''' + Downloads an image from a URL and saves it as a local thumbnail attached to a + Media object. + ''' + try: + media = Media.objects.get(pk=media_id) + except Media.DoesNotExist: + # Task triggered but the media no longer exists, ignore task + return + i = get_remote_image(url) + max_width, max_height = getattr(settings, 'MAX_MEDIA_THUMBNAIL_SIZE', (512, 512)) + if i.width > max_width or i.height > max_height: + # Image is larger than we want to save, resize it + i.thumbnail(size=(max_width, max_height)) + image_file = BytesIO() + i.save(image_file, 'JPEG', quality=80, optimize=True, progressive=True) + image_file.seek(0) + media.thumb.save( + 'thumb', + SimpleUploadedFile( + 'thumb', + image_file.read(), + 'image/jpeg', + ), + save=True + ) + return True diff --git a/app/sync/templates/sync/media-item.html b/app/sync/templates/sync/media-item.html new file mode 100644 index 0000000..bbb3d70 --- /dev/null +++ b/app/sync/templates/sync/media-item.html @@ -0,0 +1,39 @@ +{% extends 'base.html' %} + +{% block headtitle %}Media - {{ media.key }}{% endblock %} + +{% block content %} +
+
+

Media {{ media.key }}

+

{{ media.url }}

+

Saving to: {{ media.source.directory_path }}

+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + +
SourceSource
{{ media.source }}
TitleTitle
{{ media.title }}
FilenameFilename
{{ media.filename }}
Desired formatDesired format
{{ media.source.format_summary }}
DownloadedDownloaded
{% if media.downloaded %}{% else %}{% endif %}
+
+
+{% endblock %} diff --git a/app/sync/templates/sync/media.html b/app/sync/templates/sync/media.html index ab4a5ab..d779797 100644 --- a/app/sync/templates/sync/media.html +++ b/app/sync/templates/sync/media.html @@ -1,11 +1,30 @@ -{% extends 'base.html' %} +{% extends 'base.html' %}{% load static %} -{% block headtitle %}Media{% endblock %} +{% block headtitle %}Media{% if source %} - {{ source }}{% endif %}{% endblock %} {% block content %} -
-
- media +{% include 'infobox.html' with message=message %} +
+ {% for m in media %} +
+ +
+
+ + {{ m.source }}
+ {{ m }} + +
+
+
+ {% empty %} +
+
+ No media has been indexed. +
+
+ {% endfor %}
+{% include 'pagination.html' with pagination=sources.paginator %} {% endblock %} diff --git a/app/sync/urls.py b/app/sync/urls.py index d6ee938..d60386a 100644 --- a/app/sync/urls.py +++ b/app/sync/urls.py @@ -1,7 +1,7 @@ from django.urls import path from .views import (DashboardView, SourcesView, ValidateSourceView, AddSourceView, SourceView, UpdateSourceView, DeleteSourceView, MediaView, - TasksView, LogsView) + MediaThumbView, MediaItemView, TasksView, LogsView) app_name = 'sync' @@ -41,12 +41,20 @@ urlpatterns = [ DeleteSourceView.as_view(), name='delete-source'), - # Media URLs (note /media/ is the static media URL, don't use that) + # Media URLs - path('mediafiles', + path('media', MediaView.as_view(), name='media'), + path('media-thumb/', + MediaThumbView.as_view(), + name='media-thumb'), + + path('media-item/', + MediaItemView.as_view(), + name='media-item'), + # Task URLs path('tasks', diff --git a/app/sync/utils.py b/app/sync/utils.py index 3fec18e..5bde56b 100644 --- a/app/sync/utils.py +++ b/app/sync/utils.py @@ -1,4 +1,6 @@ import re +import requests +from PIL import Image from urllib.parse import urlsplit, parse_qs from django.forms import ValidationError @@ -42,3 +44,13 @@ def validate_url(url, validator): elif extract_from == 'qs_args': extract_value = url_query_parts[extract_param][0] return extract_value + + +def get_remote_image(url): + headers = { + 'user-agent': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/69.0.3497.64 Safari/537.36') + } + r = requests.get(url, headers=headers, stream=True, timeout=60) + r.raw.decode_content = True + return Image.open(r.raw) diff --git a/app/sync/views.py b/app/sync/views.py index 8fabbbe..78f55af 100644 --- a/app/sync/views.py +++ b/app/sync/views.py @@ -1,16 +1,19 @@ +from base64 import b64decode from django.conf import settings from django.http import Http404 from django.views.generic import TemplateView, ListView, DetailView from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateView, DeleteView) +from django.http import HttpResponse from django.urls import reverse_lazy from django.forms import ValidationError from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ from common.utils import append_uri_params -from .models import Source +from .models import Source, Media from .forms import ValidateSourceForm, ConfirmDeleteSourceForm from .utils import validate_url +from . import signals from . import youtube @@ -196,8 +199,8 @@ class AddSourceView(CreateView): template_name = 'sync/source-add.html' model = Source fields = ('source_type', 'key', 'name', 'directory', 'delete_old_media', - 'days_to_keep', 'source_profile', 'prefer_60fps', 'prefer_hdr', - 'output_format', 'fallback') + 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', + 'prefer_60fps', 'prefer_hdr', 'fallback') def __init__(self, *args, **kwargs): self.prepopulated_data = {} @@ -240,8 +243,8 @@ class UpdateSourceView(UpdateView): template_name = 'sync/source-update.html' model = Source fields = ('source_type', 'key', 'name', 'directory', 'delete_old_media', - 'days_to_keep', 'source_profile', 'prefer_60fps', 'prefer_hdr', - 'output_format', 'fallback') + 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', + 'prefer_60fps', 'prefer_hdr', 'fallback') def get_success_url(self): url = reverse_lazy('sync:sources') @@ -272,16 +275,76 @@ class DeleteSourceView(DeleteView, FormMixin): return append_uri_params(url, {'message': 'source-deleted'}) -class MediaView(TemplateView): +class MediaView(ListView): ''' A bare list of media added with their states. ''' template_name = 'sync/media.html' + context_object_name = 'media' + paginate_by = settings.MEDIA_PER_PAGE + messages = { + 'filter': _('Viewing media for source: {name}'), + } + + def __init__(self, *args, **kwargs): + self.filter_source = None + super().__init__(*args, **kwargs) def dispatch(self, request, *args, **kwargs): + filter_by = request.GET.get('filter', '') + if filter_by: + try: + self.filter_source = Source.objects.get(pk=filter_by) + except Source.DoesNotExist: + self.filter_source = None return super().dispatch(request, *args, **kwargs) + def get_queryset(self): + if self.filter_source: + return Media.objects.filter(source=self.filter_source).order_by('-created') + else: + return Media.objects.all().order_by('-created') + + def get_context_data(self, *args, **kwargs): + data = super().get_context_data(*args, **kwargs) + data['message'] = '' + data['source'] = None + if self.filter_source: + message = str(self.messages.get('filter', '')) + print(message) + data['message'] = message.format(name=self.filter_source.name) + data['source'] = self.filter_source + print(data) + return data + + +class MediaThumbView(DetailView): + ''' + Shows a media thumbnail. Whitenose doesn't support post-start media image + serving and the images here are pretty small, just serve them manually. This + isn't fast, but it's not likely to be a serious bottleneck. + ''' + + model = Media + + def get(self, request, *args, **kwargs): + media = self.get_object() + if media.thumb: + thumb = open(media.thumb.path, 'rb').read() + content_type = 'image/jpeg' + else: + thumb = b64decode('R0lGODlhAQABAIABAP///wAAACH5BAEKAAEALAA' + 'AAAABAAEAAAICTAEAOw==') + content_type = 'image/gif' + return HttpResponse(thumb, content_type=content_type) + + +class MediaItemView(DetailView): + + template_name = 'sync/media-item.html' + model = Media + class TasksView(TemplateView): ''' diff --git a/app/sync/youtube.py b/app/sync/youtube.py index 27de315..0c3ac97 100644 --- a/app/sync/youtube.py +++ b/app/sync/youtube.py @@ -15,22 +15,20 @@ class YouTubeError(youtube_dl.utils.DownloadError): ''' Generic wrapped error for all errors that could be raised by youtube-dl. ''' - pass -def extract_info(url): +def get_media_info(url): ''' Extracts information from a YouTube URL and returns it as a dict. For a channel or playlist this returns a dict of all the videos on the channel or playlist as well as associated metadata. ''' + opts = _defaults.update({ 'skip_download': True, 'forcejson': True, 'simulate': True, - 'extract_flat': 'in_playlist', - 'playlist_items': 1, }) response = {} with youtube_dl.YoutubeDL(opts) as y: diff --git a/app/tubesync/settings.py b/app/tubesync/settings.py index e657aeb..77d882d 100644 --- a/app/tubesync/settings.py +++ b/app/tubesync/settings.py @@ -96,7 +96,7 @@ USE_TZ = True STATIC_URL = '/static/' STATIC_ROOT = BASE_DIR / 'static' -MEDIA_URL = '/media/' +#MEDIA_URL = '/media/' MEDIA_ROOT = BASE_DIR / 'media' SYNC_VIDEO_ROOT = BASE_DIR / 'downloads' / 'video' SYNC_AUDIO_ROOT = BASE_DIR / 'downloads' / 'audio' @@ -121,10 +121,17 @@ BACKGROUND_TASK_PRIORITY_ORDERING = 'DESC' # Process high priority tasks first SOURCES_PER_PAGE = 25 +MEDIA_PER_PAGE = 25 + + +INDEX_SOURCE_EVERY = 60 # Seconds between indexing sources, 21600 = every 6 hours + + +MAX_MEDIA_THUMBNAIL_SIZE = (320, 240) # Max size in pixels for media thumbnails YOUTUBE_DEFAULTS = { - 'age_limit': 99, + 'age_limit': 99, # Age in years to spoof the client as }