media indexing and management
This commit is contained in:
meeb
parent
85844549df
commit
149d1357e6
1
Pipfile
1
Pipfile
|
|
@ -16,6 +16,7 @@ django-compressor = "*"
|
|||
httptools = "*"
|
||||
youtube-dl = "*"
|
||||
django-background-tasks = "*"
|
||||
requests = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "229a3783dcfd9d030ac58856ecb5d52d806db6acfa40eb6346eb6b58d79e91ab"
|
||||
"sha256": "a4bb556fc61ee4583f9588980450b071814298ee4d1a1023fad149c14d14aaba"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
|
|
@ -23,6 +23,20 @@
|
|||
],
|
||||
"version": "==3.3.1"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd",
|
||||
"sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4"
|
||||
],
|
||||
"version": "==2020.11.8"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
|
||||
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
|
||||
],
|
||||
"version": "==3.0.4"
|
||||
},
|
||||
"django": {
|
||||
"hashes": [
|
||||
"sha256:5c866205f15e7a7123f1eec6ab939d22d5bde1416635cab259684af66d8e48a2",
|
||||
|
|
@ -92,6 +106,13 @@
|
|||
"index": "pypi",
|
||||
"version": "==0.1.1"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
|
||||
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
|
||||
],
|
||||
"version": "==2.10"
|
||||
},
|
||||
"libsass": {
|
||||
"hashes": [
|
||||
"sha256:1521d2a8d4b397c6ec90640a1f6b5529077035efc48ef1c2e53095544e713d1b",
|
||||
|
|
@ -158,6 +179,14 @@
|
|||
],
|
||||
"version": "==1.0.6"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8",
|
||||
"sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.25.0"
|
||||
},
|
||||
"rjsmin": {
|
||||
"hashes": [
|
||||
"sha256:0ab825839125eaca57cc59581d72e596e58a7a56fbc0839996b7528f0343a0a8",
|
||||
|
|
@ -190,6 +219,13 @@
|
|||
],
|
||||
"version": "==0.4.1"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
|
||||
"sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
|
||||
],
|
||||
"version": "==1.26.2"
|
||||
},
|
||||
"whitenoise": {
|
||||
"hashes": [
|
||||
"sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7",
|
||||
|
|
@ -200,11 +236,11 @@
|
|||
},
|
||||
"youtube-dl": {
|
||||
"hashes": [
|
||||
"sha256:bc82acb0b59b25b822fad85bef0cbe78e5754ca532e3bd6899fe06386e2b8e7c",
|
||||
"sha256:daa514c0d36af478fe249ea93ca63ae3aaa60ac5f82aa3d5514bce9ba7ed8451"
|
||||
"sha256:c73c79ccaabb7eabc223b36889ad9d3fbe04433d933312e8752d6a2c2bdc028d",
|
||||
"sha256:cd7a072314ee67472de75a0f770bffc1363be46921c28bfc47a786a5188c417f"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2020.12.2"
|
||||
"version": "==2020.12.5"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 4.5 KiB |
|
|
@ -40,6 +40,9 @@ $collection-no-items-text-colour: $colour-near-black;
|
|||
$collection-background-hover-colour: $colour-orange;
|
||||
$collection-text-hover-colour: $colour-near-white;
|
||||
|
||||
$mediacard-title-background-colour: $colour-white;
|
||||
$mediacard-title-text-colour: $colour-black;
|
||||
|
||||
$box-error-background-colour: $colour-red;
|
||||
$box-error-text-colour: $colour-near-white;
|
||||
|
||||
|
|
|
|||
|
|
@ -104,6 +104,23 @@ main {
|
|||
}
|
||||
}
|
||||
|
||||
.mediacard {
|
||||
.card-title {
|
||||
padding: 3px 10px 3px 5px !important;
|
||||
margin: 5px !important;
|
||||
line-height: 1.5rem;
|
||||
font-size: 1.5rem;
|
||||
left: initial !important;
|
||||
right: 0 !important;
|
||||
max-width: 75% !important;
|
||||
opacity: 80%;
|
||||
background-color: $mediacard-title-background-colour !important;
|
||||
color: $mediacard-title-text-colour !important;
|
||||
span {
|
||||
font-size: 1rem;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.pagination {
|
||||
width: 100%;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
<div class="col s12">
|
||||
<div class="card infobox">
|
||||
<div class="card-content">
|
||||
<i class="fas fa-info-circle"></i> {{ message }}
|
||||
<i class="fas fa-info-circle"></i> {{ message|safe }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
# Generated by Django 3.1.3 on 2020-12-05 04:11
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0004_remove_source_url'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='media',
|
||||
name='thumb_height',
|
||||
field=models.PositiveSmallIntegerField(blank=True, help_text='Height (Y) of the thumbnail', null=True, verbose_name='thumb height'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='media',
|
||||
name='thumb_width',
|
||||
field=models.PositiveSmallIntegerField(blank=True, help_text='Width (X) of the thumbnail', null=True, verbose_name='thumb width'),
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
# Generated by Django 3.1.3 on 2020-12-05 05:02
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0005_auto_20201205_0411'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='source',
|
||||
name='output_format',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='source',
|
||||
name='source_profile',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='source_resolution',
|
||||
field=models.CharField(choices=[('360p', '360p (SD)'), ('480p', '480p (SD)'), ('720p', '720p (HD)'), ('1080p', '1080p (Full HD)'), ('2160p', '2160p (4K)'), ('audio', 'Audio only')], db_index=True, default='1080p', help_text='Source resolution, desired video resolution to download', max_length=8, verbose_name='source resolution'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='source_vcodec',
|
||||
field=models.CharField(choices=[('M4A', 'M4A'), ('OPUS', 'OPUS'), ('AAC', 'AAC')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='source',
|
||||
name='prefer_60fps',
|
||||
field=models.BooleanField(default=True, help_text='Where possible, prefer 60fps media for this source', verbose_name='prefer 60fps'),
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
# Generated by Django 3.1.3 on 2020-12-05 05:09
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0006_auto_20201205_0502'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='source',
|
||||
name='source_vcodec',
|
||||
field=models.CharField(choices=[('M4A', 'M4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'),
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
# Generated by Django 3.1.3 on 2020-12-05 05:12
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0007_auto_20201205_0509'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='source_acodec',
|
||||
field=models.CharField(choices=[('M4A', 'M4A'), ('OPUS', 'OPUS')], db_index=True, default='OPUS', help_text='Source audio codec, desired audio encoding format to download', max_length=8, verbose_name='source audio codec'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='source',
|
||||
name='source_vcodec',
|
||||
field=models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download', max_length=8, verbose_name='source video codec'),
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
# Generated by Django 3.1.3 on 2020-12-05 05:12
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0008_auto_20201205_0512'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='source',
|
||||
name='source_vcodec',
|
||||
field=models.CharField(choices=[('AVC1', 'AVC1 (H.264)'), ('VP9', 'VP9')], db_index=True, default='VP9', help_text='Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)', max_length=8, verbose_name='source video codec'),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,7 +1,12 @@
|
|||
import uuid
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
from django.utils.text import slugify
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from .youtube import get_media_info as get_youtube_media_info
|
||||
|
||||
|
||||
class Source(models.Model):
|
||||
|
|
@ -18,36 +23,40 @@ class Source(models.Model):
|
|||
(SOURCE_TYPE_YOUTUBE_PLAYLIST, _('YouTube playlist')),
|
||||
)
|
||||
|
||||
SOURCE_PROFILE_360p = '360p'
|
||||
SOURCE_PROFILE_480p = '480p'
|
||||
SOURCE_PROFILE_720P = '720p'
|
||||
SOURCE_PROFILE_1080P = '1080p'
|
||||
SOURCE_PROFILE_2160P = '2160p'
|
||||
SOURCE_PROFILE_AUDIO = 'audio'
|
||||
SOURCE_PROFILES = (SOURCE_PROFILE_360p, SOURCE_PROFILE_480p, SOURCE_PROFILE_720P,
|
||||
SOURCE_PROFILE_1080P, SOURCE_PROFILE_2160P,
|
||||
SOURCE_PROFILE_AUDIO)
|
||||
SOURCE_PROFILE_CHOICES = (
|
||||
(SOURCE_PROFILE_360p, _('360p (SD)')),
|
||||
(SOURCE_PROFILE_480p, _('480p (SD)')),
|
||||
(SOURCE_PROFILE_720P, _('720p (HD)')),
|
||||
(SOURCE_PROFILE_1080P, _('1080p (Full HD)')),
|
||||
(SOURCE_PROFILE_2160P, _('2160p (4K)')),
|
||||
(SOURCE_PROFILE_AUDIO, _('Audio only')),
|
||||
SOURCE_RESOLUTION_360p = '360p'
|
||||
SOURCE_RESOLUTION_480p = '480p'
|
||||
SOURCE_RESOLUTION_720P = '720p'
|
||||
SOURCE_RESOLUTION_1080P = '1080p'
|
||||
SOURCE_RESOLUTION_2160P = '2160p'
|
||||
SOURCE_RESOLUTION_AUDIO = 'audio'
|
||||
SOURCE_RESOLUTIONS = (SOURCE_RESOLUTION_360p, SOURCE_RESOLUTION_480p,
|
||||
SOURCE_RESOLUTION_720P, SOURCE_RESOLUTION_1080P,
|
||||
SOURCE_RESOLUTION_2160P, SOURCE_RESOLUTION_AUDIO)
|
||||
SOURCE_RESOLUTION_CHOICES = (
|
||||
(SOURCE_RESOLUTION_360p, _('360p (SD)')),
|
||||
(SOURCE_RESOLUTION_480p, _('480p (SD)')),
|
||||
(SOURCE_RESOLUTION_720P, _('720p (HD)')),
|
||||
(SOURCE_RESOLUTION_1080P, _('1080p (Full HD)')),
|
||||
(SOURCE_RESOLUTION_2160P, _('2160p (4K)')),
|
||||
(SOURCE_RESOLUTION_AUDIO, _('Audio only')),
|
||||
)
|
||||
|
||||
OUTPUT_FORMAT_MP4 = 'mp4'
|
||||
OUTPUT_FORMAT_MKV = 'mkv'
|
||||
OUTPUT_FORMAT_M4A = 'm4a'
|
||||
OUTPUT_FORMAT_OGG = 'ogg'
|
||||
OUTPUT_FORMATS = (OUTPUT_FORMAT_MP4, OUTPUT_FORMAT_MKV, OUTPUT_FORMAT_M4A,
|
||||
OUTPUT_FORMAT_OGG)
|
||||
OUTPUT_FORMAT_CHOICES = (
|
||||
(OUTPUT_FORMAT_MP4, _('.mp4 container')),
|
||||
(OUTPUT_FORMAT_MKV, _('.mkv container')),
|
||||
(OUTPUT_FORMAT_MKV, _('.webm container')),
|
||||
(OUTPUT_FORMAT_M4A, _('.m4a container (audio only)')),
|
||||
(OUTPUT_FORMAT_OGG, _('.ogg container (audio only)')),
|
||||
SOURCE_VCODEC_AVC1 = 'AVC1'
|
||||
SOURCE_VCODEC_VP9 = 'VP9'
|
||||
SOURCE_VCODECS = (SOURCE_VCODEC_AVC1, SOURCE_VCODEC_VP9)
|
||||
SOURCE_VCODECS_PRIORITY = (SOURCE_VCODEC_VP9, SOURCE_VCODEC_AVC1)
|
||||
SOURCE_VCODEC_CHOICES = (
|
||||
(SOURCE_VCODEC_AVC1, _('AVC1 (H.264)')),
|
||||
(SOURCE_VCODEC_VP9, _('VP9')),
|
||||
)
|
||||
|
||||
SOURCE_ACODEC_M4A = 'M4A'
|
||||
SOURCE_ACODEC_OPUS = 'OPUS'
|
||||
SOURCE_ACODECS = (SOURCE_ACODEC_M4A, SOURCE_ACODEC_OPUS)
|
||||
SOURCE_ACODEC_PRIORITY = (SOURCE_ACODEC_OPUS, SOURCE_ACODEC_M4A)
|
||||
SOURCE_ACODEC_CHOICES = (
|
||||
(SOURCE_ACODEC_M4A, _('M4A')),
|
||||
(SOURCE_ACODEC_OPUS, _('OPUS')),
|
||||
)
|
||||
|
||||
FALLBACK_FAIL = 'f'
|
||||
|
|
@ -56,8 +65,8 @@ class Source(models.Model):
|
|||
FALLBACKS = (FALLBACK_FAIL, FALLBACK_NEXT_SD, FALLBACK_NEXT_HD)
|
||||
FALLBACK_CHOICES = (
|
||||
(FALLBACK_FAIL, _('Fail, do not download any media')),
|
||||
(FALLBACK_NEXT_SD, _('Get next best SD media instead')),
|
||||
(FALLBACK_NEXT_HD, _('Get next best HD media instead')),
|
||||
(FALLBACK_NEXT_SD, _('Get next best SD media or codec instead')),
|
||||
(FALLBACK_NEXT_HD, _('Get next best HD media or codec instead')),
|
||||
)
|
||||
|
||||
ICONS = {
|
||||
|
|
@ -70,6 +79,16 @@ class Source(models.Model):
|
|||
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
|
||||
}
|
||||
|
||||
INDEXERS = {
|
||||
SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
|
||||
SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
|
||||
}
|
||||
|
||||
KEY_FIELD = { # Field returned by indexing which contains a unique key
|
||||
SOURCE_TYPE_YOUTUBE_CHANNEL: 'id',
|
||||
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id',
|
||||
}
|
||||
|
||||
uuid = models.UUIDField(
|
||||
_('uuid'),
|
||||
primary_key=True,
|
||||
|
|
@ -127,17 +146,33 @@ class Source(models.Model):
|
|||
help_text=_('If "delete old media" is ticked, the number of days after which '
|
||||
'to automatically delete media')
|
||||
)
|
||||
source_profile = models.CharField(
|
||||
_('source profile'),
|
||||
source_resolution = models.CharField(
|
||||
_('source resolution'),
|
||||
max_length=8,
|
||||
db_index=True,
|
||||
choices=SOURCE_PROFILE_CHOICES,
|
||||
default=SOURCE_PROFILE_1080P,
|
||||
help_text=_('Source profile, the quality to attempt to download media')
|
||||
choices=SOURCE_RESOLUTION_CHOICES,
|
||||
default=SOURCE_RESOLUTION_1080P,
|
||||
help_text=_('Source resolution, desired video resolution to download')
|
||||
)
|
||||
source_vcodec = models.CharField(
|
||||
_('source video codec'),
|
||||
max_length=8,
|
||||
db_index=True,
|
||||
choices=SOURCE_VCODEC_CHOICES,
|
||||
default=SOURCE_VCODEC_VP9,
|
||||
help_text=_('Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)')
|
||||
)
|
||||
source_acodec = models.CharField(
|
||||
_('source audio codec'),
|
||||
max_length=8,
|
||||
db_index=True,
|
||||
choices=SOURCE_ACODEC_CHOICES,
|
||||
default=SOURCE_ACODEC_OPUS,
|
||||
help_text=_('Source audio codec, desired audio encoding format to download')
|
||||
)
|
||||
prefer_60fps = models.BooleanField(
|
||||
_('prefer 60fps'),
|
||||
default=False,
|
||||
default=True,
|
||||
help_text=_('Where possible, prefer 60fps media for this source')
|
||||
)
|
||||
prefer_hdr = models.BooleanField(
|
||||
|
|
@ -145,21 +180,13 @@ class Source(models.Model):
|
|||
default=False,
|
||||
help_text=_('Where possible, prefer HDR media for this source')
|
||||
)
|
||||
output_format = models.CharField(
|
||||
_('output format'),
|
||||
max_length=8,
|
||||
db_index=True,
|
||||
choices=OUTPUT_FORMAT_CHOICES,
|
||||
default=OUTPUT_FORMAT_MKV,
|
||||
help_text=_('Output format, the file format container in which to save media')
|
||||
)
|
||||
fallback = models.CharField(
|
||||
_('fallback'),
|
||||
max_length=1,
|
||||
db_index=True,
|
||||
choices=FALLBACK_CHOICES,
|
||||
default=FALLBACK_FAIL,
|
||||
help_text=_('What do do when media in your source profile is not available')
|
||||
help_text=_('What do do when media in your source resolution and codecs is not available')
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
|
|
@ -182,19 +209,47 @@ class Source(models.Model):
|
|||
def url(self):
|
||||
return Source.create_url(self.source_type, self.key)
|
||||
|
||||
@property
|
||||
def format_summary(self):
|
||||
if self.source_resolution == Source.SOURCE_RESOLUTION_AUDIO:
|
||||
vc = 'none'
|
||||
else:
|
||||
vc = self.source_vcodec
|
||||
ac = self.source_acodec
|
||||
f = '60FPS' if self.prefer_60fps else ''
|
||||
h = 'HDR' if self.prefer_hdr else ''
|
||||
return f'{self.source_resolution} (video:{vc}, audio:{ac}) {f} {h}'.strip()
|
||||
|
||||
@property
|
||||
def directory_path(self):
|
||||
if self.source_profile == self.SOURCE_PROFILE_AUDIO:
|
||||
if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO:
|
||||
return settings.SYNC_AUDIO_ROOT / self.directory
|
||||
else:
|
||||
return settings.SYNC_VIDEO_ROOT / self.directory
|
||||
|
||||
@property
|
||||
def key_field(self):
|
||||
return self.KEY_FIELD.get(self.source_type, '')
|
||||
|
||||
def index_media(self):
|
||||
'''
|
||||
Index the media source returning a list of media metadata as dicts.
|
||||
'''
|
||||
indexer = self.INDEXERS.get(self.source_type, None)
|
||||
if not callable(indexer):
|
||||
raise Exception(f'Source type f"{self.source_type}" has no indexer')
|
||||
response = indexer(self.url)
|
||||
return response.get('entries', [])
|
||||
|
||||
|
||||
def get_media_thumb_path(instance, filename):
|
||||
fileid = str(instance.uuid)
|
||||
filename = f'{fileid.lower()}.{instance.image_type.lower()}'
|
||||
filename = f'{fileid.lower()}.jpg'
|
||||
prefix = fileid[:2]
|
||||
return os.path.join('thumbs', prefix, filename)
|
||||
return Path('thumbs') / prefix / filename
|
||||
|
||||
|
||||
_metadata_cache = {}
|
||||
|
||||
|
||||
class Media(models.Model):
|
||||
|
|
@ -203,6 +258,11 @@ class Media(models.Model):
|
|||
Source.
|
||||
'''
|
||||
|
||||
URLS = {
|
||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/watch?v={key}',
|
||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}',
|
||||
}
|
||||
|
||||
uuid = models.UUIDField(
|
||||
_('uuid'),
|
||||
primary_key=True,
|
||||
|
|
@ -248,11 +308,13 @@ class Media(models.Model):
|
|||
thumb_width = models.PositiveSmallIntegerField(
|
||||
_('thumb width'),
|
||||
blank=True,
|
||||
null=True,
|
||||
help_text=_('Width (X) of the thumbnail')
|
||||
)
|
||||
thumb_height = models.PositiveSmallIntegerField(
|
||||
_('thumb height'),
|
||||
blank=True,
|
||||
null=True,
|
||||
help_text=_('Height (Y) of the thumbnail')
|
||||
)
|
||||
metadata = models.TextField(
|
||||
|
|
@ -317,3 +379,53 @@ class Media(models.Model):
|
|||
class Meta:
|
||||
verbose_name = _('Media')
|
||||
verbose_name_plural = _('Media')
|
||||
|
||||
@property
|
||||
def loaded_metadata(self):
|
||||
if self.pk in _metadata_cache:
|
||||
return _metadata_cache[self.pk]
|
||||
_metadata_cache[self.pk] = json.loads(self.metadata)
|
||||
return _metadata_cache[self.pk]
|
||||
|
||||
@property
|
||||
def extension(self):
|
||||
'''
|
||||
The extension is also used by youtube-dl to set the output container. As
|
||||
it is possible to quite easily pick combinations of codecs and containers
|
||||
which are invalid (e.g. OPUS audio in an MP4 container) just set this for
|
||||
people. All video is set to mkv containers, audio-only is set to m4a or ogg
|
||||
depending on audio codec.
|
||||
'''
|
||||
if self.source.source_resolution == Source.SOURCE_RESOLUTION_AUDIO:
|
||||
if self.source.source_acodec == Source.SOURCE_ACODEC_M4A:
|
||||
return 'm4a'
|
||||
elif self.source.source_acodec == Source.SOURCE_ACODEC_OPUS:
|
||||
return 'ogg'
|
||||
else:
|
||||
raise ValueError('Unable to choose audio extension, uknown acodec')
|
||||
else:
|
||||
return 'mkv'
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
url = self.URLS.get(self.source.source_type, '')
|
||||
return url.format(key=self.key)
|
||||
|
||||
@property
|
||||
def title(self):
|
||||
return self.loaded_metadata.get('title', '').strip()
|
||||
|
||||
@property
|
||||
def upload_date(self):
|
||||
upload_date_str = self.loaded_metadata.get('upload_date', '').strip()
|
||||
try:
|
||||
return datetime.strptime(upload_date_str, '%Y%m%d')
|
||||
except ValueError as e:
|
||||
return None
|
||||
|
||||
@property
|
||||
def filename(self):
|
||||
upload_date = self.upload_date.strftime('%Y-%m-%d')
|
||||
title = slugify(self.title.replace('&', 'and').replace('+', 'and'))
|
||||
ext = self.extension
|
||||
return f'{upload_date}_{title}.{ext}'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
from django.conf import settings
|
||||
from django.db.models.signals import post_save, post_delete
|
||||
from django.dispatch import receiver
|
||||
|
||||
from .models import Source, Media
|
||||
from .tasks import delete_index_source_task, index_source_task, download_media_thumbnail
|
||||
|
||||
|
||||
@receiver(post_save, sender=Source)
|
||||
def source_post_save(sender, instance, created, **kwargs):
|
||||
# Triggered when a source is saved
|
||||
if created:
|
||||
# If the source is newly created schedule its indexing
|
||||
index_source_task(str(instance.pk), repeat=settings.INDEX_SOURCE_EVERY)
|
||||
|
||||
|
||||
@receiver(post_delete, sender=Source)
|
||||
def source_post_delete(sender, instance, **kwargs):
|
||||
# Triggered when a source is deleted
|
||||
delete_index_source_task(str(instance.pk))
|
||||
|
||||
|
||||
@receiver(post_save, sender=Media)
|
||||
def media_post_save(sender, instance, created, **kwargs):
|
||||
# Triggered when media is saved
|
||||
if created:
|
||||
# If the media is newly created fire a task off to download its thumbnail
|
||||
metadata = instance.loaded_metadata
|
||||
thumbnail_url = metadata.get('thumbnail', '')
|
||||
if thumbnail_url:
|
||||
download_media_thumbnail(str(instance.pk), thumbnail_url)
|
||||
|
||||
|
||||
@receiver(post_delete, sender=Media)
|
||||
def media_post_delete(sender, instance, **kwargs):
|
||||
# Triggered when media is deleted
|
||||
pass
|
||||
# TODO: delete thumbnail and media file from disk
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
'''
|
||||
Start, stop and manage scheduled tasks. These are generally triggered by Django
|
||||
signals (see signals.py).
|
||||
'''
|
||||
|
||||
|
||||
import json
|
||||
from io import BytesIO
|
||||
from django.conf import settings
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from background_task import background
|
||||
from background_task.models import Task
|
||||
from .models import Source, Media
|
||||
from .utils import get_remote_image
|
||||
|
||||
|
||||
def delete_index_source_task(source_id):
|
||||
task = None
|
||||
try:
|
||||
# get_task currently returns a QuerySet, but catch DoesNotExist just in case
|
||||
task = Task.objects.get_task('sync.tasks.index_source_task', args=(source_id,))
|
||||
except Task.DoesNotExist:
|
||||
pass
|
||||
if task:
|
||||
# A scheduled task exists for this Source, delete it
|
||||
task.delete()
|
||||
|
||||
|
||||
@background(schedule=0)
|
||||
def index_source_task(source_id):
|
||||
'''
|
||||
Indexes media available from a Source object.
|
||||
'''
|
||||
try:
|
||||
source = Source.objects.get(pk=source_id)
|
||||
except Source.DoesNotExist:
|
||||
# Task triggered but the Source has been deleted, delete the task
|
||||
delete_index_source_task(source_id)
|
||||
return
|
||||
videos = source.index_media()
|
||||
for video in videos:
|
||||
# Create or update each video as a Media object
|
||||
key = video.get(source.key_field, None)
|
||||
if not key:
|
||||
# Video has no unique key (ID), it can't be indexed
|
||||
continue
|
||||
try:
|
||||
media = Media.objects.get(key=key)
|
||||
except Media.DoesNotExist:
|
||||
media = Media(key=key)
|
||||
media.source = source
|
||||
media.metadata = json.dumps(video)
|
||||
media.save()
|
||||
|
||||
|
||||
@background(schedule=0)
|
||||
def download_media_thumbnail(media_id, url):
|
||||
'''
|
||||
Downloads an image from a URL and saves it as a local thumbnail attached to a
|
||||
Media object.
|
||||
'''
|
||||
try:
|
||||
media = Media.objects.get(pk=media_id)
|
||||
except Media.DoesNotExist:
|
||||
# Task triggered but the media no longer exists, ignore task
|
||||
return
|
||||
i = get_remote_image(url)
|
||||
max_width, max_height = getattr(settings, 'MAX_MEDIA_THUMBNAIL_SIZE', (512, 512))
|
||||
if i.width > max_width or i.height > max_height:
|
||||
# Image is larger than we want to save, resize it
|
||||
i.thumbnail(size=(max_width, max_height))
|
||||
image_file = BytesIO()
|
||||
i.save(image_file, 'JPEG', quality=80, optimize=True, progressive=True)
|
||||
image_file.seek(0)
|
||||
media.thumb.save(
|
||||
'thumb',
|
||||
SimpleUploadedFile(
|
||||
'thumb',
|
||||
image_file.read(),
|
||||
'image/jpeg',
|
||||
),
|
||||
save=True
|
||||
)
|
||||
return True
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
{% extends 'base.html' %}
|
||||
|
||||
{% block headtitle %}Media - {{ media.key }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="row">
|
||||
<div class="col s12">
|
||||
<h1 class="truncate">Media <strong>{{ media.key }}</strong></h1>
|
||||
<p class="truncate"><strong><a href="{{ media.url }}" target="_blank"><i class="fas fa-link"></i> {{ media.url }}</a></strong></p>
|
||||
<p class="truncate">Saving to: <strong>{{ media.source.directory_path }}</strong></p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col s12">
|
||||
<table class="striped">
|
||||
<tr title="The media source">
|
||||
<td class="hide-on-small-only">Source</td>
|
||||
<td><span class="hide-on-med-and-up">Source<br></span><strong><a href="{% url 'sync:source' pk=media.source.pk %}">{{ media.source }}</a></strong></td>
|
||||
</tr>
|
||||
<tr title="The media title">
|
||||
<td class="hide-on-small-only">Title</td>
|
||||
<td><span class="hide-on-med-and-up">Title<br></span><strong>{{ media.title }}</strong></td>
|
||||
</tr>
|
||||
<tr title="The filename the media will be downloaded as">
|
||||
<td class="hide-on-small-only">Filename</td>
|
||||
<td><span class="hide-on-med-and-up">Filename<br></span><strong>{{ media.filename }}</strong></td>
|
||||
</tr>
|
||||
<tr title="The desired format">
|
||||
<td class="hide-on-small-only">Desired format</td>
|
||||
<td><span class="hide-on-med-and-up">Desired format<br></span><strong>{{ media.source.format_summary }}</strong></td>
|
||||
</tr>
|
||||
<tr title="Has the media been downloaded">
|
||||
<td class="hide-on-small-only">Downloaded</td>
|
||||
<td><span class="hide-on-med-and-up">Downloaded<br></span><strong>{% if media.downloaded %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
|
@ -1,11 +1,30 @@
|
|||
{% extends 'base.html' %}
|
||||
{% extends 'base.html' %}{% load static %}
|
||||
|
||||
{% block headtitle %}Media{% endblock %}
|
||||
{% block headtitle %}Media{% if source %} - {{ source }}{% endif %}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="row">
|
||||
{% include 'infobox.html' with message=message %}
|
||||
<div class="row no-margin-bottom">
|
||||
{% for m in media %}
|
||||
<div class="col s12 m6 l4 xl3">
|
||||
<a href="{% url 'sync:media-item' pk=m.pk %}" class="collection-item">
|
||||
<div class="card mediacard">
|
||||
<div class="card-image">
|
||||
<img src="{% if m.thumb %}{% url 'sync:media-thumb' pk=m.pk %}{% else %}{% static 'images/nothumb.jpg' %} {% endif %}">
|
||||
<span class="card-title truncate">{{ m.source }}<br>
|
||||
<span>{{ m }}</span>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
{% empty %}
|
||||
<div class="col s12">
|
||||
media
|
||||
<div class="collection">
|
||||
<span class="collection-item no-items"><i class="fas fa-info-circle"></i> No media has been indexed.</span>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% include 'pagination.html' with pagination=sources.paginator %}
|
||||
{% endblock %}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from django.urls import path
|
||||
from .views import (DashboardView, SourcesView, ValidateSourceView, AddSourceView,
|
||||
SourceView, UpdateSourceView, DeleteSourceView, MediaView,
|
||||
TasksView, LogsView)
|
||||
MediaThumbView, MediaItemView, TasksView, LogsView)
|
||||
|
||||
|
||||
app_name = 'sync'
|
||||
|
|
@ -41,12 +41,20 @@ urlpatterns = [
|
|||
DeleteSourceView.as_view(),
|
||||
name='delete-source'),
|
||||
|
||||
# Media URLs (note /media/ is the static media URL, don't use that)
|
||||
# Media URLs
|
||||
|
||||
path('mediafiles',
|
||||
path('media',
|
||||
MediaView.as_view(),
|
||||
name='media'),
|
||||
|
||||
path('media-thumb/<uuid:pk>',
|
||||
MediaThumbView.as_view(),
|
||||
name='media-thumb'),
|
||||
|
||||
path('media-item/<uuid:pk>',
|
||||
MediaItemView.as_view(),
|
||||
name='media-item'),
|
||||
|
||||
# Task URLs
|
||||
|
||||
path('tasks',
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import re
|
||||
import requests
|
||||
from PIL import Image
|
||||
from urllib.parse import urlsplit, parse_qs
|
||||
from django.forms import ValidationError
|
||||
|
||||
|
|
@ -42,3 +44,13 @@ def validate_url(url, validator):
|
|||
elif extract_from == 'qs_args':
|
||||
extract_value = url_query_parts[extract_param][0]
|
||||
return extract_value
|
||||
|
||||
|
||||
def get_remote_image(url):
|
||||
headers = {
|
||||
'user-agent': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||||
'(KHTML, like Gecko) Chrome/69.0.3497.64 Safari/537.36')
|
||||
}
|
||||
r = requests.get(url, headers=headers, stream=True, timeout=60)
|
||||
r.raw.decode_content = True
|
||||
return Image.open(r.raw)
|
||||
|
|
|
|||
|
|
@ -1,16 +1,19 @@
|
|||
from base64 import b64decode
|
||||
from django.conf import settings
|
||||
from django.http import Http404
|
||||
from django.views.generic import TemplateView, ListView, DetailView
|
||||
from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateView,
|
||||
DeleteView)
|
||||
from django.http import HttpResponse
|
||||
from django.urls import reverse_lazy
|
||||
from django.forms import ValidationError
|
||||
from django.utils.text import slugify
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from common.utils import append_uri_params
|
||||
from .models import Source
|
||||
from .models import Source, Media
|
||||
from .forms import ValidateSourceForm, ConfirmDeleteSourceForm
|
||||
from .utils import validate_url
|
||||
from . import signals
|
||||
from . import youtube
|
||||
|
||||
|
||||
|
|
@ -196,8 +199,8 @@ class AddSourceView(CreateView):
|
|||
template_name = 'sync/source-add.html'
|
||||
model = Source
|
||||
fields = ('source_type', 'key', 'name', 'directory', 'delete_old_media',
|
||||
'days_to_keep', 'source_profile', 'prefer_60fps', 'prefer_hdr',
|
||||
'output_format', 'fallback')
|
||||
'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
|
||||
'prefer_60fps', 'prefer_hdr', 'fallback')
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.prepopulated_data = {}
|
||||
|
|
@ -240,8 +243,8 @@ class UpdateSourceView(UpdateView):
|
|||
template_name = 'sync/source-update.html'
|
||||
model = Source
|
||||
fields = ('source_type', 'key', 'name', 'directory', 'delete_old_media',
|
||||
'days_to_keep', 'source_profile', 'prefer_60fps', 'prefer_hdr',
|
||||
'output_format', 'fallback')
|
||||
'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
|
||||
'prefer_60fps', 'prefer_hdr', 'fallback')
|
||||
|
||||
def get_success_url(self):
|
||||
url = reverse_lazy('sync:sources')
|
||||
|
|
@ -272,16 +275,76 @@ class DeleteSourceView(DeleteView, FormMixin):
|
|||
return append_uri_params(url, {'message': 'source-deleted'})
|
||||
|
||||
|
||||
class MediaView(TemplateView):
|
||||
class MediaView(ListView):
|
||||
'''
|
||||
A bare list of media added with their states.
|
||||
'''
|
||||
|
||||
template_name = 'sync/media.html'
|
||||
context_object_name = 'media'
|
||||
paginate_by = settings.MEDIA_PER_PAGE
|
||||
messages = {
|
||||
'filter': _('Viewing media for source: <strong>{name}</strong>'),
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.filter_source = None
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def dispatch(self, request, *args, **kwargs):
|
||||
filter_by = request.GET.get('filter', '')
|
||||
if filter_by:
|
||||
try:
|
||||
self.filter_source = Source.objects.get(pk=filter_by)
|
||||
except Source.DoesNotExist:
|
||||
self.filter_source = None
|
||||
return super().dispatch(request, *args, **kwargs)
|
||||
|
||||
def get_queryset(self):
|
||||
if self.filter_source:
|
||||
return Media.objects.filter(source=self.filter_source).order_by('-created')
|
||||
else:
|
||||
return Media.objects.all().order_by('-created')
|
||||
|
||||
def get_context_data(self, *args, **kwargs):
|
||||
data = super().get_context_data(*args, **kwargs)
|
||||
data['message'] = ''
|
||||
data['source'] = None
|
||||
if self.filter_source:
|
||||
message = str(self.messages.get('filter', ''))
|
||||
print(message)
|
||||
data['message'] = message.format(name=self.filter_source.name)
|
||||
data['source'] = self.filter_source
|
||||
print(data)
|
||||
return data
|
||||
|
||||
|
||||
class MediaThumbView(DetailView):
|
||||
'''
|
||||
Shows a media thumbnail. Whitenose doesn't support post-start media image
|
||||
serving and the images here are pretty small, just serve them manually. This
|
||||
isn't fast, but it's not likely to be a serious bottleneck.
|
||||
'''
|
||||
|
||||
model = Media
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
media = self.get_object()
|
||||
if media.thumb:
|
||||
thumb = open(media.thumb.path, 'rb').read()
|
||||
content_type = 'image/jpeg'
|
||||
else:
|
||||
thumb = b64decode('R0lGODlhAQABAIABAP///wAAACH5BAEKAAEALAA'
|
||||
'AAAABAAEAAAICTAEAOw==')
|
||||
content_type = 'image/gif'
|
||||
return HttpResponse(thumb, content_type=content_type)
|
||||
|
||||
|
||||
class MediaItemView(DetailView):
|
||||
|
||||
template_name = 'sync/media-item.html'
|
||||
model = Media
|
||||
|
||||
|
||||
class TasksView(TemplateView):
|
||||
'''
|
||||
|
|
|
|||
|
|
@ -15,22 +15,20 @@ class YouTubeError(youtube_dl.utils.DownloadError):
|
|||
'''
|
||||
Generic wrapped error for all errors that could be raised by youtube-dl.
|
||||
'''
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def extract_info(url):
|
||||
def get_media_info(url):
|
||||
'''
|
||||
Extracts information from a YouTube URL and returns it as a dict. For a channel
|
||||
or playlist this returns a dict of all the videos on the channel or playlist
|
||||
as well as associated metadata.
|
||||
'''
|
||||
|
||||
opts = _defaults.update({
|
||||
'skip_download': True,
|
||||
'forcejson': True,
|
||||
'simulate': True,
|
||||
'extract_flat': 'in_playlist',
|
||||
'playlist_items': 1,
|
||||
})
|
||||
response = {}
|
||||
with youtube_dl.YoutubeDL(opts) as y:
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ USE_TZ = True
|
|||
|
||||
STATIC_URL = '/static/'
|
||||
STATIC_ROOT = BASE_DIR / 'static'
|
||||
MEDIA_URL = '/media/'
|
||||
#MEDIA_URL = '/media/'
|
||||
MEDIA_ROOT = BASE_DIR / 'media'
|
||||
SYNC_VIDEO_ROOT = BASE_DIR / 'downloads' / 'video'
|
||||
SYNC_AUDIO_ROOT = BASE_DIR / 'downloads' / 'audio'
|
||||
|
|
@ -121,10 +121,17 @@ BACKGROUND_TASK_PRIORITY_ORDERING = 'DESC' # Process high priority tasks first
|
|||
|
||||
|
||||
SOURCES_PER_PAGE = 25
|
||||
MEDIA_PER_PAGE = 25
|
||||
|
||||
|
||||
INDEX_SOURCE_EVERY = 60 # Seconds between indexing sources, 21600 = every 6 hours
|
||||
|
||||
|
||||
MAX_MEDIA_THUMBNAIL_SIZE = (320, 240) # Max size in pixels for media thumbnails
|
||||
|
||||
|
||||
YOUTUBE_DEFAULTS = {
|
||||
'age_limit': 99,
|
||||
'age_limit': 99, # Age in years to spoof the client as
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue