media downloading
This commit is contained in:
parent
602eb9b90f
commit
90a8918e40
|
@ -4,3 +4,19 @@ class NoMediaException(Exception):
|
||||||
playlist name or similar, or the upstream source returned an error.
|
playlist name or similar, or the upstream source returned an error.
|
||||||
'''
|
'''
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class NoFormatException(Exception):
|
||||||
|
'''
|
||||||
|
Raised when a media item is attempted to be downloaded but it has no valid
|
||||||
|
format combination.
|
||||||
|
'''
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadFailedException(Exception):
|
||||||
|
'''
|
||||||
|
Raised when a downloaded media file is expected to be present, but doesn't
|
||||||
|
exist.
|
||||||
|
'''
|
||||||
|
pass
|
||||||
|
|
|
@ -77,7 +77,7 @@ main {
|
||||||
|
|
||||||
h2 {
|
h2 {
|
||||||
margin: 0;
|
margin: 0;
|
||||||
padding: 2rem 0 0.5rem 0;
|
padding: 1.5rem 0 1rem 0;
|
||||||
font-size: 1.5rem;
|
font-size: 1.5rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
<div class="col s12">
|
<div class="col s12">
|
||||||
<div class="card infobox">
|
<div class="card infobox">
|
||||||
<div class="card-content">
|
<div class="card-content">
|
||||||
<i class="fas fa-info-circle"></i> {{ message|safe }}
|
<i class="fas fa-fw fa-info-circle"></i> {{ message|safe }}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -0,0 +1,24 @@
|
||||||
|
# Generated by Django 3.1.4 on 2020-12-09 08:57
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
import sync.models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('sync', '0018_media_can_download'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='media',
|
||||||
|
name='media_file',
|
||||||
|
field=models.FileField(blank=True, help_text='Media file', max_length=200, null=True, upload_to=sync.models.get_media_file_path, verbose_name='media file'),
|
||||||
|
),
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name='media',
|
||||||
|
name='thumb',
|
||||||
|
field=models.ImageField(blank=True, height_field='thumb_height', help_text='Thumbnail', max_length=200, null=True, upload_to=sync.models.get_media_thumb_path, verbose_name='thumb', width_field='thumb_width'),
|
||||||
|
),
|
||||||
|
]
|
|
@ -1,3 +1,4 @@
|
||||||
|
import os
|
||||||
import uuid
|
import uuid
|
||||||
import json
|
import json
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
@ -6,7 +7,9 @@ from django.conf import settings
|
||||||
from django.db import models
|
from django.db import models
|
||||||
from django.utils.text import slugify
|
from django.utils.text import slugify
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
from .youtube import get_media_info as get_youtube_media_info
|
from common.errors import NoFormatException
|
||||||
|
from .youtube import (get_media_info as get_youtube_media_info,
|
||||||
|
download_media as download_youtube_media)
|
||||||
from .utils import seconds_to_timestr, parse_media_format
|
from .utils import seconds_to_timestr, parse_media_format
|
||||||
from .matching import (get_best_combined_format, get_best_audio_format,
|
from .matching import (get_best_combined_format, get_best_audio_format,
|
||||||
get_best_video_format)
|
get_best_video_format)
|
||||||
|
@ -297,6 +300,13 @@ class Source(models.Model):
|
||||||
else:
|
else:
|
||||||
return settings.SYNC_VIDEO_ROOT / self.directory
|
return settings.SYNC_VIDEO_ROOT / self.directory
|
||||||
|
|
||||||
|
def make_directory(self):
|
||||||
|
return os.makedirs(self.directory_path, exist_ok=True)
|
||||||
|
|
||||||
|
def directory_exists(self):
|
||||||
|
return (os.path.isdir(self.directory_path) and
|
||||||
|
os.access(self.directory_path, os.W_OK))
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def key_field(self):
|
def key_field(self):
|
||||||
return self.KEY_FIELD.get(self.source_type, '')
|
return self.KEY_FIELD.get(self.source_type, '')
|
||||||
|
@ -344,6 +354,10 @@ def get_media_thumb_path(instance, filename):
|
||||||
return Path('thumbs') / prefix / filename
|
return Path('thumbs') / prefix / filename
|
||||||
|
|
||||||
|
|
||||||
|
def get_media_file_path(instance, filename):
|
||||||
|
return instance.filepath
|
||||||
|
|
||||||
|
|
||||||
class Media(models.Model):
|
class Media(models.Model):
|
||||||
'''
|
'''
|
||||||
Media is a single piece of media, such as a single YouTube video linked to a
|
Media is a single piece of media, such as a single YouTube video linked to a
|
||||||
|
@ -414,7 +428,7 @@ class Media(models.Model):
|
||||||
thumb = models.ImageField(
|
thumb = models.ImageField(
|
||||||
_('thumb'),
|
_('thumb'),
|
||||||
upload_to=get_media_thumb_path,
|
upload_to=get_media_thumb_path,
|
||||||
max_length=100,
|
max_length=200,
|
||||||
blank=True,
|
blank=True,
|
||||||
null=True,
|
null=True,
|
||||||
width_field='thumb_width',
|
width_field='thumb_width',
|
||||||
|
@ -445,6 +459,14 @@ class Media(models.Model):
|
||||||
default=False,
|
default=False,
|
||||||
help_text=_('Media has a matching format and can be downloaded')
|
help_text=_('Media has a matching format and can be downloaded')
|
||||||
)
|
)
|
||||||
|
media_file = models.FileField(
|
||||||
|
_('media file'),
|
||||||
|
upload_to=get_media_file_path,
|
||||||
|
max_length=200,
|
||||||
|
blank=True,
|
||||||
|
null=True,
|
||||||
|
help_text=_('Media file')
|
||||||
|
)
|
||||||
downloaded = models.BooleanField(
|
downloaded = models.BooleanField(
|
||||||
_('downloaded'),
|
_('downloaded'),
|
||||||
db_index=True,
|
db_index=True,
|
||||||
|
@ -501,6 +523,9 @@ class Media(models.Model):
|
||||||
class Meta:
|
class Meta:
|
||||||
verbose_name = _('Media')
|
verbose_name = _('Media')
|
||||||
verbose_name_plural = _('Media')
|
verbose_name_plural = _('Media')
|
||||||
|
unique_together = (
|
||||||
|
('source', 'key'),
|
||||||
|
)
|
||||||
|
|
||||||
def get_metadata_field(self, field):
|
def get_metadata_field(self, field):
|
||||||
fields = self.METADATA_FIELDS.get(field, {})
|
fields = self.METADATA_FIELDS.get(field, {})
|
||||||
|
@ -539,17 +564,25 @@ class Media(models.Model):
|
||||||
audio_match, audio_format = self.get_best_audio_format()
|
audio_match, audio_format = self.get_best_audio_format()
|
||||||
video_match, video_format = self.get_best_video_format()
|
video_match, video_format = self.get_best_video_format()
|
||||||
if audio_format and video_format:
|
if audio_format and video_format:
|
||||||
return f'{audio_format}+{video_format}'
|
return f'{video_format}+{audio_format}'
|
||||||
else:
|
else:
|
||||||
return False
|
return False
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def get_format_by_code(self, format_code):
|
||||||
|
'''
|
||||||
|
Matches a format code, such as '22', to a processed format dict.
|
||||||
|
'''
|
||||||
|
for fmt in self.iter_formats():
|
||||||
|
if format_code == fmt['id']:
|
||||||
|
return fmt
|
||||||
|
return False
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def loaded_metadata(self):
|
def loaded_metadata(self):
|
||||||
try:
|
try:
|
||||||
return json.loads(self.metadata)
|
return json.loads(self.metadata)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print('!!!!', e)
|
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
@ -557,6 +590,11 @@ class Media(models.Model):
|
||||||
url = self.URLS.get(self.source.source_type, '')
|
url = self.URLS.get(self.source.source_type, '')
|
||||||
return url.format(key=self.key)
|
return url.format(key=self.key)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def description(self):
|
||||||
|
field = self.get_metadata_field('description')
|
||||||
|
return self.loaded_metadata.get(field, '').strip()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def title(self):
|
def title(self):
|
||||||
field = self.get_metadata_field('title')
|
field = self.get_metadata_field('title')
|
||||||
|
@ -599,11 +637,31 @@ class Media(models.Model):
|
||||||
dateobj = upload_date if upload_date else self.created
|
dateobj = upload_date if upload_date else self.created
|
||||||
datestr = dateobj.strftime('%Y-%m-%d')
|
datestr = dateobj.strftime('%Y-%m-%d')
|
||||||
source_name = slugify(self.source.name)
|
source_name = slugify(self.source.name)
|
||||||
name = slugify(self.name.replace('&', 'and').replace('+', 'and'))
|
name = slugify(self.name.replace('&', 'and').replace('+', 'and'))[:50]
|
||||||
|
key = self.key.strip()
|
||||||
|
fmt = self.source.source_resolution.lower()
|
||||||
|
codecs = []
|
||||||
|
vcodec = self.source.source_vcodec.lower()
|
||||||
|
acodec = self.source.source_acodec.lower()
|
||||||
|
if vcodec:
|
||||||
|
codecs.append(vcodec)
|
||||||
|
if acodec:
|
||||||
|
codecs.append(acodec)
|
||||||
|
codecs = '-'.join(codecs)
|
||||||
ext = self.source.extension
|
ext = self.source.extension
|
||||||
fn = f'{datestr}_{source_name}_{name}'[:100]
|
return f'{datestr}_{source_name}_{name}_{key}-{fmt}-{codecs}.{ext}'
|
||||||
return f'{fn}.{ext}'
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def filepath(self):
|
def filepath(self):
|
||||||
return self.source.directory_path / self.filename
|
return self.source.directory_path / self.filename
|
||||||
|
|
||||||
|
def download_media(self):
|
||||||
|
format_str = self.get_format_str()
|
||||||
|
if not format_str:
|
||||||
|
raise NoFormatException(f'Cannot download, media "{self.pk}" ({media}) has '
|
||||||
|
f'no valid format available')
|
||||||
|
# Download the media with youtube-dl
|
||||||
|
download_youtube_media(self.url, format_str, self.source.extension,
|
||||||
|
str(self.filepath))
|
||||||
|
# Return the download paramaters
|
||||||
|
return format_str, self.source.extension
|
||||||
|
|
|
@ -6,8 +6,9 @@ from background_task.signals import task_failed
|
||||||
from background_task.models import Task
|
from background_task.models import Task
|
||||||
from common.logger import log
|
from common.logger import log
|
||||||
from .models import Source, Media
|
from .models import Source, Media
|
||||||
from .tasks import (delete_task, index_source_task, download_media_thumbnail,
|
from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task,
|
||||||
map_task_to_instance)
|
download_media_thumbnail, map_task_to_instance,
|
||||||
|
check_source_directory_exists, download_media)
|
||||||
from .utils import delete_file
|
from .utils import delete_file
|
||||||
|
|
||||||
|
|
||||||
|
@ -22,7 +23,7 @@ def source_pre_save(sender, instance, **kwargs):
|
||||||
return
|
return
|
||||||
if existing_source.index_schedule != instance.index_schedule:
|
if existing_source.index_schedule != instance.index_schedule:
|
||||||
# Indexing schedule has changed, recreate the indexing task
|
# Indexing schedule has changed, recreate the indexing task
|
||||||
delete_task('sync.tasks.index_source_task', instance.pk)
|
delete_task_by_source('sync.tasks.index_source_task', instance.pk)
|
||||||
verbose_name = _('Index media from source "{}"')
|
verbose_name = _('Index media from source "{}"')
|
||||||
index_source_task(
|
index_source_task(
|
||||||
str(instance.pk),
|
str(instance.pk),
|
||||||
|
@ -34,10 +35,11 @@ def source_pre_save(sender, instance, **kwargs):
|
||||||
|
|
||||||
@receiver(post_save, sender=Source)
|
@receiver(post_save, sender=Source)
|
||||||
def source_post_save(sender, instance, created, **kwargs):
|
def source_post_save(sender, instance, created, **kwargs):
|
||||||
# Triggered after a source is saved
|
# Triggered after a source is saved, Create a new task to check the directory exists
|
||||||
|
check_source_directory_exists(str(instance.pk))
|
||||||
if created:
|
if created:
|
||||||
# Create a new indexing task for newly created sources
|
# Create a new indexing task for newly created sources
|
||||||
delete_task('sync.tasks.index_source_task', instance.pk)
|
delete_task_by_source('sync.tasks.index_source_task', instance.pk)
|
||||||
log.info(f'Scheduling media indexing for source: {instance.name}')
|
log.info(f'Scheduling media indexing for source: {instance.name}')
|
||||||
verbose_name = _('Index media from source "{}"')
|
verbose_name = _('Index media from source "{}"')
|
||||||
index_source_task(
|
index_source_task(
|
||||||
|
@ -65,7 +67,7 @@ def source_pre_delete(sender, instance, **kwargs):
|
||||||
def source_post_delete(sender, instance, **kwargs):
|
def source_post_delete(sender, instance, **kwargs):
|
||||||
# Triggered after a source is deleted
|
# Triggered after a source is deleted
|
||||||
log.info(f'Deleting tasks for source: {instance.name}')
|
log.info(f'Deleting tasks for source: {instance.name}')
|
||||||
delete_task('sync.tasks.index_source_task', instance.pk)
|
delete_task_by_source('sync.tasks.index_source_task', instance.pk)
|
||||||
|
|
||||||
|
|
||||||
@receiver(task_failed, sender=Task)
|
@receiver(task_failed, sender=Task)
|
||||||
|
@ -88,7 +90,7 @@ def media_post_save(sender, instance, created, **kwargs):
|
||||||
if thumbnail_url:
|
if thumbnail_url:
|
||||||
log.info(f'Scheduling task to download thumbnail for: {instance.name} '
|
log.info(f'Scheduling task to download thumbnail for: {instance.name} '
|
||||||
f'from: {thumbnail_url}')
|
f'from: {thumbnail_url}')
|
||||||
verbose_name = _('Downloading media thumbnail for "{}"')
|
verbose_name = _('Downloading thumbnail for "{}"')
|
||||||
download_media_thumbnail(
|
download_media_thumbnail(
|
||||||
str(instance.pk),
|
str(instance.pk),
|
||||||
thumbnail_url,
|
thumbnail_url,
|
||||||
|
@ -105,13 +107,22 @@ def media_post_save(sender, instance, created, **kwargs):
|
||||||
if instance.can_download:
|
if instance.can_download:
|
||||||
instance.can_download = True
|
instance.can_download = True
|
||||||
instance.save()
|
instance.save()
|
||||||
|
# If the media has not yet been downloaded schedule it to be downloaded
|
||||||
|
if not instance.downloaded:
|
||||||
|
delete_task_by_media('sync.tasks.download_media', instance.pk)
|
||||||
|
verbose_name = _('Downloading media for "{}"')
|
||||||
|
download_media(
|
||||||
|
str(instance.pk),
|
||||||
|
queue=str(instance.source.pk),
|
||||||
|
verbose_name=verbose_name.format(instance.name)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@receiver(pre_delete, sender=Media)
|
@receiver(pre_delete, sender=Media)
|
||||||
def media_pre_delete(sender, instance, **kwargs):
|
def media_pre_delete(sender, instance, **kwargs):
|
||||||
# Triggered before media is deleted, delete any scheduled tasks
|
# Triggered before media is deleted, delete any scheduled tasks
|
||||||
log.info(f'Deleting tasks for media: {instance.name}')
|
log.info(f'Deleting tasks for media: {instance.name}')
|
||||||
delete_task('sync.tasks.download_media_thumbnail', instance.source.pk)
|
delete_task_by_media('sync.tasks.download_media_thumbnail', instance.pk)
|
||||||
# Delete media thumbnail if it exists
|
# Delete media thumbnail if it exists
|
||||||
if instance.thumb:
|
if instance.thumb:
|
||||||
log.info(f'Deleting thumbnail for: {instance} path: {instance.thumb.path}')
|
log.info(f'Deleting thumbnail for: {instance} path: {instance.thumb.path}')
|
||||||
|
|
|
@ -4,6 +4,7 @@
|
||||||
'''
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
import os
|
||||||
import json
|
import json
|
||||||
import math
|
import math
|
||||||
import uuid
|
import uuid
|
||||||
|
@ -18,7 +19,7 @@ from django.db.utils import IntegrityError
|
||||||
from background_task import background
|
from background_task import background
|
||||||
from background_task.models import Task, CompletedTask
|
from background_task.models import Task, CompletedTask
|
||||||
from common.logger import log
|
from common.logger import log
|
||||||
from common.errors import NoMediaException
|
from common.errors import NoMediaException, DownloadFailedException
|
||||||
from .models import Source, Media
|
from .models import Source, Media
|
||||||
from .utils import get_remote_image, resize_image_to_height
|
from .utils import get_remote_image, resize_image_to_height
|
||||||
|
|
||||||
|
@ -39,23 +40,14 @@ def map_task_to_instance(task):
|
||||||
'''
|
'''
|
||||||
TASK_MAP = {
|
TASK_MAP = {
|
||||||
'sync.tasks.index_source_task': Source,
|
'sync.tasks.index_source_task': Source,
|
||||||
|
'sync.tasks.check_source_directory_exists': Source,
|
||||||
'sync.tasks.download_media_thumbnail': Media,
|
'sync.tasks.download_media_thumbnail': Media,
|
||||||
|
'sync.tasks.download_media': Media,
|
||||||
}
|
}
|
||||||
MODEL_URL_MAP = {
|
MODEL_URL_MAP = {
|
||||||
Source: 'sync:source',
|
Source: 'sync:source',
|
||||||
Media: 'sync:media-item',
|
Media: 'sync:media-item',
|
||||||
}
|
}
|
||||||
# If the task has a UUID set in its .queue it's probably a link to a Source
|
|
||||||
if task.queue:
|
|
||||||
try:
|
|
||||||
queue_uuid = uuid.UUID(task.queue)
|
|
||||||
try:
|
|
||||||
url = MODEL_URL_MAP.get(Source, None)
|
|
||||||
return Source.objects.get(pk=task.queue), url
|
|
||||||
except Source.DoesNotExist:
|
|
||||||
pass
|
|
||||||
except (TypeError, ValueError, AttributeError):
|
|
||||||
pass
|
|
||||||
# Unpack
|
# Unpack
|
||||||
task_func, task_args_str = task.task_name, task.task_params
|
task_func, task_args_str = task.task_name, task.task_params
|
||||||
model = TASK_MAP.get(task_func, None)
|
model = TASK_MAP.get(task_func, None)
|
||||||
|
@ -82,6 +74,7 @@ def map_task_to_instance(task):
|
||||||
instance = model.objects.get(pk=instance_uuid)
|
instance = model.objects.get(pk=instance_uuid)
|
||||||
return instance, url
|
return instance, url
|
||||||
except model.DoesNotExist:
|
except model.DoesNotExist:
|
||||||
|
print('!!!', model, instance_uuid)
|
||||||
return None, None
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
|
@ -111,10 +104,14 @@ def get_source_completed_tasks(source_id, only_errors=False):
|
||||||
return CompletedTask.objects.filter(**q).order_by('-failed_at')
|
return CompletedTask.objects.filter(**q).order_by('-failed_at')
|
||||||
|
|
||||||
|
|
||||||
def delete_task(task_name, source_id):
|
def delete_task_by_source(task_name, source_id):
|
||||||
return Task.objects.filter(task_name=task_name, queue=str(source_id)).delete()
|
return Task.objects.filter(task_name=task_name, queue=str(source_id)).delete()
|
||||||
|
|
||||||
|
|
||||||
|
def delete_task_by_media(task_name, media_id):
|
||||||
|
return Task.objects.drop_task(task_name, args=(str(media_id),))
|
||||||
|
|
||||||
|
|
||||||
def cleanup_completed_tasks():
|
def cleanup_completed_tasks():
|
||||||
days_to_keep = getattr(settings, 'COMPLETED_TASKS_DAYS_TO_KEEP', 30)
|
days_to_keep = getattr(settings, 'COMPLETED_TASKS_DAYS_TO_KEEP', 30)
|
||||||
delta = timezone.now() - timedelta(days=days_to_keep)
|
delta = timezone.now() - timedelta(days=days_to_keep)
|
||||||
|
@ -172,16 +169,37 @@ def index_source_task(source_id):
|
||||||
cleanup_completed_tasks()
|
cleanup_completed_tasks()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@background(schedule=0)
|
||||||
|
def check_source_directory_exists(source_id):
|
||||||
|
'''
|
||||||
|
Checks the output directory for a source exists and is writable, if it does
|
||||||
|
not attempt to create it. This is a task so if there are permission errors
|
||||||
|
they are logged as failed tasks.
|
||||||
|
'''
|
||||||
|
try:
|
||||||
|
source = Source.objects.get(pk=source_id)
|
||||||
|
except Source.DoesNotExist:
|
||||||
|
# Task triggered but the Source has been deleted, delete the task
|
||||||
|
delete_index_source_task(source_id)
|
||||||
|
return
|
||||||
|
# Check the source output directory exists
|
||||||
|
if not source.directory_exists():
|
||||||
|
# Try and create it
|
||||||
|
log.info(f'Creating directory: {source.directory_path}')
|
||||||
|
source.make_directory()
|
||||||
|
|
||||||
|
|
||||||
@background(schedule=0)
|
@background(schedule=0)
|
||||||
def download_media_thumbnail(media_id, url):
|
def download_media_thumbnail(media_id, url):
|
||||||
'''
|
'''
|
||||||
Downloads an image from a URL and save it as a local thumbnail attached to a
|
Downloads an image from a URL and save it as a local thumbnail attached to a
|
||||||
Media object.
|
Media instance.
|
||||||
'''
|
'''
|
||||||
try:
|
try:
|
||||||
media = Media.objects.get(pk=media_id)
|
media = Media.objects.get(pk=media_id)
|
||||||
except Media.DoesNotExist:
|
except Media.DoesNotExist:
|
||||||
# Task triggered but the media no longer exists, ignore task
|
# Task triggered but the media no longer exists, do nothing
|
||||||
return
|
return
|
||||||
width = getattr(settings, 'MEDIA_THUMBNAIL_WIDTH', 430)
|
width = getattr(settings, 'MEDIA_THUMBNAIL_WIDTH', 430)
|
||||||
height = getattr(settings, 'MEDIA_THUMBNAIL_HEIGHT', 240)
|
height = getattr(settings, 'MEDIA_THUMBNAIL_HEIGHT', 240)
|
||||||
|
@ -203,3 +221,51 @@ def download_media_thumbnail(media_id, url):
|
||||||
)
|
)
|
||||||
log.info(f'Saved thumbnail for: {media} from: {url}')
|
log.info(f'Saved thumbnail for: {media} from: {url}')
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
@background(schedule=0)
|
||||||
|
def download_media(media_id):
|
||||||
|
'''
|
||||||
|
Downloads the media to disk and attaches it to the Media instance.
|
||||||
|
'''
|
||||||
|
try:
|
||||||
|
media = Media.objects.get(pk=media_id)
|
||||||
|
except Media.DoesNotExist:
|
||||||
|
# Task triggered but the media no longer exists, do nothing
|
||||||
|
return
|
||||||
|
log.info(f'Downloading media: {media} (UUID: {media.pk}) to: {media.filepath}')
|
||||||
|
format_str, container = media.download_media()
|
||||||
|
if os.path.exists(media.filepath):
|
||||||
|
# Media has been downloaded successfully
|
||||||
|
log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: '
|
||||||
|
f'{media.filepath}')
|
||||||
|
# Link the media file to the object and update info about the download
|
||||||
|
media.media_file.name = str(media.filepath)
|
||||||
|
media.downloaded = True
|
||||||
|
if '+' in format_str:
|
||||||
|
vformat_code, aformat_code = format_str.split('+')
|
||||||
|
aformat = media.get_format_by_code(aformat_code)
|
||||||
|
vformat = media.get_format_by_code(vformat_code)
|
||||||
|
media.downloaded_audio_codec = aformat['acodec']
|
||||||
|
media.downloaded_video_codec = vformat['vcodec']
|
||||||
|
media.downloaded_container = container
|
||||||
|
media.downloaded_fps = vformat['fps']
|
||||||
|
media.downloaded_hdr = vformat['is_hdr']
|
||||||
|
media.downloaded_filesize = os.path.getsize(media.filepath)
|
||||||
|
else:
|
||||||
|
cformat_code = format_str
|
||||||
|
cformat = media.get_format_by_code(cformat_code)
|
||||||
|
media.downloaded_audio_codec = cformat['acodec']
|
||||||
|
media.downloaded_video_codec = cformat['vcodec']
|
||||||
|
media.downloaded_container = container
|
||||||
|
media.downloaded_fps = cformat['fps']
|
||||||
|
media.downloaded_hdr = cformat['is_hdr']
|
||||||
|
media.downloaded_filesize = os.path.getsize(media.filepath)
|
||||||
|
media.save()
|
||||||
|
else:
|
||||||
|
# Expected file doesn't exist on disk
|
||||||
|
err = (f'Failed to download media: {media} (UUID: {media.pk}) to disk, '
|
||||||
|
f'expected outfile does not exist: {media.filepath}')
|
||||||
|
log.error(err)
|
||||||
|
# Raising an error here triggers the task to be re-attempted (or fail)
|
||||||
|
raise DownloadFailedException(err)
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
{% extends 'base.html' %}
|
{% extends 'base.html' %}{% load static %}
|
||||||
|
|
||||||
{% block headtitle %}Media - {{ media.key }}{% endblock %}
|
{% block headtitle %}Media - {{ media.key }}{% endblock %}
|
||||||
|
|
||||||
|
@ -6,11 +6,26 @@
|
||||||
<div class="row">
|
<div class="row">
|
||||||
<div class="col s12">
|
<div class="col s12">
|
||||||
<h1 class="truncate">Media <strong>{{ media.key }}</strong></h1>
|
<h1 class="truncate">Media <strong>{{ media.key }}</strong></h1>
|
||||||
|
{% if media.title %}<h2 class="truncate"><strong>{{ media.title }}</strong></h2>{% endif %}
|
||||||
<p class="truncate"><strong><a href="{{ media.url }}" target="_blank"><i class="fas fa-link"></i> {{ media.url }}</a></strong></p>
|
<p class="truncate"><strong><a href="{{ media.url }}" target="_blank"><i class="fas fa-link"></i> {{ media.url }}</a></strong></p>
|
||||||
<p class="truncate">Saving to: <strong>{{ media.source.directory_path }}</strong></p>
|
<p class="truncate">Downloading to: <strong>{{ media.source.directory_path }}</strong></p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
|
{% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
|
||||||
|
<div class="row">
|
||||||
|
<div class="col s12 m7">
|
||||||
|
<div><i class="fas fa-quote-left"></i></div>
|
||||||
|
<p>{% if media.description %}{{ media.description|truncatewords:200 }}{% else %}(Media has no description).{% endif %}</p>
|
||||||
|
<div class="right-align"><i class="fas fa-quote-right"></i></div>
|
||||||
|
</div>
|
||||||
|
<div class="col s12 m5">
|
||||||
|
<div class="card mediacard">
|
||||||
|
<div class="card-image">
|
||||||
|
<img src="{% if media.thumb %}{% url 'sync:media-thumb' pk=media.pk %}{% else %}{% static 'images/nothumb.png' %}{% endif %}">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<div class="row">
|
<div class="row">
|
||||||
<div class="col s12">
|
<div class="col s12">
|
||||||
<table class="striped">
|
<table class="striped">
|
||||||
|
@ -18,18 +33,10 @@
|
||||||
<td class="hide-on-small-only">Source</td>
|
<td class="hide-on-small-only">Source</td>
|
||||||
<td><span class="hide-on-med-and-up">Source<br></span><strong><a href="{% url 'sync:source' pk=media.source.pk %}">{{ media.source }}</a></strong></td>
|
<td><span class="hide-on-med-and-up">Source<br></span><strong><a href="{% url 'sync:source' pk=media.source.pk %}">{{ media.source }}</a></strong></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr title="The media title">
|
|
||||||
<td class="hide-on-small-only">Title</td>
|
|
||||||
<td><span class="hide-on-med-and-up">Title<br></span><strong>{{ media.title }}</strong></td>
|
|
||||||
</tr>
|
|
||||||
<tr title="The media duration">
|
<tr title="The media duration">
|
||||||
<td class="hide-on-small-only">Duration</td>
|
<td class="hide-on-small-only">Duration</td>
|
||||||
<td><span class="hide-on-med-and-up">Duration<br></span><strong>{{ media.duration_formatted }}</strong></td>
|
<td><span class="hide-on-med-and-up">Duration<br></span><strong>{{ media.duration_formatted }}</strong></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr title="The filename the media will be downloaded as">
|
|
||||||
<td class="hide-on-small-only">Filename</td>
|
|
||||||
<td><span class="hide-on-med-and-up">Filename<br></span><strong>{{ media.filename }}</strong></td>
|
|
||||||
</tr>
|
|
||||||
<tr title="The desired format">
|
<tr title="The desired format">
|
||||||
<td class="hide-on-small-only">Desired format</td>
|
<td class="hide-on-small-only">Desired format</td>
|
||||||
<td><span class="hide-on-med-and-up">Desired format<br></span><strong>{{ media.source.format_summary }}</strong></td>
|
<td><span class="hide-on-med-and-up">Desired format<br></span><strong>{{ media.source.format_summary }}</strong></td>
|
||||||
|
@ -38,19 +45,51 @@
|
||||||
<td class="hide-on-small-only">Fallback</td>
|
<td class="hide-on-small-only">Fallback</td>
|
||||||
<td><span class="hide-on-med-and-up">Fallback<br></span><strong>{{ media.source.get_fallback_display }}</strong></td>
|
<td><span class="hide-on-med-and-up">Fallback<br></span><strong>{{ media.source.get_fallback_display }}</strong></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr title="Has the media been downloaded">
|
<tr title="Has the media been downloaded?">
|
||||||
<td class="hide-on-small-only">Downloaded</td>
|
<td class="hide-on-small-only">Downloaded?</td>
|
||||||
<td><span class="hide-on-med-and-up">Downloaded<br></span><strong>{% if media.downloaded %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
<td><span class="hide-on-med-and-up">Downloaded?<br></span><strong>{% if media.downloaded %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr title="Can the media be downloaded">
|
{% if media.downloaded %}
|
||||||
<td class="hide-on-small-only">Can download</td>
|
<tr title="The filename the media will be downloaded as">
|
||||||
<td><span class="hide-on-med-and-up">Can download<br></span><strong>{% if youtube_dl_format %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
<td class="hide-on-small-only">Filename</td>
|
||||||
|
<td><span class="hide-on-med-and-up">Filename<br></span><strong>{{ media.filename }}</strong></td>
|
||||||
</tr>
|
</tr>
|
||||||
|
<tr title="Size of the file on disk">
|
||||||
|
<td class="hide-on-small-only">File size</td>
|
||||||
|
<td><span class="hide-on-med-and-up">File size<br></span><strong>{{ media.downloaded_filesize|filesizeformat }}</strong></td>
|
||||||
|
</tr>
|
||||||
|
<tr title="Codecs used in the downloaded file">
|
||||||
|
<td class="hide-on-small-only">Downloaded codecs</td>
|
||||||
|
<td><span class="hide-on-med-and-up">Downloaded codecs<br></span><strong>audio:{{ media.downloaded_audio_codec }}{% if media.downloaded_video_codec %}, video:{{ media.downloaded_video_codec }}{% endif %}</strong></td>
|
||||||
|
</tr>
|
||||||
|
<tr title="Container file format used in the download file">
|
||||||
|
<td class="hide-on-small-only">Container</td>
|
||||||
|
<td><span class="hide-on-med-and-up">Container<br></span><strong>{{ media.downloaded_container|upper }}</strong></td>
|
||||||
|
</tr>
|
||||||
|
<tr title="Frames per second in the downloaded file">
|
||||||
|
<td class="hide-on-small-only">Downloaded FPS</td>
|
||||||
|
<td><span class="hide-on-med-and-up">Downloaded FPS<br></span><strong>{{ media.downloaded_fps }} FPS</strong></td>
|
||||||
|
</tr>
|
||||||
|
<tr title="Does the downloaded file have high dynamic range?">
|
||||||
|
<td class="hide-on-small-only">Downloaded HDR?</td>
|
||||||
|
<td><span class="hide-on-med-and-up">Downloaded HDR?<br></span><strong>{% if media.downloaded_hdr %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
||||||
|
</tr>
|
||||||
|
{% else %}
|
||||||
|
<tr title="Can the media be downloaded?">
|
||||||
|
<td class="hide-on-small-only">Can download?</td>
|
||||||
|
<td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if youtube_dl_format %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
||||||
|
</tr>
|
||||||
|
{% endif %}
|
||||||
<tr title="The available media formats">
|
<tr title="The available media formats">
|
||||||
<td class="hide-on-small-only">Available formats</td>
|
<td class="hide-on-small-only">Available formats</td>
|
||||||
<td><span class="hide-on-med-and-up">Available formats<br></span>
|
<td><span class="hide-on-med-and-up">Available formats<br></span>
|
||||||
{% for format in media.formats %}
|
{% for format in media.formats %}
|
||||||
<span class="truncate">ID: <strong>{{ format.format_id }}</strong>{% if format.vcodec|lower != 'none' %}, <strong>{{ format.format_note }} ({{ format.width }}x{{ format.height }})</strong>, fps:<strong>{{ format.fps|lower }}</strong>, video:<strong>{{ format.vcodec }} @{{ format.tbr }}k</strong>{% endif %}{% if format.acodec|lower != 'none' %}, audio:<strong>{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz</strong>{% endif %}</span>
|
<span class="truncate">
|
||||||
|
ID: <strong>{{ format.format_id }}</strong>
|
||||||
|
{% if format.vcodec|lower != 'none' %}, <strong>{{ format.format_note }} ({{ format.width }}x{{ format.height }})</strong>, fps:<strong>{{ format.fps|lower }}</strong>, video:<strong>{{ format.vcodec }} @{{ format.tbr }}k</strong>{% endif %}
|
||||||
|
{% if format.acodec|lower != 'none' %}, audio:<strong>{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz</strong>{% endif %}
|
||||||
|
{% if format.format_id == combined_format or format.format_id == audio_format or format.format_id == video_format %}<strong>(matched)</strong>{% endif %}
|
||||||
|
</span>
|
||||||
{% empty %}
|
{% empty %}
|
||||||
Media has no indexed available formats
|
Media has no indexed available formats
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
@ -59,15 +98,11 @@
|
||||||
<tr title="Best available format for source requirements">
|
<tr title="Best available format for source requirements">
|
||||||
<td class="hide-on-small-only">Matched formats</td>
|
<td class="hide-on-small-only">Matched formats</td>
|
||||||
<td><span class="hide-on-med-and-up">Matched formats<br></span>
|
<td><span class="hide-on-med-and-up">Matched formats<br></span>
|
||||||
Combined: <strong>{% if combined_format %}{{ combined_format }} {% if combined_exact %}(exact match){% else %}(fallback){% endif %}{% else %}No match{% endif %}</strong><br>
|
Combined: <strong>{% if combined_format %}{{ combined_format }} {% if combined_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}</strong><br>
|
||||||
Audio: <strong>{% if audio_format %}{{ audio_format }} {% if audio_exact %}(exact match){% else %}(fallback){% endif %}{% else %}No match{% endif %}</strong><br>
|
Audio: <strong>{% if audio_format %}{{ audio_format }} {% if audio_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}</strong><br>
|
||||||
Video: <strong>{% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}No match{% endif %}
|
Video: <strong>{% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}
|
||||||
</strong></td>
|
</strong></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr title="Format string passed to youtube-dl">
|
|
||||||
<td class="hide-on-small-only">youtube-dl format</td>
|
|
||||||
<td><span class="hide-on-med-and-up">youtube-dl format<br></span><strong>{% if youtube_dl_format %}{{ youtube_dl_format }}{% else %}No matching formats{% endif %}</strong></td>
|
|
||||||
</tr>
|
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -18,7 +18,7 @@
|
||||||
<img src="{% if m.thumb %}{% url 'sync:media-thumb' pk=m.pk %}{% else %}{% static 'images/nothumb.png' %}{% endif %}">
|
<img src="{% if m.thumb %}{% url 'sync:media-thumb' pk=m.pk %}{% else %}{% static 'images/nothumb.png' %}{% endif %}">
|
||||||
<span class="card-title truncate">{{ m.source }}<br>
|
<span class="card-title truncate">{{ m.source }}<br>
|
||||||
<span>{{ m.name }}</span><br>
|
<span>{{ m.name }}</span><br>
|
||||||
<span>{% if m.can_download %}{{ m.published|date:'Y-m-d' }}{% else %}<i class="fas fa-exclamation-triangle"></i> No matching formats{% endif %}</span>
|
<span>{% if m.can_download %}{% if m.downloaded %}<i class="fas fa-check-circle" title="Downloaded"></i>{% else %}<i class="far fa-clock" title="Queued waiting to download"></i>{% endif %} {{ m.published|date:'Y-m-d' }}{% else %}<i class="fas fa-exclamation-triangle"></i> No matching formats{% endif %}</span>
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
</a>
|
</a>
|
||||||
|
|
|
@ -164,6 +164,7 @@ def parse_media_format(format_dict):
|
||||||
'format_verbose': format_dict.get('format', ''),
|
'format_verbose': format_dict.get('format', ''),
|
||||||
'height': format_dict.get('height', 0),
|
'height': format_dict.get('height', 0),
|
||||||
'vcodec': vcodec,
|
'vcodec': vcodec,
|
||||||
|
'fps': format_dict.get('fps', 0),
|
||||||
'vbr': format_dict.get('tbr', 0),
|
'vbr': format_dict.get('tbr', 0),
|
||||||
'acodec': acodec,
|
'acodec': acodec,
|
||||||
'abr': format_dict.get('abr', 0),
|
'abr': format_dict.get('abr', 0),
|
||||||
|
|
|
@ -40,3 +40,23 @@ def get_media_info(url):
|
||||||
except youtube_dl.utils.DownloadError as e:
|
except youtube_dl.utils.DownloadError as e:
|
||||||
raise YouTubeError(f'Failed to extract_info for "{url}": {e}') from e
|
raise YouTubeError(f'Failed to extract_info for "{url}": {e}') from e
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def download_media(url, media_format, extension, output_file):
|
||||||
|
'''
|
||||||
|
Downloads a YouTube URL to a file on disk.
|
||||||
|
'''
|
||||||
|
opts = copy(_defaults)
|
||||||
|
opts.update({
|
||||||
|
'format': media_format,
|
||||||
|
'merge_output_format': extension,
|
||||||
|
'outtmpl': output_file,
|
||||||
|
'quiet': True,
|
||||||
|
})
|
||||||
|
print(opts)
|
||||||
|
with youtube_dl.YoutubeDL(opts) as y:
|
||||||
|
try:
|
||||||
|
return y.download([url])
|
||||||
|
except youtube_dl.utils.DownloadError as e:
|
||||||
|
raise YouTubeError(f'Failed to download for "{url}": {e}') from e
|
||||||
|
return False
|
||||||
|
|
Loading…
Reference in New Issue