PublicSnapshot
/
tubesync
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

better media task management options

This commit is contained in:
meeb 2020-12-10 15:16:01 +11:00
parent 49bf7dcd5b
commit 7a1899c363
11 changed files with 209 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
tubesync/sync/forms.py
View File

@ -22,3 +22,8 @@ class ConfirmDeleteSourceForm(forms.Form):
label=_('Also delete downloaded media'),
required=False
)
class RedownloadMediaForm(forms.Form):
pass

61
tubesync/sync/models.py
View File

@ -396,6 +396,20 @@ class Media(models.Model):
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats',
}
}
STATE_UNKNOWN = 'unknown'
STATE_SCHEDULED = 'scheduled'
STATE_DOWNLOADING = 'downloading'
STATE_DOWNLOADED = 'downloaded'
STATE_ERROR = 'error'
STATES = (STATE_UNKNOWN, STATE_SCHEDULED, STATE_DOWNLOADING, STATE_DOWNLOADED,
STATE_ERROR)
STATE_ICONS = {
STATE_UNKNOWN: '<i class="far fa-question-circle" title="Unknown download state"></i>',
STATE_SCHEDULED: '<i class="far fa-clock" title="Scheduled to download"></i>',
STATE_DOWNLOADING: '<i class="fas fa-download" title="Downloading now"></i>',
STATE_DOWNLOADED: '<i class="far fa-check-circle" title="Downloaded"></i>',
STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>',
}
uuid = models.UUIDField(
_('uuid'),
@ -649,14 +663,17 @@ class Media(models.Model):
name = slugify(self.name.replace('&', 'and').replace('+', 'and'))[:50]
key = self.key.strip()
fmt = self.source.source_resolution.lower()
codecs = []
vcodec = self.source.source_vcodec.lower()
acodec = self.source.source_acodec.lower()
if vcodec:
codecs.append(vcodec)
if acodec:
codecs.append(acodec)
codecs = '-'.join(codecs)
if self.source.is_audio():
codecs = self.source.source_acodec.lower()
else:
codecs = []
vcodec = self.source.source_vcodec.lower()
acodec = self.source.source_acodec.lower()
if vcodec:
codecs.append(vcodec)
if acodec:
codecs.append(acodec)
codecs = '-'.join(codecs)
ext = self.source.extension
return f'{datestr}_{source_name}_{name}_{key}-{fmt}-{codecs}.{ext}'
@ -664,6 +681,34 @@ class Media(models.Model):
def filepath(self):
return self.source.directory_path / self.filename
@property
def thumb_file_exists(self):
if not self.thumb:
return False
return os.path.exists(self.thumb.path)
@property
def media_file_exists(self):
if not self.media_file:
return False
return os.path.exists(self.media_file.path)
def get_download_state(self, task=None):
if self.downloaded:
return self.STATE_DOWNLOADED
if task:
if task.locked_by_pid_running():
return self.STATE_DOWNLOADING
elif task.has_error():
return self.STATE_ERROR
else:
return self.STATE_SCHEDULED
return self.STATE_UNKNOWN
def get_download_state_icon(self, task=None):
state = self.get_download_state(task)
return self.STATE_ICONS.get(state, self.STATE_ICONS[self.STATE_UNKNOWN])
def download_media(self):
format_str = self.get_format_str()
if not format_str:

29
tubesync/sync/signals.py
View File

@ -29,6 +29,7 @@ def source_pre_save(sender, instance, **kwargs):
str(instance.pk),
repeat=instance.index_schedule,
queue=str(instance.pk),
priority=5,
verbose_name=verbose_name.format(instance.name)
)
@ -46,6 +47,7 @@ def source_post_save(sender, instance, created, **kwargs):
str(instance.pk),
repeat=instance.index_schedule,
queue=str(instance.pk),
priority=5,
verbose_name=verbose_name.format(instance.name)
)
# Trigger the post_save signal for each media item linked to this source as various
@ -82,9 +84,18 @@ def task_task_failed(sender, task_id, completed_task, **kwargs):
@receiver(post_save, sender=Media)
def media_post_save(sender, instance, created, **kwargs):
# Triggered after media is saved
if created:
# If the media is newly created start a task to download its thumbnail
# Triggered after media is saved, Recalculate the "can_download" flag, this may
# need to change if the source specifications have been changed
if instance.get_format_str():
if not instance.can_download:
instance.can_download = True
instance.save()
else:
if instance.can_download:
instance.can_download = True
instance.save()
# If the media is missing a thumbnail schedule it to be downloaded
if not instance.thumb:
thumbnail_url = instance.thumbnail
if thumbnail_url:
log.info(f'Scheduling task to download thumbnail for: {instance.name} '
@ -94,18 +105,9 @@ def media_post_save(sender, instance, created, **kwargs):
str(instance.pk),
thumbnail_url,
queue=str(instance.source.pk),
priority=10,
verbose_name=verbose_name.format(instance.name)
)
# Recalculate the "can_download" flag, this may need to change if the source
# specifications have been changed
if instance.get_format_str():
if not instance.can_download:
instance.can_download = True
instance.save()
else:
if instance.can_download:
instance.can_download = True
instance.save()
# If the media has not yet been downloaded schedule it to be downloaded
if not instance.downloaded:
delete_task_by_media('sync.tasks.download_media', (str(instance.pk),))
@ -113,6 +115,7 @@ def media_post_save(sender, instance, created, **kwargs):
download_media(
str(instance.pk),
queue=str(instance.source.pk),
priority=15,
verbose_name=verbose_name.format(instance.name)
)

8
tubesync/sync/tasks.py
View File

@ -103,6 +103,14 @@ def get_source_completed_tasks(source_id, only_errors=False):
return CompletedTask.objects.filter(**q).order_by('-failed_at')
def get_media_download_task(media_id):
try:
return Task.objects.get_task('sync.tasks.download_media',
args=(str(media_id),))[0]
except IndexError:
return False
def delete_task_by_source(task_name, source_id):
return Task.objects.filter(task_name=task_name, queue=str(source_id)).delete()

23
tubesync/sync/templates/sync/media-item.html
View File

@ -5,13 +5,14 @@
{% block content %}
<div class="row">
<div class="col s12">
<h1 class="truncate">Media <strong>{{ media.key }}</strong></h1>
<h1 class="truncate">Media <strong>{{ media.key }}</strong> {{ download_state_icon|safe }}</h1>
{% if media.title %}<h2 class="truncate"><strong>{{ media.title }}</strong></h2>{% endif %}
<p class="truncate"><strong><a href="{{ media.url }}" target="_blank"><i class="fas fa-link"></i> {{ media.url }}</a></strong></p>
<p class="truncate">Downloading to: <strong>{{ media.source.directory_path }}</strong></p>
</div>
</div>
{% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
{% include 'infobox.html' with message=message %}
<div class="row">
<div class="col s12 m7">
<div><i class="fas fa-quote-left"></i></div>
@ -26,6 +27,19 @@
</div>
</div>
</div>
{% if task %}
<div class="row">
<div class="col s12">
<div class="collection">
<span class="collection-item">
<i class="fas fa-stopwatch"></i> <strong>{{ task }}</strong><br>
{% if task.instance.index_schedule %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
<i class="fas fa-redo"></i> Task will run {% if task.run_now %}<strong>immediately</strong>{% else %}at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>{% endif %}
</span>
</div>
</div>
</div>
{% endif %}
<div class="row">
<div class="col s12">
<table class="striped">
@ -106,4 +120,11 @@
</table>
</div>
</div>
{% if media.downloaded %}
<div class="row">
<div class="col s12">
<a href="{% url 'sync:redownload-media' pk=media.pk %}" class="btn">Delete and redownload media <i class="fas fa-cloud-download-alt"></i></a>
</div>
</div>
{% endif %}
{% endblock %}

29
tubesync/sync/templates/sync/media-redownload.html Normal file
View File

@ -0,0 +1,29 @@
{% extends 'base.html' %}
{% block headtitle %}Redownload media - {{ media }}{% endblock %}
{% block content %}
<div class="row no-margin-bottom">
<div class="col s12">
<h1>Redownload media <strong>{{ media }}</strong></h1>
<p>
You can delete the downloaded file for your media <strong>{{ media }}</strong> and
schedule it to be redownloaded. You might want to use this if you moved the original
file on disk and want to download it again, or, if you changed your source settings
such as changed the desired resolution and want to redownload the media in a different
format.
</p>
</div>
</div>
<div class="row">
<form method="post" action="{% url 'sync:redownload-media' pk=media.pk %}" class="col s12 simpleform">
{% csrf_token %}
{% include 'simpleform.html' with form=form %}
<div class="row no-margin-bottom padding-top">
<div class="col s12">
<button class="btn" type="submit" name="action">Really delete and redownload media <i class="fas fa-trash-alt"></i></button>
</div>
</div>
</form>
</div>
{% endblock %}

2
tubesync/sync/templates/sync/media.html
View File

@ -18,7 +18,7 @@
<img src="{% if m.thumb %}{% url 'sync:media-thumb' pk=m.pk %}{% else %}{% static 'images/nothumb.png' %}{% endif %}">
<span class="card-title truncate">{{ m.source }}<br>
<span>{{ m.name }}</span><br>
<span>{% if m.can_download %}{% if m.downloaded %}<i class="fas fa-check-circle" title="Downloaded"></i>{% else %}<i class="far fa-clock" title="Queued waiting to download"></i>{% endif %} {{ m.published|date:'Y-m-d' }}{% else %}<i class="fas fa-exclamation-triangle"></i> No matching formats{% endif %}</span>
<span>{% if m.can_download %}{% if m.downloaded %}<i class="fas fa-check-circle" title="Downloaded"></i>{% else %}<i class="far fa-clock" title="Waiting to download or downloading"></i>{% endif %} {{ m.published|date:'Y-m-d' }}{% else %}<i class="fas fa-exclamation-triangle"></i> No matching formats{% endif %}</span>
</span>
</div>
</a>

2
tubesync/sync/templates/sync/tasks.html
View File

@ -66,7 +66,7 @@
<a href="{% url task.url pk=task.instance.pk %}" class="collection-item">
<i class="fas fa-stopwatch"></i> <strong>{{ task }}</strong><br>
{% if task.instance.index_schedule %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
<i class="fas fa-redo"></i> Task will next run at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>
<i class="fas fa-redo"></i> Task will run {% if task.run_now %}<strong>immediately</strong>{% else %}at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>{% endif %}
</a>
{% empty %}
<span class="collection-item no-items"><i class="fas fa-info-circle"></i> There are no scheduled tasks.</span>

7
tubesync/sync/urls.py
View File

@ -1,7 +1,8 @@
from django.urls import path
from .views import (DashboardView, SourcesView, ValidateSourceView, AddSourceView,
SourceView, UpdateSourceView, DeleteSourceView, MediaView,
MediaThumbView, MediaItemView, TasksView, CompletedTasksView)
MediaThumbView, MediaItemView, MediaRedownloadView, TasksView,
CompletedTasksView)
app_name = 'sync'
@ -55,6 +56,10 @@ urlpatterns = [
MediaItemView.as_view(),
name='media-item'),
path('media-redownload/<uuid:pk>',
MediaRedownloadView.as_view(),
name='redownload-media'),
# Task URLs
path('tasks',

68
tubesync/sync/views.py
View File

@ -4,6 +4,7 @@ from django.http import Http404
from django.views.generic import TemplateView, ListView, DetailView
from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateView,
DeleteView)
from django.views.generic.detail import SingleObjectMixin
from django.http import HttpResponse
from django.urls import reverse_lazy
from django.db.models import Count
@ -14,9 +15,11 @@ from django.utils.translation import gettext_lazy as _
from common.utils import append_uri_params
from background_task.models import Task, CompletedTask
from .models import Source, Media
from .forms import ValidateSourceForm, ConfirmDeleteSourceForm
from .forms import ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm
from .utils import validate_url, delete_file
from .tasks import map_task_to_instance, get_error_message, get_source_completed_tasks
from .tasks import (map_task_to_instance, get_error_message,
get_source_completed_tasks, get_media_download_task,
delete_task_by_media)
from . import signals
from . import youtube
@ -295,6 +298,7 @@ class DeleteSourceView(DeleteView, FormMixin):
delete_media_val = request.POST.get('delete_media', False)
delete_media = True if delete_media_val is not False else False
if delete_media:
source = self.get_object()
for media in Media.objects.filter(source=source):
if media.media_file:
delete_file(media.media_file.name)
@ -378,12 +382,29 @@ class MediaItemView(DetailView):
template_name = 'sync/media-item.html'
model = Media
messages = {
'redownloading': _('Media file has been deleted and scheduled to redownload'),
}
def __init__(self, *args, **kwargs):
self.message = None
super().__init__(*args, **kwargs)
def dispatch(self, request, *args, **kwargs):
message_key = request.GET.get('message', '')
self.message = self.messages.get(message_key, '')
return super().dispatch(request, *args, **kwargs)
def get_context_data(self, *args, **kwargs):
data = super().get_context_data(*args, **kwargs)
data['message'] = self.message
combined_exact, combined_format = self.object.get_best_combined_format()
audio_exact, audio_format = self.object.get_best_audio_format()
video_exact, video_format = self.object.get_best_video_format()
task = get_media_download_task(self.object.pk)
data['task'] = task
data['download_state'] = self.object.get_download_state(task)
data['download_state_icon'] = self.object.get_download_state_icon(task)
data['combined_exact'] = combined_exact
data['combined_format'] = combined_format
data['audio_exact'] = audio_exact
@ -394,6 +415,48 @@ class MediaItemView(DetailView):
return data
class MediaRedownloadView(FormView, SingleObjectMixin):
template_name = 'sync/media-redownload.html'
form_class = RedownloadMediaForm
model = Media
def __init__(self, *args, **kwargs):
self.object = None
super().__init__(*args, **kwargs)
def dispatch(self, request, *args, **kwargs):
self.object = self.get_object()
return super().dispatch(request, *args, **kwargs)
def form_valid(self, form):
# Delete any active download tasks for the media
delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),))
# If the thumbnail file exists on disk, delete it
if self.object.thumb_file_exists:
delete_file(self.object.thumb.path)
self.object.thumb = None
# If the media file exists on disk, delete it
if self.object.media_file_exists:
delete_file(self.object.media_file.path)
self.object.media_file = None
# Reset all download data
self.object.downloaded = False
self.object.downloaded_audio_codec = None
self.object.downloaded_video_codec = None
self.object.downloaded_container = None
self.object.downloaded_fps = None
self.object.downloaded_hdr = False
self.object.downloaded_filesize = None
# Saving here will trigger the post_create signals to schedule new tasks
self.object.save()
return super().form_valid(form)
def get_success_url(self):
url = reverse_lazy('sync:media-item', kwargs={'pk': self.object.pk})
return append_uri_params(url, {'message': 'redownloading'})
class TasksView(ListView):
'''
A list of tasks queued to be completed. This is, for example, scraping for new
@ -420,6 +483,7 @@ class TasksView(ListView):
continue
setattr(task, 'instance', obj)
setattr(task, 'url', url)
setattr(task, 'run_now', task.run_at < now)
if task.locked_by_pid_running():
data['running'].append(task)
elif task.has_error():

4
tubesync/tubesync/settings.py
View File

@ -117,8 +117,8 @@ MAX_ATTEMPTS = 10 # Number of times tasks will be retr
MAX_RUN_TIME = 1800 # Maximum amount of time in seconds a task can run
BACKGROUND_TASK_RUN_ASYNC = True # Run tasks async in the background
BACKGROUND_TASK_ASYNC_THREADS = 2 # Number of async tasks to run at once
BACKGROUND_TASK_PRIORITY_ORDERING = 'DESC' # Process high priority tasks first
COMPLETED_TASKS_DAYS_TO_KEEP = 30 # Number of days to keep completed tasks
BACKGROUND_TASK_PRIORITY_ORDERING = 'ASC' # Use 'niceness' task priority ordering
COMPLETED_TASKS_DAYS_TO_KEEP = 30 # Number of days to keep completed tasks
SOURCES_PER_PAGE = 36