start of adding sources interface

This commit is contained in:
meeb 2020-11-26 14:03:55 +11:00
parent bf27c43bbb
commit 37d390c8d8
23 changed files with 11773 additions and 35 deletions

View File

@ -3,9 +3,6 @@ FROM debian:buster-slim
ARG DEBIAN_FRONTEND="noninteractive"
# Third party software versions
ARG YOUTUBE_DL_VERSION="2020.11.24"
ENV YOUTUBE_DL_EXPECTED_SHA256="7d70f2e2d6b42d7c948a418744cd5c89832d67f4fb36f01f1cf4ea7dc8fe537a"
ENV YOUTUBE_DL_TARBALL="https://github.com/ytdl-org/youtube-dl/releases/download/${YOUTUBE_DL_VERSION}/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz"
ARG FFMPEG_VERSION="4.3.1"
ENV FFMPEG_EXPECTED_MD5="ee235393ec7778279144ee6cbdd9eb64"
ENV FFMPEG_TARBALL="https://johnvansickle.com/ffmpeg/releases/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz"
@ -14,12 +11,7 @@ ENV FFMPEG_TARBALL="https://johnvansickle.com/ffmpeg/releases/ffmpeg-${FFMPEG_VE
RUN set -x && \
# Install required distro packages
apt-get update && \
apt-get -y --no-install-recommends install curl xz-utils ca-certificates binutils python3 python3-setuptools && \
# Install youtube-dl
curl -L ${YOUTUBE_DL_TARBALL} --output /tmp/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz && \
echo "${YOUTUBE_DL_EXPECTED_SHA256} /tmp/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz" | sha256sum -c - && \
tar -zxvf /tmp/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz -C /tmp && \
(cd /tmp/youtube-dl; python3 /tmp/youtube-dl/setup.py install) && \
apt-get -y --no-install-recommends install curl xz-utils ca-certificates binutils && \
# Install ffmpeg
curl -L ${FFMPEG_TARBALL} --output /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz && \
echo "${FFMPEG_EXPECTED_MD5} tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz" | md5sum -c - && \
@ -28,8 +20,6 @@ RUN set -x && \
ls -lat /tmp/ffmpeg-4.3.1-amd64-static && \
install -v -s -g root -o root -m 0755 -s /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static/ffmpeg -t /usr/local/bin && \
# Clean up
rm /tmp/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz && \
rm -rf /tmp/youtube-dl && \
rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar && \
rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static && \
apt-get -y autoremove --purge curl xz-utils binutils
@ -54,14 +44,14 @@ ENV UID="${default_uid}"
ENV GID="${default_gid}"
RUN set -x && \
# Install required distro packages
apt-get -y --no-install-recommends install python3-pip python3-dev gcc make && \
apt-get -y --no-install-recommends install python3 python3-setuptools python3-pip python3-dev gcc make && \
# Install wheel which is required for pipenv
pip3 --disable-pip-version-check install wheel && \
# Then install pipenv
pip3 --disable-pip-version-check install pipenv && \
# Create a 'www' user which the workers drop to
groupadd -g ${GID} www && \
useradd -M -d /dev/null -s /bin/false -u ${UID} -g www www && \
useradd -M -d /app -s /bin/false -u ${UID} -g www www && \
# Install non-distro packages
pipenv install --system && \
# Make absolutely sure we didn't accidentally bundle a SQLite dev database

View File

@ -17,6 +17,7 @@ uvicorn = "*"
uvloop = "*"
httptools = "*"
django-simple-task = "*"
youtube-dl = "*"
[requires]
python_version = "3"

10
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "10bcf36ee023c01949edbe0dbe22eefbe603580488e3498cc5f211f3d1b221ed"
"sha256": "b2f3530bcd9d615f37ba75913336690a3b61e5cb9bf2659212431cbe11dbef90"
},
"pipfile-spec": 6,
"requires": {
@ -238,6 +238,14 @@
],
"index": "pypi",
"version": "==5.2.0"
},
"youtube-dl": {
"hashes": [
"sha256:f61c8e4855559c33df66234b7e7ba303f4bcbef59639fb15825504d6484fd25f",
"sha256:f701befffe00ae4b0d56f88ed45e1295c151c340d0011efdb1005012abc81996"
],
"index": "pypi",
"version": "==2020.11.24"
}
},
"develop": {}

View File

@ -1,7 +1,9 @@
from django.conf import settings
from youtube_dl import version as yt_version
def app_details(request):
return {
'app_version': str(settings.VERSION)
'app_version': str(settings.VERSION),
'youtube_dl_version': str(yt_version.__version__)
}

View File

@ -16,6 +16,21 @@ $nav-background-colour: $colour-near-black;
$nav-text-colour: $colour-near-white;
$nav-link-background-hover-colour: $colour-orange;
$main-button-background-colour: $colour-light-blue;
$main-button-background-hover-colour: $colour-orange;
$main-button-text-colour: $colour-white;
$footer-background-colour: $colour-red;
$footer-text-colour: $colour-white;
$footer-link-colour: $colour-near-black;
$footer-link-hover-colour: $colour-orange;
$form-label-text-colour: $colour-near-black;
$form-input-border-colour: $colour-light-blue;
$form-input-border-active-colour: $colour-orange;
$form-select-border-colour: $colour-light-blue;
$form-error-background-colour: $colour-red;
$form-error-text-colour: $colour-near-white;
$box-error-background-colour: $colour-red;
$box-error-text-colour: $colour-white;

View File

@ -0,0 +1,37 @@
.simpleform {
.row {
margin-bottom: 0;
}
label {
text-transform: uppercase;
display: block;
font-size: 0.9rem;
position: relative;
transition: none;
top: initial;
left: initial !important;
transform: none;
color: $form-label-text-colour;
}
input {
width: 100%;
padding: 5px 8px 5px 8px;
font-size: 1.1rem;
border: 2px $form-input-border-colour solid;
border-radius: 2px;
outline: none;
&:focus {
outline: none;
border: 2px $form-input-border-active-colour solid;
}
}
textarea {
min-height: 150px;
}
}
select {
display: initial !important;
border: 2px $form-select-border-colour solid;
height: initial !important;
}

View File

@ -0,0 +1,24 @@
strong {
font-weight: bold;
}
.nowrap {
white-space: nowrap;
}
.no-margin-bottom {
margin-bottom: 0 !important;
}
.errors {
background-color: $box-error-background-colour;
border-radius: 2px;
padding: 10px 0 5px 0;
}
.errorlist {
li {
color: $box-error-text-colour;
padding: 0 10px 5px 10px;
}
}

View File

@ -62,6 +62,24 @@ main {
padding: 2rem 0 2rem 0;
h1 {
margin: 0;
padding: 0;
font-size: 2rem;
}
.btn {
width: 100%;
background-color: $main-button-background-colour !important;
color: $main-button-text-colour !important;
i {
font-size: 0.9rem;
}
&:hover {
background-color: $main-button-background-hover-colour !important;
}
}
}
footer {
@ -89,7 +107,8 @@ footer {
color: $footer-link-colour;
text-decoration: none;
&:hover {
text-decoration: underline;
color: $footer-link-hover-colour;
text-decoration: none;
}
}

View File

@ -8,8 +8,8 @@
@import "fonts";
@import "variables";
@import "helpers";
@import "colours";
@import "helpers";
@import "forms";
@import "template";

View File

@ -18,7 +18,7 @@
<header>
<div class="container">
<a href="{% url 'sync:index' %}">
<a href="{% url 'sync:dashboard' %}">
{% include 'tubesync.svg' with width='3rem' height='3rem' %}
<h1>TubeSync</h1>
</a>
@ -28,10 +28,11 @@
<nav>
<div class="container">
<ul>
<li><a href=""><i class="fas fa-fw fa-th-large"></i><span class="hide-on-med-and-down"> Dashboard</span></a></li>
<li><a href=""><i class="fas fa-fw fa-play"></i><span class="hide-on-med-and-down"> Sources</span></a></li>
<li><a href=""><i class="fas fa-fw fa-film"></i><span class="hide-on-med-and-down"> Media</span></a></li>
<li><a href=""><i class="fas fa-fw fa-clock"></i><span class="hide-on-med-and-down"> Tasks</span></a></li>
<li><a href="{% url 'sync:dashboard' %}"><i class="fas fa-fw fa-th-large"></i><span class="hide-on-med-and-down"> Dashboard</span></a></li>
<li><a href="{% url 'sync:sources' %}"><i class="fas fa-fw fa-play"></i><span class="hide-on-med-and-down"> Sources</span></a></li>
<li><a href="{% url 'sync:media' %}"><i class="fas fa-fw fa-film"></i><span class="hide-on-med-and-down"> Media</span></a></li>
<li><a href="{% url 'sync:tasks' %}"><i class="fas fa-fw fa-clock"></i><span class="hide-on-med-and-down"> Tasks</span></a></li>
<li><a href="{% url 'sync:logs' %}"><i class="fas fa-fw fa-list"></i><span class="hide-on-med-and-down"> Logs</span></a></li>
</ul>
</div>
</nav>
@ -45,13 +46,12 @@
<footer>
<div class="container">
<p>
<a href="{% url 'sync:index' %}">{% include 'tubesync.svg' with width='0.8rem' height='0.8rem' %} TubeSync</a>
is an open source synchronisation tool to automatically download videos from online video platforms.
<br>
The original code under a GPLv3 licence is available at
<a href="https://github.com/meeb/tubesync"><i class="fab fa-github"></i> https://github.com/meeb/tubesync</a>.
<a href="{% url 'sync:dashboard' %}" class="nowrap">{% include 'tubesync.svg' with width='0.8rem' height='0.8rem' %} TubeSync</a>
is an open source synchronisation tool to automatically download videos from online video platforms. The
original code under a GPLv3 licence is available at
<a href="https://github.com/meeb/tubesync" class="nowrap"><i class="fab fa-github"></i> https://github.com/meeb/tubesync</a>.
</p>
<p>Version {{ app_version }}.</p>
<p>TubeSync version {{ app_version }} with embedded <a href="https://yt-dl.org/"><i class="fas fa-link"></i> youtube-dl</a> version {{ youtube_dl_version }}.</p>
</div>
</footer>

View File

@ -0,0 +1,19 @@
{% if form %}
{% if form.errors %}
<ul class="errors">
{% for _, error in form.errors.items %}
<li>{{ error }}</li>
{% endfor %}
</ul>
{% endif %}
{% for field in form %}
{% if field.field.widget.input_type == 'hidden' %}{{ field }}{% else %}
<div class="row">
<div class="input-field col s12">
{{ field.label_tag }}
{{ field }}
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}

11237
app/static/styles/tubesync.css Normal file

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

15
app/sync/forms.py Normal file
View File

@ -0,0 +1,15 @@
from django import forms
class ValidateSourceForm(forms.Form):
source_type = forms.CharField(
max_length=1,
required=True,
widget=forms.HiddenInput()
)
source_url = forms.URLField(
label='Source URL',
required=True
)

View File

@ -1,6 +1,6 @@
{% extends 'base.html' %}
{% block headtitle %}Synchronize YouTube to your local media server{% endblock %}
{% block headtitle %}Dashboard{% endblock %}
{% block content %}
<div class="row">

View File

@ -0,0 +1,11 @@
{% extends 'base.html' %}
{% block headtitle %}Logs{% endblock %}
{% block content %}
<div class="row">
<div class="col s12">
logs
</div>
</div>
{% endblock %}

View File

@ -0,0 +1,11 @@
{% extends 'base.html' %}
{% block headtitle %}Media{% endblock %}
{% block content %}
<div class="row">
<div class="col s12">
media
</div>
</div>
{% endblock %}

View File

@ -0,0 +1,24 @@
{% extends 'base.html' %}
{% block headtitle %}Source - Add{% endblock %}
{% block content %}
<div class="row no-margin-bottom">
<div class="col s12">
<h1>Add a {{ help_item }}</h1>
<p>{{ help_text|safe }}</p>
<p>Example: <strong>{{ help_example }}</strong></p>
</div>
</div>
<div class="row">
<form method="post" action="{% url 'sync:validate-source' source_type=source_type %}" class="col s12 simpleform">
{% csrf_token %}
{% include 'simpleform.html' with form=form %}
<div class="row no-margin-bottom padding-top">
<div class="col s12">
<button class="btn" type="submit" name="action">Add {{ help_item }} <i class="fas fa-fw fa-plus"></i></button>
</div>
</div>
</form>
</div>
{% endblock %}

View File

@ -0,0 +1,14 @@
{% extends 'base.html' %}
{% block headtitle %}Sources{% endblock %}
{% block content %}
<div class="row">
<div class="col s12 l6">
<a href="{% url 'sync:validate-source' source_type='youtube-channel' %}" class="btn"><i class="fas fa-plus"></i> Add a YouTube channel</a>
</div>
<div class="col s12 l6">
<a href="{% url 'sync:validate-source' source_type='youtube-playlist' %}" class="btn"><i class="fas fa-plus"></i> Add a YouTube playlist</a>
</div>
</div>
{% endblock %}

View File

@ -0,0 +1,11 @@
{% extends 'base.html' %}
{% block headtitle %}Tasks{% endblock %}
{% block content %}
<div class="row">
<div class="col s12">
tasks
</div>
</div>
{% endblock %}

View File

@ -1,5 +1,6 @@
from django.urls import path
from .views import IndexView
from .views import (DashboardView, SourcesView, ValidateSourceView, MediaView,
TasksView, LogsView)
app_name = 'sync'
@ -8,7 +9,27 @@ app_name = 'sync'
urlpatterns = [
path('',
IndexView.as_view(),
name='index'),
DashboardView.as_view(),
name='dashboard'),
path('sources',
SourcesView.as_view(),
name='sources'),
path('source/validate/<slug:source_type>',
ValidateSourceView.as_view(),
name='validate-source'),
path('media',
MediaView.as_view(),
name='media'),
path('tasks',
TasksView.as_view(),
name='tasks'),
path('logs',
LogsView.as_view(),
name='logs'),
]

29
app/sync/utils.py Normal file
View File

@ -0,0 +1,29 @@
import re
from urllib.parse import urlsplit, parse_qs
from django.forms import ValidationError
def validate_url(url, validator):
'''
Validate a URL against a dict of validation requirements.
'''
valid_scheme, valid_netloc, valid_path, valid_query = (validator['scheme'],
validator['domain'], validator['path_regex'], validator['qs_args'])
url_parts = urlsplit(str(url).strip())
url_scheme = str(url_parts.scheme).strip().lower()
if url_scheme != valid_scheme:
raise ValidationError(f'scheme "{url_scheme}" must be "{valid_scheme}"')
url_netloc = str(url_parts.netloc).strip().lower()
if url_netloc != valid_netloc:
raise ValidationError(f'domain "{url_netloc}" must be "{valid_netloc}"')
url_path = str(url_parts.path).strip()
matches = re.match(valid_path, url_path)
if matches is None:
raise ValidationError(f'path "{url_path}" must match "{valid_path}"')
url_query = str(url_parts.query).strip()
url_query_parts = parse_qs(url_query)
for required_query in valid_query:
if required_query not in url_query_parts:
raise ValidationError(f'query string "{url_query}" must '
f'contain "{required_query}"')
return True

View File

@ -1,9 +1,188 @@
from django.http import Http404
from django.views.generic import TemplateView
from django.views.generic.edit import FormView
from django.urls import reverse_lazy
from django.forms import ValidationError
from django.utils.translation import gettext_lazy as _
from .models import Source
from .forms import ValidateSourceForm
from .utils import validate_url
class IndexView(TemplateView):
class DashboardView(TemplateView):
'''
The dashboard shows non-interactive totals and summaries, nothing more.
'''
template_name = 'sync/index.html'
template_name = 'sync/dashboard.html'
def dispatch(self, request, *args, **kwargs):
return super().dispatch(request, *args, **kwargs)
class SourcesView(TemplateView):
'''
A bare list of the sources which have been created with their states.
'''
template_name = 'sync/sources.html'
def dispatch(self, request, *args, **kwargs):
return super().dispatch(request, *args, **kwargs)
class ValidateSourceView(FormView):
'''
Validate a URL and prepopulate a create source view form with confirmed
accurate data. The aim here is to streamline onboarding of new sources
which otherwise may not be entirely obvious to add, such as the "key"
being just a playlist ID or some other reasonably unobvious internals.
'''
template_name = 'sync/source-validate.html'
form_class = ValidateSourceForm
errors = {
'invalid_url': _('Invalid URL, the URL must for a "{item}" must be in '
'the format of "{example}". The error was: {error}.'),
}
source_types = {
'youtube-channel': Source.SOURCE_TYPE_YOUTUBE_CHANNEL,
'youtube-playlist': Source.SOURCE_TYPE_YOUTUBE_PLAYLIST,
}
help_item = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _('YouTube channel'),
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: _('YouTube playlist'),
}
help_texts = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _(
'Enter a YouTube channel URL into the box below. A channel URL will be in '
'the format of <strong>https://www.youtube.com/CHANNELNAME</strong> '
'where <strong>CHANNELNAME</strong> is the name of the channel you want '
'to add.'
),
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: _(
'Enter a YouTube playlist URL into the box below. A playlist URL will be '
'in the format of <strong>https://www.youtube.com/watch?v=AAAAAA&list='
'BiGLoNgUnIqUeId</strong> where <strong>BiGLoNgUnIqUeId</strong> is the '
'unique ID of the playlist you want to add.'
),
}
help_examples = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/google',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('https://www.youtube.com/watch?v=DcKEPl'
'-MpLA&list=PL590L5WQmH8dpP0RyH5pCfIaDE'
'dt9nk7r')
}
validation_urls = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: {
'scheme': 'https',
'domain': 'www.youtube.com',
'path_regex': '^\/(c\/)?[^\/]+$',
'qs_args': [],
'example': 'https://www.youtube.com/SOMECHANNEL'
},
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: {
'scheme': 'https',
'domain': 'www.youtube.com',
'path_regex': '^\/watch$',
'qs_args': ['v', 'list'],
'example': 'https://www.youtube.com/watch?v=VIDEOID&list=PLAYLISTID'
},
}
def __init__(self, *args, **kwargs):
self.source_type_str = ''
self.source_type = None
super().__init__(*args, **kwargs)
def dispatch(self, request, *args, **kwargs):
self.source_type_str = kwargs.get('source_type', '').strip().lower()
self.source_type = self.source_types.get(self.source_type_str, None)
if not self.source_type:
raise Http404
return super().dispatch(request, *args, **kwargs)
def get_initial(self):
initial = super().get_initial()
initial['source_type'] = self.source_type
return initial
def get_context_data(self, *args, **kwargs):
data = super().get_context_data(*args, **kwargs)
data['source_type'] = self.source_type_str
data['help_item'] = self.help_item.get(self.source_type)
data['help_text'] = self.help_texts.get(self.source_type)
data['help_example'] = self.help_examples.get(self.source_type)
return data
def form_valid(self, form):
# Perform extra validation on the URL, we need to extract the channel name or
# playlist ID and check they are valid
source_type = form.cleaned_data['source_type']
if source_type not in self.source_types.values():
form.add_error(
'source_type',
ValidationError(self.errors['invalid_source'])
)
source_url = form.cleaned_data['source_url']
validation_url = self.validation_urls.get(self.source_type)
try:
validate_url(source_url, validation_url)
except ValidationError as e:
print(e)
error = self.errors.get('invalid_url')
item = self.help_item.get(self.source_type)
form.add_error(
'source_url',
ValidationError(error.format(
item=item,
example=validation_url['example'],
error=e.message)
)
)
if form.errors:
return super().form_invalid(form)
return super().form_valid(form)
def clean(self):
cleaned_data = super().clean()
print(cleaned_data)
return cleaned_data
def get_success_url(self):
return reverse_lazy('sync:dashboard')
class MediaView(TemplateView):
'''
A bare list of media added with their states.
'''
template_name = 'sync/media.html'
def dispatch(self, request, *args, **kwargs):
return super().dispatch(request, *args, **kwargs)
class TasksView(TemplateView):
'''
A list of tasks queued to be completed. Typically, this is scraping for new
media or downloading media.
'''
template_name = 'sync/tasks.html'
def dispatch(self, request, *args, **kwargs):
return super().dispatch(request, *args, **kwargs)
class LogsView(TemplateView):
'''
The last X days of logs.
'''
template_name = 'sync/logs.html'
def dispatch(self, request, *args, **kwargs):
return super().dispatch(request, *args, **kwargs)