base container and build process, more django

This commit is contained in:
meeb 2020-11-24 18:18:39 +11:00
parent 6ed18a6953
commit a2619bc607
18 changed files with 414 additions and 11204 deletions

116
Dockerfile Normal file
View File

@ -0,0 +1,116 @@
FROM debian:buster-slim
# Third party software versions
ARG YOUTUBE_DL_VERSION=2020.11.24
ENV YOUTUBE_DL_EXPECTED_SHA256=7d70f2e2d6b42d7c948a418744cd5c89832d67f4fb36f01f1cf4ea7dc8fe537a
ENV YOUTUBE_DL_TARBALL=https://github.com/ytdl-org/youtube-dl/releases/download/${YOUTUBE_DL_VERSION}/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz
ARG FFMPEG_VERSION=4.3.1
ENV FFMPEG_EXPECTED_MD5=ee235393ec7778279144ee6cbdd9eb64
ENV FFMPEG_TARBALL=https://johnvansickle.com/ffmpeg/releases/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz
# Install third party software
RUN set -x && \
# Install required distro packages
apt-get update && \
apt-get -y --no-install-recommends install curl xz-utils ca-certificates binutils python3 python3-setuptools && \
# Install youtube-dl
curl -L ${YOUTUBE_DL_TARBALL} --output /tmp/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz && \
echo "${YOUTUBE_DL_EXPECTED_SHA256} /tmp/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz" | sha256sum -c - && \
tar -zxvf /tmp/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz -C /tmp && \
(cd /tmp/youtube-dl; python3 /tmp/youtube-dl/setup.py install) && \
# Install ffmpeg
curl -L ${FFMPEG_TARBALL} --output /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz && \
echo "${FFMPEG_EXPECTED_MD5} tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz" | md5sum -c - && \
xz --decompress /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz && \
tar -xvf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar -C /tmp && \
ls -lat /tmp/ffmpeg-4.3.1-amd64-static && \
install -v -s -g root -o root -m 0755 -s /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static/ffmpeg -t /usr/local/bin && \
# Clean up
rm /tmp/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz && \
rm -rf /tmp/youtube-dl && \
rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar && \
rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static && \
apt-get -y autoremove --purge curl xz-utils binutils
# Defaults
ARG default_uid=10000
ARG default_gid=10000
# Copy app
COPY app /app
COPY app/tubesync/local_settings.py.container /app/tubesync/local_settings.py
# Add Pipfiles
COPY Pipfile /app/Pipfile
COPY Pipfile.lock /app/Pipfile.lock
# Switch workdir to the the app
WORKDIR /app
# Set up the app
ENV UID=$default_uid
ENV GID=$default_gid
RUN set -x && \
# Install required distro packages
apt-get -y --no-install-recommends install python3-pip python3-dev gcc && \
# Install wheel which is required for pipenv
pip3 --disable-pip-version-check install wheel && \
# Then install pipenv
pip3 --disable-pip-version-check install pipenv && \
# Create a 'www' user which the workers drop to
groupadd -g ${GID} www && \
useradd -M -d /dev/null -s /bin/false -u ${UID} -g www www && \
# Install non-distro packages
pipenv install --system && \
# Make absolutely sure we didn't accidentally bundle a SQLite dev database
rm -rf /app/db.sqlite3 && \
# Create config, downloads and run dirs we can write to
mkdir -p /run/www && \
chown -R www:www /run/www && \
chmod -R 0700 /run/www && \
mkdir -p /config && \
chown -R www:www /config && \
chmod -R 0755 /config && \
mkdir -p /downloads && \
chown -R www:www /downloads && \
chmod -R 0755 /downloads && \
# Reset permissions
mkdir -p /app/static && \
chown -R root:www /app && \
chown -R www:www /app/common/static && \
chown -R www:www /app/static && \
chmod -R 0750 /app && \
find /app -type f -exec chmod 640 {} \; && \
chmod 0750 /app/entrypoint.sh && \
# Clean up
rm /app/Pipfile && \
rm /app/Pipfile.lock && \
pipenv --clear && \
pip3 --disable-pip-version-check uninstall -y pipenv wheel virtualenv && \
apt-get -y autoremove --purge python3-pip python3-dev gcc && \
apt-get -y autoremove && \
apt-get -y autoclean && \
rm -rf /var/lib/apt/lists/* && \
rm -rf /var/cache/apt/* && \
rm -rf /tmp/* && \
# Pipenv leaves a bunch of stuff in /root, as we're not using it recreate it
rm -rf /root && \
mkdir -p /root && \
chown root:root /root && \
chmod 0700 /root
# Create a healthcheck
HEALTHCHECK --interval=1m --timeout=10s CMD /app/healthcheck.py http://127.0.0.1:8080/healthcheck
# Drop to the www user
USER www
# ENVS and ports
ENV PYTHONPATH "/app:${PYTHONPATH}"
EXPOSE 8080
# Entrypoint
ENTRYPOINT ["/app/entrypoint.sh"]
# Run gunicorn
CMD ["/usr/local/bin/gunicorn", "-c", "/app/tubesync/gunicorn.py", "--capture-output", "tubesync.wsgi:application"]

33
Makefile Normal file
View File

@ -0,0 +1,33 @@
python=/usr/bin/env python
docker=/usr/bin/docker
name=tubesync
image=$(name):latest
all: clean build
dev:
$(python) app/manage.py runserver
build:
mkdir -p app/media
mkdir -p app/static
$(python) app/manage.py collectstatic --noinput
clean:
rm -rf app/static
container: clean
$(docker) build -t $(image) .
runcontainer:
$(docker) run --rm --name $(name) --env-file dev.env --log-opt max-size=50m -ti -p 8080:8080 $(image)
test:
$(python) app/manage.py test --verbosity=2

View File

@ -10,6 +10,9 @@ django = "*"
django-sass-processor = "*"
libsass = "*"
pillow = "*"
whitenoise = "*"
gunicorn = "*"
django-compressor = "*"
[requires]
python_version = "3"

57
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "442def38f62cf644566bc2d74dff42564f728a062a71bcab5bdf816b240bb2a2"
"sha256": "09885662bbf2e7551756340f1f90f4245bdaa198e009a0abc5eb43c095952bab"
},
"pipfile-spec": 6,
"requires": {
@ -31,6 +31,21 @@
"index": "pypi",
"version": "==3.1.3"
},
"django-appconf": {
"hashes": [
"sha256:1b1d0e1069c843ebe8ae5aa48ec52403b1440402b320c3e3a206a0907e97bb06",
"sha256:be58deb54a43d77d2e1621fe59f787681376d3cd0b8bd8e4758ef6c3a6453380"
],
"version": "==1.0.4"
},
"django-compressor": {
"hashes": [
"sha256:57ac0a696d061e5fc6fbc55381d2050f353b973fb97eee5593f39247bc0f30af",
"sha256:d2ed1c6137ddaac5536233ec0a819e14009553fee0a869bea65d03e5285ba74f"
],
"index": "pypi",
"version": "==2.4"
},
"django-sass-processor": {
"hashes": [
"sha256:9b46a12ca8bdcb397d46fbcc49e6a926ff9f76a93c5efeb23b495419fd01fc7a"
@ -38,6 +53,14 @@
"index": "pypi",
"version": "==0.8.2"
},
"gunicorn": {
"hashes": [
"sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626",
"sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"
],
"index": "pypi",
"version": "==20.0.4"
},
"libsass": {
"hashes": [
"sha256:1521d2a8d4b397c6ec90640a1f6b5529077035efc48ef1c2e53095544e713d1b",
@ -98,6 +121,30 @@
],
"version": "==2020.4"
},
"rcssmin": {
"hashes": [
"sha256:ca87b695d3d7864157773a61263e5abb96006e9ff0e021eff90cbe0e1ba18270"
],
"version": "==1.0.6"
},
"rjsmin": {
"hashes": [
"sha256:0ab825839125eaca57cc59581d72e596e58a7a56fbc0839996b7528f0343a0a8",
"sha256:211c2fe8298951663bbc02acdffbf714f6793df54bfc50e1c6c9e71b3f2559a3",
"sha256:466fe70cc5647c7c51b3260c7e2e323a98b2b173564247f9c89e977720a0645f",
"sha256:585e75a84d9199b68056fd4a083d9a61e2a92dfd10ff6d4ce5bdb04bc3bdbfaf",
"sha256:6044ca86e917cd5bb2f95e6679a4192cef812122f28ee08c677513de019629b3",
"sha256:714329db774a90947e0e2086cdddb80d5e8c4ac1c70c9f92436378dedb8ae345",
"sha256:799890bd07a048892d8d3deb9042dbc20b7f5d0eb7da91e9483c561033b23ce2",
"sha256:975b69754d6a76be47c0bead12367a1ca9220d08e5393f80bab0230d4625d1f4",
"sha256:b15dc75c71f65d9493a8c7fa233fdcec823e3f1b88ad84a843ffef49b338ac32",
"sha256:dd0f4819df4243ffe4c964995794c79ca43943b5b756de84be92b445a652fb86",
"sha256:e3908b21ebb584ce74a6ac233bdb5f29485752c9d3be5e50c5484ed74169232c",
"sha256:e487a7783ac4339e79ec610b98228eb9ac72178973e3dee16eba0e3feef25924",
"sha256:ecd29f1b3e66a4c0753105baec262b331bcbceefc22fbe6f7e8bcd2067bcb4d7"
],
"version": "==1.1.0"
},
"six": {
"hashes": [
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
@ -111,6 +158,14 @@
"sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8"
],
"version": "==0.4.1"
},
"whitenoise": {
"hashes": [
"sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7",
"sha256:05d00198c777028d72d8b0bbd234db605ef6d60e9410125124002518a48e515d"
],
"index": "pypi",
"version": "==5.2.0"
}
},
"develop": {}

View File

@ -0,0 +1,7 @@
from django.conf import settings
def app_details(request):
return {
'app_version': str(settings.VERSION)
}

21
app/common/middleware.py Normal file
View File

@ -0,0 +1,21 @@
from django.forms import BaseForm
class MaterializeDefaultFieldsMiddleware:
'''
Adds 'browser-default' CSS attribute class to all form fields.
'''
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
response = self.get_response(request)
return response
def process_template_response(self, request, response):
for _, v in getattr(response, 'context_data', {}).items():
if isinstance(v, BaseForm):
for _, field in v.fields.items():
field.widget.attrs.update({'class':'browser-default'})
return response

View File

@ -12,5 +12,10 @@ $text-colour: $colour-near-black;
$header-background-colour: $colour-red;
$header-text-colour: $colour-white;
$nav-background-colour: $colour-near-black;
$nav-text-colour: $colour-near-white;
$nav-link-background-hover-colour: $colour-orange;
$footer-background-colour: $colour-red;
$footer-text-colour: $colour-white;
$footer-link-colour: $colour-near-black;

View File

@ -6,7 +6,7 @@ html {
}
header {
background-color: $header-background-colour;
color: $header-text-colour;
padding: 1.3rem 0 1.5rem 0;
@ -35,6 +35,29 @@ header {
}
nav {
background-color: $nav-background-colour;
color: $nav-text-colour;
height: 3rem;
a {
font-size: 1.1rem !important;
color: $nav-text-colour;
text-decoration: none;
line-height: 3rem;
height: 3rem;
i {
font-size: 1.1rem !important;
}
&:hover {
background-color: $nav-link-background-hover-colour !important;
}
}
}
main {
padding: 2rem 0 2rem 0;
@ -45,6 +68,29 @@ footer {
background-color: $footer-background-colour;
color: $footer-text-colour;
padding: 1.5rem 0 1.5rem 0;
padding-top: 1.5rem;
p {
margin: 0;
padding-bottom: 1.5rem;
}
svg {
path {
fill: $footer-link-colour !important;
}
}
i {
font-size: 0.85rem !important;
}
a {
color: $footer-link-colour;
text-decoration: none;
&:hover {
text-decoration: underline;
}
}
}

View File

@ -4,6 +4,7 @@
@import "fontawesome/fontawesome";
@import "fontawesome/regular";
@import "fontawesome/solid";
@import "fontawesome/brands";
@import "fonts";
@import "variables";

View File

@ -25,6 +25,16 @@
</div>
</header>
<nav>
<div class="container">
<ul>
<li><a href=""><i class="fas fa-fw fa-th-large"></i><span class="hide-on-med-and-down"> Dashboard</span></a></li>
<li><a href=""><i class="fas fa-fw fa-play"></i><span class="hide-on-med-and-down"> Sources</span></a></li>
<li><a href=""><i class="fas fa-fw fa-film"></i><span class="hide-on-med-and-down"> Media</span></a></li>
<li><a href=""><i class="fas fa-fw fa-clock"></i><span class="hide-on-med-and-down"> Tasks</span></a></li>
</ul>
</div>
</nav>
<main>
<div class="container">
@ -34,7 +44,14 @@
<footer>
<div class="container">
footer
<p>
<a href="{% url 'sync:index' %}">{% include 'tubesync.svg' with width='0.8rem' height='0.8rem' %} TubeSync</a>
is an open source synchronisation tool to automatically download videos from online video platforms.
<br>
The original code under a GPLv3 licence is available at
<a href="https://github.com/meeb/tubesync"><i class="fab fa-github"></i> https://github.com/meeb/tubesync</a>.
</p>
<p>Version {{ app_version }}.</p>
</div>
</footer>

17
app/entrypoint.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
set -x
# Compile SCSS files
/usr/bin/python3 /app/manage.py compilescss
# Collect the static files
/usr/bin/python3 /app/manage.py collectstatic --no-input --link
# Run migrations
/usr/bin/python3 /app/manage.py migrate
# Run what's in CMD
exec "$@"
# eof

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

33
app/tubesync/gunicorn.py Normal file
View File

@ -0,0 +1,33 @@
import os
import multiprocessing
def get_num_workers():
cpu_workers = multiprocessing.cpu_count() * 2 + 1
try:
num_workers = int(os.getenv('GUNICORN_WORKERS', 1))
except ValueError:
num_workers = cpu_workers
if 0 > num_workers > cpu_workers:
num_workers = cpu_workers
return num_workers
def get_bind():
host = os.getenv('LISTEN_HOST', '0.0.0.0')
port = os.getenv('LISTEN_PORT', '8080')
return '{}:{}'.format(host, port)
workers = get_num_workers()
timeout = 30
chdir = '/app'
daemon = False
pidfile = '/run/www/gunicorn.pid'
user = 'www'
group = 'www'
loglevel = 'info'
errorlog = '-'
accesslog = '-'
django_settings = 'django.settings'
bind = get_bind()

View File

@ -0,0 +1,14 @@
import os
SECRET_KEY = str(os.getenv('DJANGO_SECRET_KEY', ''))
ALLOWED_HOSTS_STR = str(os.getenv('DJANGO_ALLOWED_HOSTS', ''))
ALLOWED_HOSTS = ALLOWED_HOSTS_STR.split(',')
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': '/config/db.sqlite3',
}
}

View File

@ -4,6 +4,7 @@ from pathlib import Path
BASE_DIR = Path(__file__).resolve().parent.parent
VERSION = 0.1
SECRET_KEY = ''
DEBUG = False
ALLOWED_HOSTS = []
@ -30,6 +31,8 @@ MIDDLEWARE = [
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'whitenoise.middleware.WhiteNoiseMiddleware',
'common.middleware.MaterializeDefaultFieldsMiddleware',
]
@ -47,6 +50,7 @@ TEMPLATES = [
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
'common.context_processors.app_details',
],
},
},

3
dev.env Normal file
View File

@ -0,0 +1,3 @@
GUNICORN_WORKERS=1
DJANGO_ALLOWED_HOSTS=localhost
DJANGO_SECRET_KEY=not-a-secret

35
healthcheck.py Normal file
View File

@ -0,0 +1,35 @@
#!/usr/bin/env python3
'''
Perform an HTTP request to a URL and exit with an exit code of 1 if the
request did not return an HTTP/200 status code.
Usage:
$ ./healthcheck.py http://some.url.here/healthcheck/resource
'''
import sys
import requests
TIMEOUT = 5 # Seconds
def do_heatlhcheck(url):
headers = {'User-Agent': 'healthcheck'}
response = requests.get(url, headers=headers, timeout=TIMEOUT)
return response.status_code == 200
if __name__ == '__main__':
try:
url = sys.argv[1]
except IndexError:
sys.stderr.write('URL must be supplied\n')
sys.exit(1)
if do_heatlhcheck(url):
sys.exit(0)
else:
sys.exit(1)