base container and build process, more django

Dockerfile

@@ -0,0 +1,116 @@
+FROM debian:buster-slim
+
+# Third party software versions
+ARG YOUTUBE_DL_VERSION=2020.11.24
+ENV YOUTUBE_DL_EXPECTED_SHA256=7d70f2e2d6b42d7c948a418744cd5c89832d67f4fb36f01f1cf4ea7dc8fe537a
+ENV YOUTUBE_DL_TARBALL=https://github.com/ytdl-org/youtube-dl/releases/download/${YOUTUBE_DL_VERSION}/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz
+ARG FFMPEG_VERSION=4.3.1
+ENV FFMPEG_EXPECTED_MD5=ee235393ec7778279144ee6cbdd9eb64
+ENV FFMPEG_TARBALL=https://johnvansickle.com/ffmpeg/releases/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz
+
+# Install third party software
+RUN set -x && \
+    # Install required distro packages
+    apt-get update && \
+    apt-get -y --no-install-recommends install curl xz-utils ca-certificates binutils python3 python3-setuptools && \
+    # Install youtube-dl
+    curl -L ${YOUTUBE_DL_TARBALL} --output /tmp/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz && \
+    echo "${YOUTUBE_DL_EXPECTED_SHA256}  /tmp/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz" | sha256sum -c - && \
+    tar -zxvf /tmp/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz -C /tmp && \
+    (cd /tmp/youtube-dl; python3 /tmp/youtube-dl/setup.py install) && \
+    # Install ffmpeg
+    curl -L ${FFMPEG_TARBALL} --output /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz && \
+    echo "${FFMPEG_EXPECTED_MD5}  tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz" | md5sum -c - && \
+    xz --decompress /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz && \
+    tar -xvf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar -C /tmp && \
+    ls -lat /tmp/ffmpeg-4.3.1-amd64-static && \
+    install -v -s -g root -o root -m 0755 -s /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static/ffmpeg -t /usr/local/bin && \
+    # Clean up
+    rm /tmp/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz && \
+    rm -rf /tmp/youtube-dl && \
+    rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar && \
+    rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static && \
+    apt-get -y autoremove --purge curl xz-utils binutils
+
+# Defaults
+ARG default_uid=10000
+ARG default_gid=10000
+
+# Copy app
+COPY app /app
+COPY app/tubesync/local_settings.py.container /app/tubesync/local_settings.py
+
+# Add Pipfiles
+COPY Pipfile /app/Pipfile
+COPY Pipfile.lock /app/Pipfile.lock
+
+# Switch workdir to the the app
+WORKDIR /app
+
+# Set up the app
+ENV UID=$default_uid
+ENV GID=$default_gid
+RUN set -x && \
+  # Install required distro packages
+  apt-get -y --no-install-recommends install python3-pip python3-dev gcc && \
+  # Install wheel which is required for pipenv
+  pip3 --disable-pip-version-check install wheel && \
+  # Then install pipenv
+  pip3 --disable-pip-version-check install pipenv && \
+  # Create a 'www' user which the workers drop to
+  groupadd -g ${GID} www && \
+  useradd -M -d /dev/null -s /bin/false -u ${UID} -g www www && \
+  # Install non-distro packages
+  pipenv install --system  && \
+  # Make absolutely sure we didn't accidentally bundle a SQLite dev database
+  rm -rf /app/db.sqlite3 && \
+  # Create config, downloads and run dirs we can write to
+  mkdir -p /run/www && \
+  chown -R www:www /run/www && \
+  chmod -R 0700 /run/www && \
+  mkdir -p /config && \
+  chown -R www:www /config && \
+  chmod -R 0755 /config && \
+  mkdir -p /downloads && \
+  chown -R www:www /downloads && \
+  chmod -R 0755 /downloads && \
+  # Reset permissions
+  mkdir -p /app/static && \
+  chown -R root:www /app && \
+  chown -R www:www /app/common/static && \
+  chown -R www:www /app/static && \
+  chmod -R 0750 /app && \
+  find /app -type f -exec chmod 640 {} \; && \
+  chmod 0750 /app/entrypoint.sh && \
+  # Clean up
+  rm /app/Pipfile && \
+  rm /app/Pipfile.lock && \
+  pipenv --clear && \
+  pip3 --disable-pip-version-check uninstall -y pipenv wheel virtualenv && \
+  apt-get -y autoremove --purge python3-pip python3-dev gcc && \
+  apt-get -y autoremove && \
+  apt-get -y autoclean && \
+  rm -rf /var/lib/apt/lists/* && \
+  rm -rf /var/cache/apt/* && \
+  rm -rf /tmp/* && \
+  # Pipenv leaves a bunch of stuff in /root, as we're not using it recreate it
+  rm -rf /root && \
+  mkdir -p /root && \
+  chown root:root /root && \
+  chmod 0700 /root
+
+# Create a healthcheck
+HEALTHCHECK --interval=1m --timeout=10s CMD /app/healthcheck.py http://127.0.0.1:8080/healthcheck
+
+# Drop to the www user
+USER www
+
+# ENVS and ports
+ENV PYTHONPATH "/app:${PYTHONPATH}"
+EXPOSE 8080
+
+# Entrypoint
+ENTRYPOINT ["/app/entrypoint.sh"]
+
+# Run gunicorn
+CMD ["/usr/local/bin/gunicorn", "-c", "/app/tubesync/gunicorn.py", "--capture-output", "tubesync.wsgi:application"]

Makefile

@@ -0,0 +1,33 @@
+python=/usr/bin/env python
+docker=/usr/bin/docker
+name=tubesync
+image=$(name):latest
+
+
+all: clean build
+
+
+dev:
+	$(python) app/manage.py runserver
+
+
+build:
+	mkdir -p app/media
+	mkdir -p app/static
+	$(python) app/manage.py collectstatic --noinput
+
+
+clean:
+	rm -rf app/static
+
+
+container: clean
+	$(docker) build -t $(image) .
+
+
+runcontainer:
+	$(docker) run --rm --name $(name) --env-file dev.env --log-opt max-size=50m -ti -p 8080:8080 $(image)
+
+
+test:
+	$(python) app/manage.py test --verbosity=2

Pipfile

@@ -10,6 +10,9 @@ django = "*"
 django-sass-processor = "*"
 libsass = "*"
 pillow = "*"
+whitenoise = "*"
+gunicorn = "*"
+django-compressor = "*"
 
 [requires]
 python_version = "3"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "442def38f62cf644566bc2d74dff42564f728a062a71bcab5bdf816b240bb2a2"
+            "sha256": "09885662bbf2e7551756340f1f90f4245bdaa198e009a0abc5eb43c095952bab"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -31,6 +31,21 @@
             "index": "pypi",
             "version": "==3.1.3"
         },
+        "django-appconf": {
+            "hashes": [
+                "sha256:1b1d0e1069c843ebe8ae5aa48ec52403b1440402b320c3e3a206a0907e97bb06",
+                "sha256:be58deb54a43d77d2e1621fe59f787681376d3cd0b8bd8e4758ef6c3a6453380"
+            ],
+            "version": "==1.0.4"
+        },
+        "django-compressor": {
+            "hashes": [
+                "sha256:57ac0a696d061e5fc6fbc55381d2050f353b973fb97eee5593f39247bc0f30af",
+                "sha256:d2ed1c6137ddaac5536233ec0a819e14009553fee0a869bea65d03e5285ba74f"
+            ],
+            "index": "pypi",
+            "version": "==2.4"
+        },
         "django-sass-processor": {
             "hashes": [
                 "sha256:9b46a12ca8bdcb397d46fbcc49e6a926ff9f76a93c5efeb23b495419fd01fc7a"
@@ -38,6 +53,14 @@
             "index": "pypi",
             "version": "==0.8.2"
         },
+        "gunicorn": {
+            "hashes": [
+                "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626",
+                "sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"
+            ],
+            "index": "pypi",
+            "version": "==20.0.4"
+        },
         "libsass": {
             "hashes": [
                 "sha256:1521d2a8d4b397c6ec90640a1f6b5529077035efc48ef1c2e53095544e713d1b",
@@ -98,6 +121,30 @@
             ],
             "version": "==2020.4"
         },
+        "rcssmin": {
+            "hashes": [
+                "sha256:ca87b695d3d7864157773a61263e5abb96006e9ff0e021eff90cbe0e1ba18270"
+            ],
+            "version": "==1.0.6"
+        },
+        "rjsmin": {
+            "hashes": [
+                "sha256:0ab825839125eaca57cc59581d72e596e58a7a56fbc0839996b7528f0343a0a8",
+                "sha256:211c2fe8298951663bbc02acdffbf714f6793df54bfc50e1c6c9e71b3f2559a3",
+                "sha256:466fe70cc5647c7c51b3260c7e2e323a98b2b173564247f9c89e977720a0645f",
+                "sha256:585e75a84d9199b68056fd4a083d9a61e2a92dfd10ff6d4ce5bdb04bc3bdbfaf",
+                "sha256:6044ca86e917cd5bb2f95e6679a4192cef812122f28ee08c677513de019629b3",
+                "sha256:714329db774a90947e0e2086cdddb80d5e8c4ac1c70c9f92436378dedb8ae345",
+                "sha256:799890bd07a048892d8d3deb9042dbc20b7f5d0eb7da91e9483c561033b23ce2",
+                "sha256:975b69754d6a76be47c0bead12367a1ca9220d08e5393f80bab0230d4625d1f4",
+                "sha256:b15dc75c71f65d9493a8c7fa233fdcec823e3f1b88ad84a843ffef49b338ac32",
+                "sha256:dd0f4819df4243ffe4c964995794c79ca43943b5b756de84be92b445a652fb86",
+                "sha256:e3908b21ebb584ce74a6ac233bdb5f29485752c9d3be5e50c5484ed74169232c",
+                "sha256:e487a7783ac4339e79ec610b98228eb9ac72178973e3dee16eba0e3feef25924",
+                "sha256:ecd29f1b3e66a4c0753105baec262b331bcbceefc22fbe6f7e8bcd2067bcb4d7"
+            ],
+            "version": "==1.1.0"
+        },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
@@ -111,6 +158,14 @@
                 "sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8"
             ],
             "version": "==0.4.1"
+        },
+        "whitenoise": {
+            "hashes": [
+                "sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7",
+                "sha256:05d00198c777028d72d8b0bbd234db605ef6d60e9410125124002518a48e515d"
+            ],
+            "index": "pypi",
+            "version": "==5.2.0"
         }
     },
     "develop": {}

app/common/context_processors.py

@@ -0,0 +1,7 @@
+from django.conf import settings
+
+
+def app_details(request):
+    return {
+        'app_version': str(settings.VERSION)
+    }

app/common/middleware.py

@@ -0,0 +1,21 @@
+from django.forms import BaseForm
+
+
+class MaterializeDefaultFieldsMiddleware:
+    '''
+        Adds 'browser-default' CSS attribute class to all form fields.
+    '''
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        return response
+
+    def process_template_response(self, request, response):
+        for _, v in getattr(response, 'context_data', {}).items():
+            if isinstance(v, BaseForm):
+                for _, field in v.fields.items():
+                    field.widget.attrs.update({'class':'browser-default'})
+        return response

app/common/static/styles/_colours.scss

@@ -12,5 +12,10 @@ $text-colour: $colour-near-black;
 $header-background-colour: $colour-red;
 $header-text-colour: $colour-white;
 
+$nav-background-colour: $colour-near-black;
+$nav-text-colour: $colour-near-white;
+$nav-link-background-hover-colour: $colour-orange;
+
 $footer-background-colour: $colour-red;
 $footer-text-colour: $colour-white;
+$footer-link-colour: $colour-near-black;

app/common/static/styles/_template.scss

@@ -6,7 +6,7 @@ html {
 }
 
 header {
-    
+
     background-color: $header-background-colour;
     color: $header-text-colour;
     padding: 1.3rem 0 1.5rem 0;
@@ -35,6 +35,29 @@ header {
 
 }
 
+nav {
+
+    background-color: $nav-background-colour;
+    color: $nav-text-colour;
+    height: 3rem;
+
+    a {
+        font-size: 1.1rem !important;
+        color: $nav-text-colour;
+        text-decoration: none;
+        line-height: 3rem;
+        height: 3rem;
+        i {
+            font-size: 1.1rem !important;
+        }
+        &:hover {
+            background-color: $nav-link-background-hover-colour !important;
+        }
+    }
+
+
+}
+
 main {
     
     padding: 2rem 0 2rem 0;
@@ -45,6 +68,29 @@ footer {
 
     background-color: $footer-background-colour;
     color: $footer-text-colour;
-    padding: 1.5rem 0 1.5rem 0;
+    padding-top: 1.5rem;
+
+    p {
+        margin: 0;
+        padding-bottom: 1.5rem;
+    }
+
+    svg {
+        path {
+            fill: $footer-link-colour !important;
+        }
+    }
+
+    i {
+        font-size: 0.85rem !important;
+    }
+
+    a {
+        color: $footer-link-colour;
+        text-decoration: none;
+        &:hover {
+            text-decoration: underline;
+        }
+    }
 
 }

app/common/static/styles/tubesync.scss

@@ -4,6 +4,7 @@
 @import "fontawesome/fontawesome";
 @import "fontawesome/regular";
 @import "fontawesome/solid";
+@import "fontawesome/brands";
 
 @import "fonts";
 @import "variables";

app/common/templates/base.html

@@ -25,6 +25,16 @@
       </div>
     </header>
 
+    <nav>
+      <div class="container">
+        <ul>
+          <li><a href=""><i class="fas fa-fw fa-th-large"></i><span class="hide-on-med-and-down"> Dashboard</span></a></li>
+          <li><a href=""><i class="fas fa-fw fa-play"></i><span class="hide-on-med-and-down"> Sources</span></a></li>
+          <li><a href=""><i class="fas fa-fw fa-film"></i><span class="hide-on-med-and-down"> Media</span></a></li>
+          <li><a href=""><i class="fas fa-fw fa-clock"></i><span class="hide-on-med-and-down"> Tasks</span></a></li>
+        </ul>
+      </div>
+    </nav>
 
     <main>
       <div class="container">
@@ -34,7 +44,14 @@
 
     <footer>
       <div class="container">
-        footer
+        <p>
+          <a href="{% url 'sync:index' %}">{% include 'tubesync.svg' with width='0.8rem' height='0.8rem' %} TubeSync</a>
+          is an open source synchronisation tool to automatically download videos from online video platforms.
+          <br>
+          The original code under a GPLv3 licence is available at
+          <a href="https://github.com/meeb/tubesync"><i class="fab fa-github"></i> https://github.com/meeb/tubesync</a>.
+        </p>
+        <p>Version {{ app_version }}.</p>
       </div>
     </footer>
 

app/entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -x
+
+# Compile SCSS files
+/usr/bin/python3 /app/manage.py compilescss
+
+# Collect the static files
+/usr/bin/python3 /app/manage.py collectstatic --no-input --link
+
+# Run migrations
+/usr/bin/python3 /app/manage.py migrate
+
+# Run what's in CMD
+exec "$@"
+
+# eof

app/tubesync/gunicorn.py

@@ -0,0 +1,33 @@
+import os
+import multiprocessing
+
+
+def get_num_workers():
+    cpu_workers = multiprocessing.cpu_count() * 2 + 1
+    try:
+        num_workers = int(os.getenv('GUNICORN_WORKERS', 1))
+    except ValueError:
+        num_workers = cpu_workers
+    if 0 > num_workers > cpu_workers:
+        num_workers = cpu_workers
+    return num_workers
+
+
+def get_bind():
+    host = os.getenv('LISTEN_HOST', '0.0.0.0')
+    port = os.getenv('LISTEN_PORT', '8080')
+    return '{}:{}'.format(host, port)
+
+
+workers = get_num_workers()
+timeout = 30
+chdir = '/app'
+daemon = False
+pidfile = '/run/www/gunicorn.pid'
+user = 'www'
+group = 'www'
+loglevel = 'info'
+errorlog = '-'
+accesslog = '-'
+django_settings = 'django.settings'
+bind = get_bind()

app/tubesync/local_settings.py.container

@@ -0,0 +1,14 @@
+import os
+
+
+SECRET_KEY = str(os.getenv('DJANGO_SECRET_KEY', ''))
+ALLOWED_HOSTS_STR = str(os.getenv('DJANGO_ALLOWED_HOSTS', ''))
+ALLOWED_HOSTS = ALLOWED_HOSTS_STR.split(',')
+
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': '/config/db.sqlite3',
+    }
+}

app/tubesync/settings.py

@@ -4,6 +4,7 @@ from pathlib import Path
 BASE_DIR = Path(__file__).resolve().parent.parent
 
 
+VERSION = 0.1
 SECRET_KEY = ''
 DEBUG = False
 ALLOWED_HOSTS = []
@@ -30,6 +31,8 @@ MIDDLEWARE = [
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'whitenoise.middleware.WhiteNoiseMiddleware',
+    'common.middleware.MaterializeDefaultFieldsMiddleware',
 ]
 
 
@@ -47,6 +50,7 @@ TEMPLATES = [
                 'django.template.context_processors.request',
                 'django.contrib.auth.context_processors.auth',
                 'django.contrib.messages.context_processors.messages',
+                'common.context_processors.app_details',
             ],
         },
     },

dev.env

@@ -0,0 +1,3 @@
+GUNICORN_WORKERS=1
+DJANGO_ALLOWED_HOSTS=localhost
+DJANGO_SECRET_KEY=not-a-secret

healthcheck.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+'''
+
+    Perform an HTTP request to a URL and exit with an exit code of 1 if the
+    request did not return an HTTP/200 status code.
+
+    Usage:
+    $ ./healthcheck.py http://some.url.here/healthcheck/resource
+
+'''
+
+
+import sys
+import requests
+
+
+TIMEOUT = 5  # Seconds
+
+
+def do_heatlhcheck(url):
+    headers = {'User-Agent': 'healthcheck'}
+    response = requests.get(url, headers=headers, timeout=TIMEOUT)
+    return response.status_code == 200
+
+
+if __name__ == '__main__':
+    try:
+        url = sys.argv[1]
+    except IndexError:
+        sys.stderr.write('URL must be supplied\n')
+        sys.exit(1)
+    if do_heatlhcheck(url):
+        sys.exit(0)
+    else:
+        sys.exit(1)