functioning container build

.dockerignore

@@ -3,3 +3,5 @@
 .github
 .gitattributes
 README.md
+tubesync/media
+tubesync/downloads

Dockerfile

@@ -1,33 +1,45 @@
 FROM debian:buster-slim
 
-ARG DEBIAN_FRONTEND="noninteractive"
+ARG ARCH="amd64"
-
+ARG S6_VERSION="2.1.0.2"
-# Third party software versions
 ARG FFMPEG_VERSION="4.3.1"
-ENV FFMPEG_EXPECTED_MD5="ee235393ec7778279144ee6cbdd9eb64"
+
-ENV FFMPEG_TARBALL="https://johnvansickle.com/ffmpeg/releases/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz"
+ENV DEBIAN_FRONTEND="noninteractive" \
+    HOME="/root" \
+    LANGUAGE="en_US.UTF-8" \
+    LANG="en_US.UTF-8" \
+    LC_ALL="en_US.UTF-8" \
+    TERM="xterm" \
+    S6_EXPECTED_SHA256="52460473413601ff7a84ae690b161a074217ddc734990c2cdee9847166cf669e" \
+    S6_DOWNLOAD="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${ARCH}.tar.gz" \
+    FFMPEG_EXPECTED_SHA256="47d95c0129fba27d051748a442a44a73ce1bd38d1e3f9fe1e9dd7258c7581fa5" \
+    FFMPEG_DOWNLOAD="https://johnvansickle.com/ffmpeg/releases/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static.tar.xz"
 
 # Install third party software
 RUN set -x && \
-    # Install required distro packages
     apt-get update && \
+    apt-get -y --no-install-recommends install locales && \
+    echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
+    locale-gen en_US.UTF-8 && \
+    # Install required distro packages
     apt-get -y --no-install-recommends install curl xz-utils ca-certificates binutils && \
+    # Install s6
+    curl -L ${S6_DOWNLOAD} --output /tmp/s6-overlay-${ARCH}.tar.gz && \
+    sha256sum /tmp/s6-overlay-${ARCH}.tar.gz && \
+    echo "${S6_EXPECTED_SHA256}  /tmp/s6-overlay-${ARCH}.tar.gz" | sha256sum -c - && \
+    tar xzf /tmp/s6-overlay-${ARCH}.tar.gz -C / && \
     # Install ffmpeg
-    curl -L ${FFMPEG_TARBALL} --output /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz && \
+    curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg-${ARCH}-static.tar.xz && \
-    echo "${FFMPEG_EXPECTED_MD5}  tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz" | md5sum -c - && \
+    echo "${FFMPEG_EXPECTED_SHA256}  /tmp/ffmpeg-${ARCH}-static.tar.xz" | sha256sum -c - && \
-    xz --decompress /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz && \
+    xz --decompress /tmp/ffmpeg-${ARCH}-static.tar.xz && \
-    tar -xvf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar -C /tmp && \
+    tar -xvf /tmp/ffmpeg-${ARCH}-static.tar -C /tmp && \
-    ls -lat /tmp/ffmpeg-4.3.1-amd64-static && \
+    install -v -s -g root -o root -m 0755 -s /tmp/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static/ffmpeg -t /usr/local/bin && \
-    install -v -s -g root -o root -m 0755 -s /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static/ffmpeg -t /usr/local/bin && \
     # Clean up
-    rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar && \
+    rm -rf /tmp/s6-overlay-${ARCH}.tar.gz && \
-    rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static && \
+    rm -rf /tmp/ffmpeg-${ARCH}-static.tar && \
+    rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static && \
     apt-get -y autoremove --purge curl xz-utils binutils
 
-# Defaults
-ARG default_uid="10000"
-ARG default_gid="10000"
-
 # Copy app
 COPY tubesync /app
 COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings.py
@@ -35,7 +47,7 @@ COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings.
 # Append container bundled software versions
 RUN echo "ffmpeg_version = '${FFMPEG_VERSION}-static'" >> /app/common/third_party_versions.py
 
-# Add Pipfiles
+# Add Pipfile
 COPY Pipfile /app/Pipfile
 COPY Pipfile.lock /app/Pipfile.lock
 
@@ -43,40 +55,27 @@ COPY Pipfile.lock /app/Pipfile.lock
 WORKDIR /app
 
 # Set up the app
-ENV UID="${default_uid}"
-ENV GID="${default_gid}"
 RUN set -x && \
   # Install required distro packages
-  apt-get -y --no-install-recommends install python3 python3-setuptools python3-pip python3-dev gcc make psmisc procps && \
+  apt-get -y install nginx-light && \
-  # Install wheel which is required for pipenv
+  apt-get -y --no-install-recommends install python3 python3-setuptools python3-pip python3-dev gcc make && \
-  pip3 --disable-pip-version-check install wheel && \
+  # Install pipenv
-  # Then install pipenv
   pip3 --disable-pip-version-check install pipenv && \
-  # Create a 'www' user which the workers drop to
+  # Create a 'app' user which the application will run as
-  groupadd -g ${GID} www && \
+  groupadd app && \
-  useradd -M -d /app -s /bin/false -u ${UID} -g www www && \
+  useradd -M -d /app -s /bin/false -g app app && \
   # Install non-distro packages
-  pipenv install --system --verbose && \
+  pipenv install --system && \
   # Make absolutely sure we didn't accidentally bundle a SQLite dev database
   rm -rf /app/db.sqlite3 && \
-  # Create config, downloads and run dirs we can write to
+  # Run any required app commands
-  mkdir -p /run/www && \
+  /usr/bin/python3 /app/manage.py compilescss && \
-  chown -R www:www /run/www && \
+  /usr/bin/python3 /app/manage.py collectstatic --no-input --link && \
-  chmod -R 0700 /run/www && \
+  # Create config, downloads and run dirs
+  mkdir -p /run/app && \
   mkdir -p /config/media && \
-  chown -R www:www /config && \
+  mkdir -p /downloads/audio && \
-  chmod -R 0755 /config && \
+  mkdir -p /downloads/video && \
-  mkdir -p /downloads/{audio,video} && \
-  chown -R www:www /downloads && \
-  chmod -R 0755 /downloads && \
-  # Reset permissions
-  mkdir -p /app/static && \
-  chown -R root:www /app && \
-  chown -R www:www /app/common/static && \
-  chown -R www:www /app/static && \
-  chmod -R 0750 /app && \
-  find /app -type f -exec chmod 640 {} \; && \
-  chmod 0750 /app/entrypoint.sh && \
   # Clean up
   rm /app/Pipfile && \
   rm /app/Pipfile.lock && \
@@ -94,18 +93,18 @@ RUN set -x && \
   chown root:root /root && \
   chmod 0700 /root
 
+# Copy root
+COPY config/root /
+
 # Create a healthcheck
 HEALTHCHECK --interval=1m --timeout=10s CMD /app/healthcheck.py http://127.0.0.1:8080/healthcheck
 
-# Drop to the www user
-#USER www
-
 # ENVS and ports
 ENV PYTHONPATH "/app:${PYTHONPATH}"
 EXPOSE 8080
 
-# Entrypoint
+# Volumes
-ENTRYPOINT ["/app/entrypoint.sh"]
+VOLUME ["/config", "/downloads"]
 
-# Run gunicorn
+# Entrypoint, start s6 init
-CMD ["/usr/local/bin/gunicorn", "-c", "/app/tubesync/gunicorn.py", "--capture-output", "tubesync.wsgi:application"]
+ENTRYPOINT ["/init"]

Makefile

@@ -26,7 +26,7 @@ container: clean
 
 
 runcontainer:
-	$(docker) run --rm --name $(name) --env-file dev.env --log-opt max-size=50m -ti -p 8080:8080 $(image)
+	$(docker) run --rm --name $(name) --env-file dev.env --log-opt max-size=50m -ti -p 4848:4848 $(image)
 
 
 test:

config/root/docs/index.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
+  <title>Something has gone very wrong</title>
+  <style>
+    body {
+      max-width: 600px;
+      padding: 10px 20px 20px 20px;
+      font-family: Sans-Serif;
+    }
+  </style>
+</head>
+<body>
+  <header>
+    <h1>Something has gone very wrong</h1>
+  </header>
+  <main>
+    <p>
+      If you can see this message then the front end web server has not forwarded the
+      connection on to the TubeSync back end server. This probably means something has
+      gone wrong with the container build or a process has crashed. Try restarting it.
+    </p>
+  </main>
+</body>
+</html>

config/root/etc/cont-init.d/10-tubesync

@@ -0,0 +1,22 @@
+#!/usr/bin/with-contenv bash
+
+# Change runtime user UID and GID
+PUID=${PUID:-911}
+PGID=${PGID:-911}
+groupmod -o -g "$PGID" app
+usermod -o -u "$PUID" app
+
+# Reset permissions
+chown -R app:app /run/app && \
+chmod -R 0700 /run/app && \
+chown -R app:app /config && \
+chmod -R 0755 /config && \
+chown -R app:app /downloads && \
+chmod -R 0755 /downloads && \
+chown -R root:app /app && \
+chmod -R 0750 /app && \
+find /app -type f -exec chmod 640 {} \; && \
+
+# Run migrations
+exec s6-setuidgid app \
+    /usr/bin/python3 /app/manage.py migrate

config/root/etc/nginx/nginx.conf

@@ -0,0 +1,85 @@
+daemon off;
+
+user app;
+worker_processes auto;
+pid /run/nginx.pid;
+
+events {
+  worker_connections 1024;
+}
+
+http {
+
+  # Basic settings
+  sendfile on;
+  tcp_nopush on;
+  tcp_nodelay on;
+  keepalive_timeout 300;
+  types_hash_max_size 2048;
+  server_tokens off;
+  server_names_hash_bucket_size 64;
+  server_name_in_redirect off;
+  client_body_in_file_only clean;
+  client_body_buffer_size 32K;
+  client_max_body_size 100M;
+  send_timeout 300s;
+  large_client_header_buffers 4 8k;
+
+  # Mime type handling
+  include /etc/nginx/mime.types;
+  default_type application/octet-stream;
+
+  # Default security headers
+  add_header X-Frame-Options SAMEORIGIN;
+  add_header X-Content-Type-Options nosniff;
+  add_header X-XSS-Protection "1; mode=block";
+
+  # Logging
+  log_format host '$remote_addr - $remote_user [$time_local] "[$host] $request" $status $bytes_sent "$http_referer" "$http_user_agent" "$gzip_ratio"';
+  access_log /dev/stdout;
+  error_log stderr;
+
+  # GZIP
+  gzip on;
+  gzip_disable "msie6";
+  gzip_vary on;
+  gzip_proxied any;
+  gzip_comp_level 6;
+  gzip_buffers 16 8k;
+  gzip_http_version 1.1;
+  gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
+
+    # Site
+    server {
+
+        # Ports
+        listen 4848;
+        listen [::]:4848;
+
+        # Web root
+        root /docs;
+        index index.html;
+
+        # Proxy
+        proxy_buffers 32 4k;
+        proxy_set_header Connection "";
+
+        # Server domain name
+        server_name _;
+
+        # Authentication and proxying
+        location / {
+            proxy_pass http://127.0.0.1:8080;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_redirect off;
+            proxy_read_timeout 59;
+            proxy_connect_timeout 10;
+        }
+
+    }
+
+
+}

config/root/etc/services.d/gunicorn/run

@@ -0,0 +1,9 @@
+#!/usr/bin/with-contenv bash
+
+UMASK_SET=${UMASK_SET:-022}
+umask "$UMASK_SET"
+
+cd /app || exit
+
+exec s6-setuidgid app \
+    /usr/local/bin/gunicorn -c /app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application

config/root/etc/services.d/nginx/run

@@ -0,0 +1,5 @@
+#!/usr/bin/with-contenv bash
+
+cd /
+
+/usr/sbin/nginx

config/root/etc/services.d/tubesync-worker/run

@@ -0,0 +1,4 @@
+#!/usr/bin/with-contenv bash
+
+exec s6-setuidgid app \
+    /usr/bin/python3 /app/manage.py process_tasks

dev.env

@@ -1,3 +1,5 @@
 GUNICORN_WORKERS=1
 DJANGO_ALLOWED_HOSTS=localhost
 DJANGO_SECRET_KEY=not-a-secret
+PUID=1234
+PGID=1234

tubesync/entrypoint.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-set -x
-
-# Compile SCSS files
-/usr/bin/python3 /app/manage.py compilescss
-
-# Collect the static files
-/usr/bin/python3 /app/manage.py collectstatic --no-input --link
-
-# Run migrations
-/usr/bin/python3 /app/manage.py migrate
-
-# Run what's in CMD
-exec "$@"
-
-# eof

tubesync/tubesync/gunicorn.py

@@ -3,9 +3,11 @@ import multiprocessing
 
 
 def get_num_workers():
+    # Sane max workers to allow to be spawned
     cpu_workers = multiprocessing.cpu_count() * 2 + 1
+    # But default to 3
     try:
-        num_workers = int(os.getenv('GUNICORN_WORKERS', 2))
+        num_workers = int(os.getenv('GUNICORN_WORKERS', 3))
     except ValueError:
         num_workers = cpu_workers
     if 0 > num_workers > cpu_workers:
@@ -14,7 +16,7 @@ def get_num_workers():
 
 
 def get_bind():
-    host = os.getenv('LISTEN_HOST', '0.0.0.0')
+    host = os.getenv('LISTEN_HOST', '127.0.0.1')
     port = os.getenv('LISTEN_PORT', '8080')
     return '{}:{}'.format(host, port)
 
@@ -23,11 +25,11 @@ workers = get_num_workers()
 timeout = 30
 chdir = '/app'
 daemon = False
-pidfile = '/run/www/gunicorn.pid'
+pidfile = '/run/app/gunicorn.pid'
-user = 'www'
+user = 'app'
-group = 'www'
+group = 'app'
 loglevel = 'info'
 errorlog = '-'
-accesslog = '-'
+accesslog = '/dev/null'  # Access logs are printed to stdout from nginx
 django_settings = 'django.settings'
 bind = get_bind()

tubesync/tubesync/local_settings.py.container

@@ -1,6 +1,6 @@
 import os
 from pathlib import Path
-from 
+from binascii import hexlify
 
 
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -9,20 +9,20 @@ ROOT_DIR = Path('/')
 
 RANDOM_SECRET = hexlify(os.urandom(32)).decode()
 SECRET_KEY = str(os.getenv('DJANGO_SECRET_KEY', RANDOM_SECRET))
-ALLOWED_HOSTS_STR = str(os.getenv('TUBESYNC_HOSTS', 'localhost'))
+ALLOWED_HOSTS_STR = str(os.getenv('TUBESYNC_HOSTS', '127.0.0.1,localhost'))
 ALLOWED_HOSTS = ALLOWED_HOSTS_STR.split(',')
 
 
+TIME_ZONE = os.getenv('TZ', 'UTC')
+
+
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': '/config/db.sqlite3',
+        'NAME': ROOT_DIR / 'config' / 'db.sqlite3',
     }
 }
 
 
-BACKGROUND_TASK_ASYNC_THREADS = int(os.get('TUBESYNC_WORKERS', 2))
-
-
 MEDIA_ROOT = ROOT_DIR / 'config' / 'media'
 DOWNLOAD_ROOT = ROOT_DIR / 'downloads'

tubesync/tubesync/local_settings.py.example

@@ -0,0 +1,19 @@
+from pathlib import Path
+
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+SECRET_KEY = 'example-secret-key'
+DEBUG = False
+
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': BASE_DIR / 'db.sqlite3',
+    }
+}
+
+
+DOWNLOAD_ROOT = BASE_DIR / 'downloads'