From bf27c43bbb66a142cbbb1b2a32e4c4892557548a Mon Sep 17 00:00:00 2001 From: meeb Date: Tue, 24 Nov 2020 19:07:10 +1100 Subject: [PATCH] switch to using uvicorn workers and add simple task --- .dockerignore | 5 +++ Dockerfile | 28 ++++++++------- Pipfile | 4 +++ Pipfile.lock | 74 +++++++++++++++++++++++++++++++++++++++- app/tubesync/asgi.py | 3 +- app/tubesync/settings.py | 4 +++ 6 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..be9b48c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +.gitignore +.github +.gitattributes +README.md diff --git a/Dockerfile b/Dockerfile index 31bc398..398c9f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,14 @@ FROM debian:buster-slim +ARG DEBIAN_FRONTEND="noninteractive" + # Third party software versions -ARG YOUTUBE_DL_VERSION=2020.11.24 -ENV YOUTUBE_DL_EXPECTED_SHA256=7d70f2e2d6b42d7c948a418744cd5c89832d67f4fb36f01f1cf4ea7dc8fe537a -ENV YOUTUBE_DL_TARBALL=https://github.com/ytdl-org/youtube-dl/releases/download/${YOUTUBE_DL_VERSION}/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz -ARG FFMPEG_VERSION=4.3.1 -ENV FFMPEG_EXPECTED_MD5=ee235393ec7778279144ee6cbdd9eb64 -ENV FFMPEG_TARBALL=https://johnvansickle.com/ffmpeg/releases/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz +ARG YOUTUBE_DL_VERSION="2020.11.24" +ENV YOUTUBE_DL_EXPECTED_SHA256="7d70f2e2d6b42d7c948a418744cd5c89832d67f4fb36f01f1cf4ea7dc8fe537a" +ENV YOUTUBE_DL_TARBALL="https://github.com/ytdl-org/youtube-dl/releases/download/${YOUTUBE_DL_VERSION}/youtube-dl-${YOUTUBE_DL_VERSION}.tar.gz" +ARG FFMPEG_VERSION="4.3.1" +ENV FFMPEG_EXPECTED_MD5="ee235393ec7778279144ee6cbdd9eb64" +ENV FFMPEG_TARBALL="https://johnvansickle.com/ffmpeg/releases/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz" # Install third party software RUN set -x && \ @@ -33,8 +35,8 @@ RUN set -x && \ apt-get -y autoremove --purge curl xz-utils binutils # Defaults -ARG default_uid=10000 -ARG default_gid=10000 +ARG default_uid="10000" +ARG default_gid="10000" # Copy app COPY app /app @@ -48,11 +50,11 @@ COPY Pipfile.lock /app/Pipfile.lock WORKDIR /app # Set up the app -ENV UID=$default_uid -ENV GID=$default_gid +ENV UID="${default_uid}" +ENV GID="${default_gid}" RUN set -x && \ # Install required distro packages - apt-get -y --no-install-recommends install python3-pip python3-dev gcc && \ + apt-get -y --no-install-recommends install python3-pip python3-dev gcc make && \ # Install wheel which is required for pipenv pip3 --disable-pip-version-check install wheel && \ # Then install pipenv @@ -87,7 +89,7 @@ RUN set -x && \ rm /app/Pipfile.lock && \ pipenv --clear && \ pip3 --disable-pip-version-check uninstall -y pipenv wheel virtualenv && \ - apt-get -y autoremove --purge python3-pip python3-dev gcc && \ + apt-get -y autoremove --purge python3-pip python3-dev gcc make && \ apt-get -y autoremove && \ apt-get -y autoclean && \ rm -rf /var/lib/apt/lists/* && \ @@ -113,4 +115,4 @@ EXPOSE 8080 ENTRYPOINT ["/app/entrypoint.sh"] # Run gunicorn -CMD ["/usr/local/bin/gunicorn", "-c", "/app/tubesync/gunicorn.py", "--capture-output", "tubesync.wsgi:application"] +CMD ["/usr/local/bin/gunicorn", "-c", "/app/tubesync/gunicorn.py", "-k", "uvicorn.workers.UvicornWorker", "--capture-output", "tubesync.asgi:application"] diff --git a/Pipfile b/Pipfile index 509f223..ae9f5a1 100644 --- a/Pipfile +++ b/Pipfile @@ -13,6 +13,10 @@ pillow = "*" whitenoise = "*" gunicorn = "*" django-compressor = "*" +uvicorn = "*" +uvloop = "*" +httptools = "*" +django-simple-task = "*" [requires] python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index 32bd3b0..0995e82 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "09885662bbf2e7551756340f1f90f4245bdaa198e009a0abc5eb43c095952bab" + "sha256": "10bcf36ee023c01949edbe0dbe22eefbe603580488e3498cc5f211f3d1b221ed" }, "pipfile-spec": 6, "requires": { @@ -23,6 +23,13 @@ ], "version": "==3.3.1" }, + "click": { + "hashes": [ + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "version": "==7.1.2" + }, "django": { "hashes": [ "sha256:14a4b7cd77297fba516fc0d92444cc2e2e388aa9de32d7a68d4a83d58f5a4927", @@ -53,6 +60,14 @@ "index": "pypi", "version": "==0.8.2" }, + "django-simple-task": { + "hashes": [ + "sha256:c70c5fd1db9ba58026ea3a00831c2c1a4d335f29c106321bb2dce643b7a71319", + "sha256:fe6494f5297711f5ec767835163dad3e4f6b751aa8857f0c338e5271a0721a4f" + ], + "index": "pypi", + "version": "==0.1.2" + }, "gunicorn": { "hashes": [ "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626", @@ -61,6 +76,31 @@ "index": "pypi", "version": "==20.0.4" }, + "h11": { + "hashes": [ + "sha256:3c6c61d69c6f13d41f1b80ab0322f1872702a3ba26e12aa864c928f6a43fbaab", + "sha256:ab6c335e1b6ef34b205d5ca3e228c9299cc7218b049819ec84a388c2525e5d87" + ], + "version": "==0.11.0" + }, + "httptools": { + "hashes": [ + "sha256:0a4b1b2012b28e68306575ad14ad5e9120b34fccd02a81eb08838d7e3bbb48be", + "sha256:3592e854424ec94bd17dc3e0c96a64e459ec4147e6d53c0a42d0ebcef9cb9c5d", + "sha256:41b573cf33f64a8f8f3400d0a7faf48e1888582b6f6e02b82b9bd4f0bf7497ce", + "sha256:56b6393c6ac7abe632f2294da53f30d279130a92e8ae39d8d14ee2e1b05ad1f2", + "sha256:86c6acd66765a934e8730bf0e9dfaac6fdcf2a4334212bd4a0a1c78f16475ca6", + "sha256:96da81e1992be8ac2fd5597bf0283d832287e20cb3cfde8996d2b00356d4e17f", + "sha256:96eb359252aeed57ea5c7b3d79839aaa0382c9d3149f7d24dd7172b1bcecb009", + "sha256:a2719e1d7a84bb131c4f1e0cb79705034b48de6ae486eb5297a139d6a3296dce", + "sha256:ac0aa11e99454b6a66989aa2d44bca41d4e0f968e395a0a8f164b401fefe359a", + "sha256:bc3114b9edbca5a1eb7ae7db698c669eb53eb8afbbebdde116c174925260849c", + "sha256:fa3cd71e31436911a44620473e873a256851e1f53dee56669dae403ba41756a4", + "sha256:fea04e126014169384dee76a153d4573d90d0cbd1d12185da089f73c78390437" + ], + "index": "pypi", + "version": "==0.1.1" + }, "libsass": { "hashes": [ "sha256:1521d2a8d4b397c6ec90640a1f6b5529077035efc48ef1c2e53095544e713d1b", @@ -159,6 +199,38 @@ ], "version": "==0.4.1" }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "markers": "python_version < '3.8'", + "version": "==3.7.4.3" + }, + "uvicorn": { + "hashes": [ + "sha256:562ef6aaa8fa723ab6b82cf9e67a774088179d0ec57cb17e447b15d58b603bcf", + "sha256:5836edaf4d278fe67ba0298c0537bdb6398cf359eb644f79e6500ca1aad232b3" + ], + "index": "pypi", + "version": "==0.12.3" + }, + "uvloop": { + "hashes": [ + "sha256:08b109f0213af392150e2fe6f81d33261bb5ce968a288eb698aad4f46eb711bd", + "sha256:123ac9c0c7dd71464f58f1b4ee0bbd81285d96cdda8bc3519281b8973e3a461e", + "sha256:4315d2ec3ca393dd5bc0b0089d23101276778c304d42faff5dc4579cb6caef09", + "sha256:4544dcf77d74f3a84f03dd6278174575c44c67d7165d4c42c71db3fdc3860726", + "sha256:afd5513c0ae414ec71d24f6f123614a80f3d27ca655a4fcf6cabe50994cc1891", + "sha256:b4f591aa4b3fa7f32fb51e2ee9fea1b495eb75b0b3c8d0ca52514ad675ae63f7", + "sha256:bcac356d62edd330080aed082e78d4b580ff260a677508718f88016333e2c9c5", + "sha256:e7514d7a48c063226b7d06617cbb12a14278d4323a065a8d46a7962686ce2e95", + "sha256:f07909cd9fc08c52d294b1570bba92186181ca01fe3dc9ffba68955273dd7362" + ], + "index": "pypi", + "version": "==0.14.0" + }, "whitenoise": { "hashes": [ "sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7", diff --git a/app/tubesync/asgi.py b/app/tubesync/asgi.py index 1fd3c87..d6443f9 100644 --- a/app/tubesync/asgi.py +++ b/app/tubesync/asgi.py @@ -1,6 +1,7 @@ import os from django.core.asgi import get_asgi_application +from django_simple_task import django_simple_task_middlware os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tubesync.settings') -application = get_asgi_application() +application = django_simple_task_middlware(get_asgi_application()) diff --git a/app/tubesync/settings.py b/app/tubesync/settings.py index b6d006c..79fa22c 100644 --- a/app/tubesync/settings.py +++ b/app/tubesync/settings.py @@ -18,6 +18,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'sass_processor', + 'django_simple_task', 'common', 'sync', ] @@ -110,6 +111,9 @@ HEALTHCHECK_FIREWALL = True HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',) +DJANGO_SIMPLE_TASK_WORKERS = 2 + + try: from .local_settings import * except ImportError as e: