108 Commits

Author SHA1 Message Date
meeb
e8d75a79c5 fix release build 2021-09-20 17:27:39 +10:00
meeb
ff4be7cfa0 fix release build 2021-09-20 17:14:59 +10:00
meeb
c1cb19259e bump to 0.10.0 2021-09-20 17:08:22 +10:00
meeb
837b6c3107 switch to yt-dlp with required library updates, resolves #145 2021-09-20 16:49:03 +10:00
meeb
ced6314a62 bump libs 2021-09-20 16:25:01 +10:00
meeb
bb6c195ae7 bump libs 2021-09-20 00:18:52 +10:00
meeb
c280b76777 bump libs 2021-09-11 14:58:35 +10:00
meeb
248da767b0 bump libs 2021-09-08 15:19:02 +10:00
meeb
1069b87295 remove outdated image version from readme 2021-09-04 20:09:39 +10:00
meeb
3525a65cd6 add webp support, part of Pillow lib upgrade, discussed in #150 2021-09-04 19:06:23 +10:00
meeb
c51a5bb365 bump libs 2021-09-04 19:00:31 +10:00
meeb
7f4b9aff14 add libjpeg to container build, resolves #149 2021-09-04 17:13:21 +10:00
meeb
a59e7fe65f remove broken playlist_index support in media formatting options, resolves #142 2021-09-04 15:22:45 +10:00
meeb
3e0a71f2ef add cli commands to list and delete sources, useful for deleting massive sources that currently time out as discussed in #148 2021-09-04 15:00:23 +10:00
meeb
3dfbca2af4 update container build to support pillow>=3, add webp image support 2021-09-03 14:46:13 +10:00
meeb
0c256f59d8 bump libs 2021-09-03 14:32:31 +10:00
meeb
dbbae72c25 add yt-dlp to build 2021-09-03 14:31:17 +10:00
meeb
b1b852d82c add check for stale PID file or old processes to gracefully handle hard container stop/starts, discussed in #144 2021-08-27 17:13:25 +10:00
meeb
437bb17f75 bump libs 2021-08-27 04:06:29 +10:00
meeb
fdfcb5fd33 bump libs 2021-08-14 01:16:21 +10:00
meeb
ff35f791f6 handle media having no metadata at all gracefully, resolves #138 2021-08-07 03:01:44 +10:00
meeb
b2ea37ffec bump libs 2021-08-03 15:53:41 +10:00
meeb
d89530d5b8 Merge pull request #136 from hangrymuppet/patch-1
Add libpq5 to docker image
2021-08-03 15:45:56 +10:00
H Bajaj
f00050008b Add libpq5 to docker image 2021-08-02 22:29:10 -07:00
meeb
68604d19c7 bump libs 2021-07-22 18:09:27 +10:00
meeb
55e5b5632f bump libs 2021-07-11 17:11:17 +10:00
meeb
5e18cb92dd bump libs 2021-07-04 16:05:55 +10:00
meeb
6178e0baa0 bump libs 2021-06-28 17:39:34 +10:00
meeb
8050bac507 add libpq-dev to container build 2021-06-19 15:36:39 +10:00
meeb
6dcdac1647 add postgresql-common to container build for new postgresql-binary 2021-06-19 15:30:59 +10:00
meeb
763f6b89ef bump libs 2021-06-19 15:22:19 +10:00
meeb
6c28292918 bump youtube-dl 2021-06-11 16:21:24 +10:00
meeb
574fc55a5e bump django 2021-06-04 14:03:39 +10:00
meeb
c8fd74b3a4 Merge branch 'main' of github.com:meeb/tubesync into main 2021-06-02 15:13:00 +10:00
meeb
6622e17a5a bump libs 2021-06-02 15:12:50 +10:00
meeb
ea05bd0b13 Merge pull request #125 from Bad3r/main
README.md - Line: 15 - Spelling Error
2021-06-02 15:08:30 +10:00
Bad3r
019c98dc76 Spelling Error
implemenations --> implementations
2021-06-01 18:26:08 -04:00
meeb
72dfe51a46 use env vars in image names and tags 2021-05-27 20:49:48 +10:00
meeb
22cebba8ac remove explicit Dockerfile path 2021-05-27 20:33:52 +10:00
meeb
d51d198f94 remove $ in yaml docker tags 2021-05-27 20:28:50 +10:00
meeb
ed0c2d7dd3 switch to actions v2 format 2021-05-27 20:26:11 +10:00
meeb
5ced901ae8 streamline multi-arch build, part of #81 2021-05-27 20:12:02 +10:00
meeb
afda481046 custom builder just building amd64 for now pending multi-arch ffmpeg releases being available, part of #81 2021-05-27 20:01:29 +10:00
meeb
a986864f77 add custom docker builder, part of #81 2021-05-27 19:56:21 +10:00
meeb
ad1c4ecbc9 add multi-platform container image builds to latest image build process, part of #81 2021-05-27 19:49:41 +10:00
meeb
54b7de4442 add multi-platform container image builds to ci process, part of #81 2021-05-27 19:48:11 +10:00
meeb
d1996aee80 add addmetadata as a default youtube-dl flag, part of #111 2021-05-27 19:22:56 +10:00
meeb
326cefbec1 add hack to force database connection to stay alive by hooking into DatabaseWrapper.ensure_connection, resolves #121 2021-05-27 19:12:29 +10:00
meeb
d6e81c6af7 tweak can_download evaluation logic so post-metadata checks trigger a save if required, part of #108 2021-05-27 18:57:43 +10:00
meeb
a000f8f2c0 update libs 2021-05-27 18:43:24 +10:00
meeb
cbab09e931 bump libs 2021-05-22 03:01:29 +10:00
meeb
414fca08ca bump libs 2021-05-21 01:44:27 +10:00
meeb
874c71b7aa add mysql utf8mb4 encoding note to docs, resolves #117 2021-05-17 15:28:43 +10:00
meeb
5b101825f5 update tests 2021-05-17 01:30:04 +10:00
meeb
0db8db4351 force mysql to use utf8mb4 for connections, related to #117 2021-05-17 01:27:50 +10:00
meeb
d4fd148089 update django 2021-05-15 15:00:20 +10:00
meeb
c739d594d8 update libs 2021-05-06 23:10:45 +10:00
meeb
05e8ad8e89 update libs 2021-05-05 15:49:59 +10:00
meeb
024ab72e5f update libs 2021-05-04 14:25:09 +10:00
meeb
66ec3a29ec update tests to match changes in #115 2021-04-30 14:03:36 +10:00
meeb
28a565737f set database connections CONN_MAX_AGE to a default of 300, resolves #115 2021-04-30 14:00:04 +10:00
meeb
2c7116f6ba fix AutoField warning in django >= 3.2, part of #115 2021-04-30 13:55:33 +10:00
meeb
9ccb9db6de add missing libmariadb3 package, update gunicorn package hash, resolves #113 2021-04-28 23:20:21 +10:00
meeb
2d992cbb90 update libs 2021-04-26 13:17:05 +10:00
meeb
302a3614cf update libs 2021-04-25 13:32:48 +10:00
meeb
ea546013de bump libs 2021-04-21 15:43:08 +10:00
meeb
fb18610893 bump libs 2021-04-07 21:00:38 +10:00
meeb
2364432088 bump libs 2021-04-06 11:31:36 +10:00
meeb
655bed14fd fix loaddata import example in docs 2021-04-05 23:59:48 +10:00
meeb
721399f665 fix container build deps for python mysql client, related to #72 2021-04-05 00:31:50 +10:00
meeb
694ed5c581 update example database settings 2021-04-04 23:09:05 +10:00
meeb
a98f2462ed tweak docs 2021-04-04 23:06:44 +10:00
meeb
5461a5357d check dicts output of parse_database_connection_string() in tests, part of #72 2021-04-04 23:04:51 +10:00
meeb
20df9f4044 support external postgresql, mysql and mariadb databases, resolves #72 2021-04-04 23:01:15 +10:00
meeb
3ec4f7c525 fix test running in Makefile 2021-04-04 22:16:22 +10:00
meeb
443fb827d0 update libs 2021-04-02 21:28:14 +11:00
meeb
a810303f52 bump libs 2021-03-30 11:54:01 +11:00
meeb
9370a481f9 Merge pull request #100 from micahmo/main
Related to #88: Fix 4320p in test data.
2021-03-18 12:19:45 +11:00
Micah Morrison
1478c95d59 Related to #88: Fix 4320p in test data.
RESOLUTION_MAP expects lowercase p in resolution name.
2021-03-17 17:19:15 -04:00
meeb
f69fa747af fix download cap date comparison check in signal, resolves #97 2021-03-15 01:13:16 +11:00
meeb
a29a92893f Merge pull request #95 from micahmo/main
Fix meeb/tubesync#90: Match media on source and id
2021-03-13 23:05:45 +11:00
Micah Morrison
7d471056c1 Fix meeb/tubesync#90: Match media on source and id 2021-03-11 20:37:44 -05:00
meeb
119493c181 disable warning on skipping an already skipped video with no publish date, related to #77 2021-03-08 13:26:34 +11:00
meeb
02a0f924b4 typo 2021-03-08 13:08:38 +11:00
meeb
38665eb00d add a secondary check when download tasks are triggered for download caps, related to #77 2021-03-08 12:20:44 +11:00
meeb
c32358bcef media requires a published date, update tests to match, related to #77 2021-03-08 12:08:10 +11:00
meeb
df9316bede handle media which may not have a published date set for some erroneous reason, related to #77 2021-03-08 11:55:23 +11:00
meeb
8525d920a0 update ffmpeg to 4.3.2, resolves #75 2021-03-07 17:38:35 +11:00
meeb
a6e08d9a10 update s6-init to 2.2.0.3, resolves #76 2021-03-07 17:12:40 +11:00
meeb
2e0d0385b0 recalculate media skip flag when source download cap is updated, resolves #77 2021-03-07 14:03:19 +11:00
meeb
972c184c70 add longer source indexing options and a never option, resolves #68 2021-03-07 13:39:49 +11:00
meeb
adeafbfcb4 update libs 2021-03-07 13:10:10 +11:00
meeb
2c1c45e829 bump to 0.9.1 2021-03-05 14:20:23 +11:00
meeb
c64f54bcb4 bump to 0.9.1 2021-03-05 14:16:17 +11:00
meeb
6ce55b0337 bump libs 2021-03-04 15:39:51 +11:00
meeb
d06c4beae0 bump upstream libs 2021-02-22 19:22:30 +11:00
meeb
db651e16b9 raise a manual exception when youtube-dl extract_info returns no data to trigger backoffs, increase backoff retry and timers, resolves #66 2021-02-22 13:24:11 +11:00
meeb
86068790ed missing import 2021-02-21 11:52:14 +11:00
meeb
ea72671351 add version details for http basic auth instructions 2021-02-21 11:45:56 +11:00
meeb
96b9eddf43 add a reset tasks cli command 2021-02-21 11:44:52 +11:00
meeb
bceefc8b01 skip media which has no publish date in locally stored metadata 2021-02-21 11:15:57 +11:00
meeb
820cc69937 typo 2021-02-19 14:40:41 +11:00
meeb
1e8711be51 add media items downloaded counter to sources list overview 2021-02-19 14:37:56 +11:00
meeb
e3423bc2d2 preserve media filter when toggling skipped media 2021-02-19 14:26:01 +11:00
meeb
6fbf72d0e7 optional basic HTTP authentication, resolves #62 2021-02-19 12:58:34 +11:00
meeb
d6852bf828 account for metadata loading as None for upload_date, resolves #59 2021-02-18 20:50:13 +11:00
meeb
f6f4f244d7 hide skipped media by default and add a show skipped media button 2021-02-18 19:34:44 +11:00
meeb
df35aa2a5f increase media per page, tweak pagination button layout 2021-02-18 19:21:41 +11:00
37 changed files with 1074 additions and 253 deletions

View File

@@ -35,13 +35,19 @@ jobs:
containerise: containerise:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - name: Set up QEMU
- name: Build the container image uses: docker/setup-qemu-action@v1
run: docker build . --tag $IMAGE_NAME - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Log into GitHub Container Registry - name: Log into GitHub Container Registry
run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
- name: Push image to GitHub Container Registry - name: Build and push
run: | uses: docker/build-push-action@v2
LATEST_TAG=ghcr.io/meeb/$IMAGE_NAME:latest with:
docker tag $IMAGE_NAME $LATEST_TAG platforms: linux/amd64
docker push $LATEST_TAG push: true
tags: ghcr.io/meeb/${{ env.IMAGE_NAME }}:latest
cache-from: type=registry,ref=ghcr.io/meeb/${{ env.IMAGE_NAME }}:latest
cache-to: type=inline
build-args: |
IMAGE_NAME=${{ env.IMAGE_NAME }}

View File

@@ -11,18 +11,23 @@ jobs:
containerise: containerise:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Get tag - name: Get tag
id: vars id: tag
run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} uses: dawidd6/action-get-tag@v1
- name: Build the container image - uses: docker/build-push-action@v2
run: docker build . --tag $IMAGE_NAME - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Log into GitHub Container Registry - name: Log into GitHub Container Registry
run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
- name: Push image to GitHub Container Registry - name: Build and push
env: uses: docker/build-push-action@v2
RELEASE_TAG: ${{ steps.vars.outputs.tag }} with:
run: | platforms: linux/amd64
REF_TAG=ghcr.io/meeb/$IMAGE_NAME:$RELEASE_TAG push: true
docker tag $IMAGE_NAME $REF_TAG tags: ghcr.io/meeb/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
docker push $REF_TAG cache-from: type=registry,ref=ghcr.io/meeb/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
cache-to: type=inline
build-args: |
IMAGE_NAME=${{ env.IMAGE_NAME }}

View File

@@ -1,8 +1,8 @@
FROM debian:buster-slim FROM debian:buster-slim
ARG ARCH="amd64" ARG ARCH="amd64"
ARG S6_VERSION="2.1.0.2" ARG S6_VERSION="2.2.0.3"
ARG FFMPEG_VERSION="4.3.1" ARG FFMPEG_VERSION="4.3.2"
ENV DEBIAN_FRONTEND="noninteractive" \ ENV DEBIAN_FRONTEND="noninteractive" \
HOME="/root" \ HOME="/root" \
@@ -10,9 +10,9 @@ ENV DEBIAN_FRONTEND="noninteractive" \
LANG="en_US.UTF-8" \ LANG="en_US.UTF-8" \
LC_ALL="en_US.UTF-8" \ LC_ALL="en_US.UTF-8" \
TERM="xterm" \ TERM="xterm" \
S6_EXPECTED_SHA256="52460473413601ff7a84ae690b161a074217ddc734990c2cdee9847166cf669e" \ S6_EXPECTED_SHA256="a7076cf205b331e9f8479bbb09d9df77dbb5cd8f7d12e9b74920902e0c16dd98" \
S6_DOWNLOAD="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${ARCH}.tar.gz" \ S6_DOWNLOAD="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${ARCH}.tar.gz" \
FFMPEG_EXPECTED_SHA256="47d95c0129fba27d051748a442a44a73ce1bd38d1e3f9fe1e9dd7258c7581fa5" \ FFMPEG_EXPECTED_SHA256="34bffcd0b58695e3ee5eba2573b37f06cb5088050733ca96265815f58bd61d35" \
FFMPEG_DOWNLOAD="https://tubesync.sfo2.digitaloceanspaces.com/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static.tar.xz" FFMPEG_DOWNLOAD="https://tubesync.sfo2.digitaloceanspaces.com/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static.tar.xz"
@@ -57,11 +57,28 @@ WORKDIR /app
# Set up the app # Set up the app
RUN set -x && \ RUN set -x && \
apt-get update && \
# Install required distro packages # Install required distro packages
apt-get -y install nginx-light && \ apt-get -y install nginx-light && \
apt-get -y --no-install-recommends install python3 python3-setuptools python3-pip python3-dev gcc make && \ apt-get -y --no-install-recommends install \
python3 \
python3-setuptools \
python3-pip \
python3-dev \
gcc \
make \
default-libmysqlclient-dev \
libmariadb3 \
postgresql-common \
libpq-dev \
libpq5 \
libjpeg62-turbo \
libwebp6 \
libjpeg-dev \
zlib1g-dev \
libwebp-dev && \
# Install pipenv # Install pipenv
pip3 --disable-pip-version-check install pipenv && \ pip3 --disable-pip-version-check install wheel pipenv && \
# Create a 'app' user which the application will run as # Create a 'app' user which the application will run as
groupadd app && \ groupadd app && \
useradd -M -d /app -s /bin/false -g app app && \ useradd -M -d /app -s /bin/false -g app app && \
@@ -82,7 +99,17 @@ RUN set -x && \
rm /app/Pipfile.lock && \ rm /app/Pipfile.lock && \
pipenv --clear && \ pipenv --clear && \
pip3 --disable-pip-version-check uninstall -y pipenv wheel virtualenv && \ pip3 --disable-pip-version-check uninstall -y pipenv wheel virtualenv && \
apt-get -y autoremove --purge python3-pip python3-dev gcc make && \ apt-get -y autoremove --purge \
python3-pip \
python3-dev \
gcc \
make \
default-libmysqlclient-dev \
postgresql-common \
libpq-dev \
libjpeg-dev \
zlib1g-dev \
libwebp-dev && \
apt-get -y autoremove && \ apt-get -y autoremove && \
apt-get -y autoclean && \ apt-get -y autoclean && \
rm -rf /var/lib/apt/lists/* && \ rm -rf /var/lib/apt/lists/* && \

View File

@@ -30,4 +30,4 @@ runcontainer:
test: test:
$(python) app/manage.py test --verbosity=2 cd tubesync && $(python) manage.py test --verbosity=2 && cd ..

View File

@@ -14,9 +14,12 @@ whitenoise = "*"
gunicorn = "*" gunicorn = "*"
django-compressor = "*" django-compressor = "*"
httptools = "*" httptools = "*"
youtube-dl = "*"
django-background-tasks = "*" django-background-tasks = "*"
requests = "*" requests = "*"
django-basicauth = "*"
psycopg2-binary = "*"
mysqlclient = "*"
yt-dlp = "*"
[requires] [requires]
python_version = "3" python_version = "3"

373
Pipfile.lock generated
View File

@@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "a4bb556fc61ee4583f9588980450b071814298ee4d1a1023fad149c14d14aaba" "sha256": "ac12e45a1719945b2e19d4a12b03136225f1f5e81affd1adf44a7b3c8dd36b8a"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@@ -18,32 +18,34 @@
"default": { "default": {
"asgiref": { "asgiref": {
"hashes": [ "hashes": [
"sha256:5ee950735509d04eb673bd7f7120f8fa1c9e2df495394992c73234d526907e17", "sha256:4ef1ab46b484e3c706329cedeff284a5d40824200638503f5768edb6de7d58e9",
"sha256:7162a3cb30ab0609f1a4c95938fd73e8604f63bdba516a7f7d64b83ff09478f0" "sha256:ffc141aa908e6f175673e7b1b3b7af4fdb0ecb738fc5c8b88f69f055c2415214"
], ],
"version": "==3.3.1" "markers": "python_version >= '3.6'",
"version": "==3.4.1"
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
"sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
"sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
], ],
"version": "==2020.12.5" "version": "==2021.5.30"
}, },
"chardet": { "charset-normalizer": {
"hashes": [ "hashes": [
"sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", "sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6",
"sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" "sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f"
], ],
"version": "==4.0.0" "markers": "python_version >= '3'",
"version": "==2.0.6"
}, },
"django": { "django": {
"hashes": [ "hashes": [
"sha256:169e2e7b4839a7910b393eec127fd7cbae62e80fa55f89c6510426abf673fe5f", "sha256:95b318319d6997bac3595517101ad9cc83fe5672ac498ba48d1a410f47afecd2",
"sha256:c6c0462b8b361f8691171af1fb87eceb4442da28477e12200c40420176206ba7" "sha256:e93c93565005b37ddebf2396b4dc4b6913c1838baa82efdfb79acedd5816c240"
], ],
"index": "pypi", "index": "pypi",
"version": "==3.1.6" "version": "==3.2.7"
}, },
"django-appconf": { "django-appconf": {
"hashes": [ "hashes": [
@@ -59,6 +61,14 @@
"index": "pypi", "index": "pypi",
"version": "==1.2.5" "version": "==1.2.5"
}, },
"django-basicauth": {
"hashes": [
"sha256:15e9e366f698f53c71b1e794dafea060f990a2ac556bae6b7330dd25324a091c",
"sha256:e5e47d1acdc1943bedcc1bf673059d6c15e257dfe9eef67a22fb824f79546c0d"
],
"index": "pypi",
"version": "==0.5.3"
},
"django-compat": { "django-compat": {
"hashes": [ "hashes": [
"sha256:3ac9a3bedc56b9365d9eb241bc5157d0c193769bf995f9a78dc1bc24e7c2331b" "sha256:3ac9a3bedc56b9365d9eb241bc5157d0c193769bf995f9a78dc1bc24e7c2331b"
@@ -67,108 +77,229 @@
}, },
"django-compressor": { "django-compressor": {
"hashes": [ "hashes": [
"sha256:57ac0a696d061e5fc6fbc55381d2050f353b973fb97eee5593f39247bc0f30af", "sha256:3358077605c146fdcca5f9eaffb50aa5dbe15f238f8854679115ebf31c0415e0",
"sha256:d2ed1c6137ddaac5536233ec0a819e14009553fee0a869bea65d03e5285ba74f" "sha256:f8313f59d5e65712fc28787d084fe834997c9dfa92d064a1a3ec3d3366594d04"
], ],
"index": "pypi", "index": "pypi",
"version": "==2.4" "version": "==2.4.1"
}, },
"django-sass-processor": { "django-sass-processor": {
"hashes": [ "hashes": [
"sha256:9b46a12ca8bdcb397d46fbcc49e6a926ff9f76a93c5efeb23b495419fd01fc7a" "sha256:1f043180c47754018e803a77da003377f5ea6558de57cd6946eb27a32e9c16a2",
"sha256:dcaad47c591a2d52689c1bd209259e922e902d886293f0d5c9e0d1a4eb85eda2"
], ],
"index": "pypi", "index": "pypi",
"version": "==0.8.2" "version": "==1.0.1"
}, },
"gunicorn": { "gunicorn": {
"hashes": [ "hashes": [
"sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626", "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e",
"sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c" "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"
], ],
"index": "pypi", "index": "pypi",
"version": "==20.0.4" "version": "==20.1.0"
}, },
"httptools": { "httptools": {
"hashes": [ "hashes": [
"sha256:0a4b1b2012b28e68306575ad14ad5e9120b34fccd02a81eb08838d7e3bbb48be", "sha256:04114db99605c9b56ea22a8ec4d7b1485b908128ed4f4a8f6438489c428da794",
"sha256:3592e854424ec94bd17dc3e0c96a64e459ec4147e6d53c0a42d0ebcef9cb9c5d", "sha256:074afd8afdeec0fa6786cd4a1676e0c0be23dc9a017a86647efa6b695168104f",
"sha256:41b573cf33f64a8f8f3400d0a7faf48e1888582b6f6e02b82b9bd4f0bf7497ce", "sha256:113816f9af7dcfc4aa71ebb5354d77365f666ecf96ac7ff2aa1d24b6bca44165",
"sha256:56b6393c6ac7abe632f2294da53f30d279130a92e8ae39d8d14ee2e1b05ad1f2", "sha256:1a8f26327023fa1a947d36e60a0582149e182fbbc949c8a65ec8665754dbbe69",
"sha256:86c6acd66765a934e8730bf0e9dfaac6fdcf2a4334212bd4a0a1c78f16475ca6", "sha256:2119fa619a4c53311f594f25c0205d619350fcb32140ec5057f861952e9b2b4f",
"sha256:96da81e1992be8ac2fd5597bf0283d832287e20cb3cfde8996d2b00356d4e17f", "sha256:21e948034f70e47c8abfa2d5e6f1a5661f87a2cddc7bcc70f61579cc87897c70",
"sha256:96eb359252aeed57ea5c7b3d79839aaa0382c9d3149f7d24dd7172b1bcecb009", "sha256:32a10a5903b5bc0eb647d01cd1e95bec3bb614a9bf53f0af1e01360b2debdf81",
"sha256:a2719e1d7a84bb131c4f1e0cb79705034b48de6ae486eb5297a139d6a3296dce", "sha256:3787c1f46e9722ef7f07ea5c76b0103037483d1b12e34a02c53ceca5afa4e09a",
"sha256:ac0aa11e99454b6a66989aa2d44bca41d4e0f968e395a0a8f164b401fefe359a", "sha256:3f82eb106e1474c63dba36a176067e65b48385f4cecddf3616411aa5d1fbdfec",
"sha256:bc3114b9edbca5a1eb7ae7db698c669eb53eb8afbbebdde116c174925260849c", "sha256:3f9b4856d46ba1f0c850f4e84b264a9a8b4460acb20e865ec00978ad9fbaa4cf",
"sha256:fa3cd71e31436911a44620473e873a256851e1f53dee56669dae403ba41756a4", "sha256:4137137de8976511a392e27bfdcf231bd926ac13d375e0414e927b08217d779e",
"sha256:fea04e126014169384dee76a153d4573d90d0cbd1d12185da089f73c78390437" "sha256:4687dfc116a9f1eb22a7d797f0dc6f6e17190d406ca4e729634b38aa98044b17",
"sha256:47dba2345aaa01b87e4981e8756af441349340708d5b60712c98c55a4d28f4af",
"sha256:5a836bd85ae1fb4304f674808488dae403e136d274aa5bafd0e6ee456f11c371",
"sha256:6e676bc3bb911b11f3d7e2144b9a53600bf6b9b21e0e4437aa308e1eef094d97",
"sha256:72ee0e3fb9c6437ab3ae34e9abee67fcee6876f4f58504e3f613dd5882aafdb7",
"sha256:79717080dc3f8b1eeb7f820b9b81528acbc04be6041f323fdd97550da2062575",
"sha256:8ac842df4fc3952efa7820b277961ea55e068bbc54cb59a0820400de7ae358d8",
"sha256:9f475b642c48b1b78584bdd12a5143e2c512485664331eade9c29ef769a17598",
"sha256:b8ac7dee63af4346e02b1e6d32202e3b5b3706a9928bec6da6d7a5b066217422",
"sha256:c0ac2e0ce6733c55858932e7d37fcc7b67ba6bb23e9648593c55f663de031b93",
"sha256:c14576b737d9e6e4f2a86af04918dbe9b62f57ce8102a8695c9a382dbe405c7f",
"sha256:cdc3975db86c29817e6d13df14e037c931fc893a710fb71097777a4147090068",
"sha256:eda95634027200f4b2a6d499e7c2e7fa9b8ee57e045dfda26958ea0af27c070b"
], ],
"index": "pypi", "index": "pypi",
"version": "==0.1.1" "version": "==0.3.0"
}, },
"idna": { "idna": {
"hashes": [ "hashes": [
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a",
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"
], ],
"version": "==2.10" "markers": "python_version >= '3'",
"version": "==3.2"
}, },
"libsass": { "libsass": {
"hashes": [ "hashes": [
"sha256:1521d2a8d4b397c6ec90640a1f6b5529077035efc48ef1c2e53095544e713d1b", "sha256:06c8776417fe930714bdc930a3d7e795ae3d72be6ac883ff72a1b8f7c49e5ffb",
"sha256:1b2d415bbf6fa7da33ef46e549db1418498267b459978eff8357e5e823962d35", "sha256:12f39712de38689a8b785b7db41d3ba2ea1d46f9379d81ea4595802d91fa6529",
"sha256:25ebc2085f5eee574761ccc8d9cd29a9b436fc970546d5ef08c6fa41eb57dff1", "sha256:1e25dd9047a9392d3c59a0b869e0404f2b325a03871ee45285ee33b3664f5613",
"sha256:2ae806427b28bc1bb7cb0258666d854fcf92ba52a04656b0b17ba5e190fb48a9", "sha256:659ae41af8708681fa3ec73f47b9735a6725e71c3b66ff570bfce78952f2314e",
"sha256:4a246e4b88fd279abef8b669206228c92534d96ddcd0770d7012088c408dff23", "sha256:6b984510ed94993708c0d697b4fef2d118929bbfffc3b90037be0f5ccadf55e7",
"sha256:553e5096414a8d4fb48d0a48f5a038d3411abe254d79deac5e008516c019e63a", "sha256:a005f298f64624f313a3ac618ab03f844c71d84ae4f4a4aec4b68d2a4ffe75eb",
"sha256:697f0f9fa8a1367ca9ec6869437cb235b1c537fc8519983d1d890178614a8903", "sha256:abc29357ee540849faf1383e1746d40d69ed5cb6d4c346df276b258f5aa8977a",
"sha256:a8fd4af9f853e8bf42b1425c5e48dd90b504fa2e70d7dac5ac80b8c0a5a5fe85", "sha256:d5ba529d9ce668be9380563279f3ffe988f27bc5b299c5a28453df2e0b0fbaf2",
"sha256:c9411fec76f480ffbacc97d8188322e02a5abca6fc78e70b86a2a2b421eae8a2", "sha256:e2b1a7d093f2e76dc694c17c0c285e846d0b0deb0e8b21dc852ba1a3a4e2f1d6"
"sha256:daa98a51086d92aa7e9c8871cf1a8258124b90e2abf4697852a3dca619838618",
"sha256:e0e60836eccbf2d9e24ec978a805cd6642fa92515fbd95e3493fee276af76f8a",
"sha256:e64ae2587f1a683e831409aad03ba547c245ef997e1329fffadf7a866d2510b8",
"sha256:f6852828e9e104d2ce0358b73c550d26dd86cc3a69439438c3b618811b9584f5"
], ],
"index": "pypi", "index": "pypi",
"version": "==0.20.1" "version": "==0.21.0"
},
"mutagen": {
"hashes": [
"sha256:6397602efb3c2d7baebd2166ed85731ae1c1d475abca22090b7141ff5034b3e1",
"sha256:9c9f243fcec7f410f138cb12c21c84c64fde4195481a30c9bfb05b5f003adfed"
],
"markers": "python_version >= '3.5' and python_version < '4'",
"version": "==1.45.1"
},
"mysqlclient": {
"hashes": [
"sha256:0ac0dd759c4ca02c35a9fedc24bc982cf75171651e8187c2495ec957a87dfff7",
"sha256:3381ca1a4f37ff1155fcfde20836b46416d66531add8843f6aa6d968982731c3",
"sha256:71c4b330cf2313bbda0307fc858cc9055e64493ba9bf28454d25cf8b3ee8d7f5",
"sha256:f6ebea7c008f155baeefe16c56cd3ee6239f7a5a9ae42396c2f1860f08a7c432",
"sha256:fc575093cf81b6605bed84653e48b277318b880dc9becf42dd47fa11ffd3e2b6"
],
"index": "pypi",
"version": "==2.0.3"
}, },
"pillow": { "pillow": {
"hashes": [ "hashes": [
"sha256:165c88bc9d8dba670110c689e3cc5c71dbe4bfb984ffa7cbebf1fac9554071d6", "sha256:0412516dcc9de9b0a1e0ae25a280015809de8270f134cc2c1e32c4eeb397cf30",
"sha256:1d208e670abfeb41b6143537a681299ef86e92d2a3dac299d3cd6830d5c7bded", "sha256:04835e68ef12904bc3e1fd002b33eea0779320d4346082bd5b24bec12ad9c3e9",
"sha256:22d070ca2e60c99929ef274cfced04294d2368193e935c5d6febfd8b601bf865", "sha256:06d1adaa284696785375fa80a6a8eb309be722cf4ef8949518beb34487a3df71",
"sha256:2353834b2c49b95e1313fb34edf18fca4d57446675d05298bb694bca4b194174", "sha256:085a90a99404b859a4b6c3daa42afde17cb3ad3115e44a75f0d7b4a32f06a6c9",
"sha256:39725acf2d2e9c17356e6835dccebe7a697db55f25a09207e38b835d5e1bc032", "sha256:0b9911ec70731711c3b6ebcde26caea620cbdd9dcb73c67b0730c8817f24711b",
"sha256:3de6b2ee4f78c6b3d89d184ade5d8fa68af0848f9b6b6da2b9ab7943ec46971a", "sha256:10e00f7336780ca7d3653cf3ac26f068fa11b5a96894ea29a64d3dc4b810d630",
"sha256:47c0d93ee9c8b181f353dbead6530b26980fe4f5485aa18be8f1fd3c3cbc685e", "sha256:11c27e74bab423eb3c9232d97553111cc0be81b74b47165f07ebfdd29d825875",
"sha256:5e2fe3bb2363b862671eba632537cd3a823847db4d98be95690b7e382f3d6378", "sha256:11eb7f98165d56042545c9e6db3ce394ed8b45089a67124298f0473b29cb60b2",
"sha256:604815c55fd92e735f9738f65dabf4edc3e79f88541c221d292faec1904a4b17", "sha256:13654b521fb98abdecec105ea3fb5ba863d1548c9b58831dd5105bb3873569f1",
"sha256:6c5275bd82711cd3dcd0af8ce0bb99113ae8911fc2952805f1d012de7d600a4c", "sha256:15ccb81a6ffc57ea0137f9f3ac2737ffa1d11f786244d719639df17476d399a7",
"sha256:731ca5aabe9085160cf68b2dbef95fc1991015bc0a3a6ea46a371ab88f3d0913", "sha256:18a07a683805d32826c09acfce44a90bf474e6a66ce482b1c7fcd3757d588df3",
"sha256:7612520e5e1a371d77e1d1ca3a3ee6227eef00d0a9cddb4ef7ecb0b7396eddf7", "sha256:19ec4cfe4b961edc249b0e04b5618666c23a83bc35842dea2bfd5dfa0157f81b",
"sha256:7916cbc94f1c6b1301ac04510d0881b9e9feb20ae34094d3615a8a7c3db0dcc0", "sha256:1c3ff00110835bdda2b1e2b07f4a2548a39744bb7de5946dc8e95517c4fb2ca6",
"sha256:81c3fa9a75d9f1afafdb916d5995633f319db09bd773cb56b8e39f1e98d90820", "sha256:27a330bf7014ee034046db43ccbb05c766aa9e70b8d6c5260bfc38d73103b0ba",
"sha256:887668e792b7edbfb1d3c9d8b5d8c859269a0f0eba4dda562adb95500f60dbba", "sha256:2b11c9d310a3522b0fd3c35667914271f570576a0e387701f370eb39d45f08a4",
"sha256:93a473b53cc6e0b3ce6bf51b1b95b7b1e7e6084be3a07e40f79b42e83503fbf2", "sha256:2c661542c6f71dfd9dc82d9d29a8386287e82813b0375b3a02983feac69ef864",
"sha256:96d4dc103d1a0fa6d47c6c55a47de5f5dafd5ef0114fa10c85a1fd8e0216284b", "sha256:2cde7a4d3687f21cffdf5bb171172070bb95e02af448c4c8b2f223d783214056",
"sha256:a3d3e086474ef12ef13d42e5f9b7bbf09d39cf6bd4940f982263d6954b13f6a9", "sha256:2d5e9dc0bf1b5d9048a94c48d0813b6c96fccfa4ccf276d9c36308840f40c228",
"sha256:b02a0b9f332086657852b1f7cb380f6a42403a6d9c42a4c34a561aa4530d5234", "sha256:2f23b2d3079522fdf3c09de6517f625f7a964f916c956527bed805ac043799b8",
"sha256:b09e10ec453de97f9a23a5aa5e30b334195e8d2ddd1ce76cc32e52ba63c8b31d", "sha256:35d27687f027ad25a8d0ef45dd5208ef044c588003cdcedf05afb00dbc5c2deb",
"sha256:b6f00ad5ebe846cc91763b1d0c6d30a8042e02b2316e27b05de04fa6ec831ec5", "sha256:35d409030bf3bd05fa66fb5fdedc39c521b397f61ad04309c90444e893d05f7d",
"sha256:bba80df38cfc17f490ec651c73bb37cd896bc2400cfba27d078c2135223c1206", "sha256:4326ea1e2722f3dc00ed77c36d3b5354b8fb7399fb59230249ea6d59cbed90da",
"sha256:c3d911614b008e8a576b8e5303e3db29224b455d3d66d1b2848ba6ca83f9ece9", "sha256:4abc247b31a98f29e5224f2d31ef15f86a71f79c7f4d2ac345a5d551d6393073",
"sha256:ca20739e303254287138234485579b28cb0d524401f83d5129b5ff9d606cb0a8", "sha256:4d89a2e9219a526401015153c0e9dd48319ea6ab9fe3b066a20aa9aee23d9fd3",
"sha256:cb192176b477d49b0a327b2a5a4979552b7a58cd42037034316b8018ac3ebb59", "sha256:4e59e99fd680e2b8b11bbd463f3c9450ab799305d5f2bafb74fefba6ac058616",
"sha256:cdbbe7dff4a677fb555a54f9bc0450f2a21a93c5ba2b44e09e54fcb72d2bd13d", "sha256:548794f99ff52a73a156771a0402f5e1c35285bd981046a502d7e4793e8facaa",
"sha256:cf6e33d92b1526190a1de904df21663c46a456758c0424e4f947ae9aa6088bf7", "sha256:56fd98c8294f57636084f4b076b75f86c57b2a63a8410c0cd172bc93695ee979",
"sha256:d355502dce85ade85a2511b40b4c61a128902f246504f7de29bbeec1ae27933a", "sha256:59697568a0455764a094585b2551fd76bfd6b959c9f92d4bdec9d0e14616303a",
"sha256:d673c4990acd016229a5c1c4ee8a9e6d8f481b27ade5fc3d95938697fa443ce0", "sha256:6bff50ba9891be0a004ef48828e012babaaf7da204d81ab9be37480b9020a82b",
"sha256:dc577f4cfdda354db3ae37a572428a90ffdbe4e51eda7849bf442fb803f09c9b", "sha256:6cb3dd7f23b044b0737317f892d399f9e2f0b3a02b22b2c692851fb8120d82c6",
"sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d", "sha256:7dbfbc0020aa1d9bc1b0b8bcf255a7d73f4ad0336f8fd2533fcc54a4ccfb9441",
"sha256:f50e7a98b0453f39000619d845be8b06e611e56ee6e8186f7f60c3b1e2f0feae" "sha256:838eb85de6d9307c19c655c726f8d13b8b646f144ca6b3771fa62b711ebf7624",
"sha256:8b68f565a4175e12e68ca900af8910e8fe48aaa48fd3ca853494f384e11c8bcd",
"sha256:8f284dc1695caf71a74f24993b7c7473d77bc760be45f776a2c2f4e04c170550",
"sha256:963ebdc5365d748185fdb06daf2ac758116deecb2277ec5ae98139f93844bc09",
"sha256:a048dad5ed6ad1fad338c02c609b862dfaa921fcd065d747194a6805f91f2196",
"sha256:a1bd983c565f92779be456ece2479840ec39d386007cd4ae83382646293d681b",
"sha256:a66566f8a22561fc1a88dc87606c69b84fa9ce724f99522cf922c801ec68f5c1",
"sha256:bcb04ff12e79b28be6c9988f275e7ab69f01cc2ba319fb3114f87817bb7c74b6",
"sha256:bd24054aaf21e70a51e2a2a5ed1183560d3a69e6f9594a4bfe360a46f94eba83",
"sha256:be25cb93442c6d2f8702c599b51184bd3ccd83adebd08886b682173e09ef0c3f",
"sha256:c691b26283c3a31594683217d746f1dad59a7ae1d4cfc24626d7a064a11197d4",
"sha256:cc9d0dec711c914ed500f1d0d3822868760954dce98dfb0b7382a854aee55d19",
"sha256:ce2e5e04bb86da6187f96d7bab3f93a7877830981b37f0287dd6479e27a10341",
"sha256:ce651ca46d0202c302a535d3047c55a0131a720cf554a578fc1b8a2aff0e7d96",
"sha256:d0c8ebbfd439c37624db98f3877d9ed12c137cadd99dde2d2eae0dab0bbfc355",
"sha256:d675a876b295afa114ca8bf42d7f86b5fb1298e1b6bb9a24405a3f6c8338811c",
"sha256:dde3f3ed8d00c72631bc19cbfff8ad3b6215062a5eed402381ad365f82f0c18c",
"sha256:e5a31c07cea5edbaeb4bdba6f2b87db7d3dc0f446f379d907e51cc70ea375629",
"sha256:f514c2717012859ccb349c97862568fdc0479aad85b0270d6b5a6509dbc142e2",
"sha256:fc0db32f7223b094964e71729c0361f93db43664dd1ec86d3df217853cedda87",
"sha256:fd4fd83aa912d7b89b4b4a1580d30e2a4242f3936882a3f433586e5ab97ed0d5",
"sha256:feb5db446e96bfecfec078b943cc07744cc759893cef045aa8b8b6d6aaa8274e"
], ],
"index": "pypi", "index": "pypi",
"version": "==8.1.0" "version": "==8.3.2"
},
"psycopg2-binary": {
"hashes": [
"sha256:0b7dae87f0b729922e06f85f667de7bf16455d411971b2043bbd9577af9d1975",
"sha256:0f2e04bd2a2ab54fa44ee67fe2d002bb90cee1c0f1cc0ebc3148af7b02034cbd",
"sha256:123c3fb684e9abfc47218d3784c7b4c47c8587951ea4dd5bc38b6636ac57f616",
"sha256:1473c0215b0613dd938db54a653f68251a45a78b05f6fc21af4326f40e8360a2",
"sha256:14db1752acdd2187d99cb2ca0a1a6dfe57fc65c3281e0f20e597aac8d2a5bd90",
"sha256:1e3a362790edc0a365385b1ac4cc0acc429a0c0d662d829a50b6ce743ae61b5a",
"sha256:1e85b74cbbb3056e3656f1cc4781294df03383127a8114cbc6531e8b8367bf1e",
"sha256:20f1ab44d8c352074e2d7ca67dc00843067788791be373e67a0911998787ce7d",
"sha256:2f62c207d1740b0bde5c4e949f857b044818f734a3d57f1d0d0edc65050532ed",
"sha256:3242b9619de955ab44581a03a64bdd7d5e470cc4183e8fcadd85ab9d3756ce7a",
"sha256:35c4310f8febe41f442d3c65066ca93cccefd75013df3d8c736c5b93ec288140",
"sha256:4235f9d5ddcab0b8dbd723dca56ea2922b485ea00e1dafacf33b0c7e840b3d32",
"sha256:5ced67f1e34e1a450cdb48eb53ca73b60aa0af21c46b9b35ac3e581cf9f00e31",
"sha256:7360647ea04db2e7dff1648d1da825c8cf68dc5fbd80b8fb5b3ee9f068dcd21a",
"sha256:8c13d72ed6af7fd2c8acbd95661cf9477f94e381fce0792c04981a8283b52917",
"sha256:988b47ac70d204aed01589ed342303da7c4d84b56c2f4c4b8b00deda123372bf",
"sha256:995fc41ebda5a7a663a254a1dcac52638c3e847f48307b5416ee373da15075d7",
"sha256:a36c7eb6152ba5467fb264d73844877be8b0847874d4822b7cf2d3c0cb8cdcb0",
"sha256:aed4a9a7e3221b3e252c39d0bf794c438dc5453bc2963e8befe9d4cd324dff72",
"sha256:aef9aee84ec78af51107181d02fe8773b100b01c5dfde351184ad9223eab3698",
"sha256:b0221ca5a9837e040ebf61f48899926b5783668b7807419e4adae8175a31f773",
"sha256:b4d7679a08fea64573c969f6994a2631908bb2c0e69a7235648642f3d2e39a68",
"sha256:c250a7ec489b652c892e4f0a5d122cc14c3780f9f643e1a326754aedf82d9a76",
"sha256:ca86db5b561b894f9e5f115d6a159fff2a2570a652e07889d8a383b5fae66eb4",
"sha256:cfc523edecddaef56f6740d7de1ce24a2fdf94fd5e704091856a201872e37f9f",
"sha256:da113b70f6ec40e7d81b43d1b139b9db6a05727ab8be1ee559f3a69854a69d34",
"sha256:f6fac64a38f6768e7bc7b035b9e10d8a538a9fadce06b983fb3e6fa55ac5f5ce",
"sha256:f8559617b1fcf59a9aedba2c9838b5b6aa211ffedecabca412b92a1ff75aac1a",
"sha256:fbb42a541b1093385a2d8c7eec94d26d30437d0e77c1d25dae1dcc46741a385e"
],
"index": "pypi",
"version": "==2.9.1"
},
"pycryptodome": {
"hashes": [
"sha256:09c1555a3fa450e7eaca41ea11cd00afe7c91fef52353488e65663777d8524e0",
"sha256:12222a5edc9ca4a29de15fbd5339099c4c26c56e13c2ceddf0b920794f26165d",
"sha256:1723ebee5561628ce96748501cdaa7afaa67329d753933296321f0be55358dce",
"sha256:1c5e1ca507de2ad93474be5cfe2bfa76b7cf039a1a32fc196f40935944871a06",
"sha256:2603c98ae04aac675fefcf71a6c87dc4bb74a75e9071ae3923bbc91a59f08d35",
"sha256:2dea65df54349cdfa43d6b2e8edb83f5f8d6861e5cf7b1fbc3e34c5694c85e27",
"sha256:31c1df17b3dc5f39600a4057d7db53ac372f492c955b9b75dd439f5d8b460129",
"sha256:38661348ecb71476037f1e1f553159b80d256c00f6c0b00502acac891f7116d9",
"sha256:3e2e3a06580c5f190df843cdb90ea28d61099cf4924334d5297a995de68e4673",
"sha256:3f840c49d38986f6e17dbc0673d37947c88bc9d2d9dba1c01b979b36f8447db1",
"sha256:501ab36aae360e31d0ec370cf5ce8ace6cb4112060d099b993bc02b36ac83fb6",
"sha256:60386d1d4cfaad299803b45a5bc2089696eaf6cdd56f9fc17479a6f89595cfc8",
"sha256:6260e24d41149268122dd39d4ebd5941e9d107f49463f7e071fd397e29923b0c",
"sha256:6bbf7fee7b7948b29d7e71fcacf48bac0c57fb41332007061a933f2d996f9713",
"sha256:6d2df5223b12437e644ce0a3be7809471ffa71de44ccd28b02180401982594a6",
"sha256:758949ca62690b1540dfb24ad773c6da9cd0e425189e83e39c038bbd52b8e438",
"sha256:77997519d8eb8a4adcd9a47b9cec18f9b323e296986528186c0e9a7a15d6a07e",
"sha256:7fd519b89585abf57bf47d90166903ec7b43af4fe23c92273ea09e6336af5c07",
"sha256:98213ac2b18dc1969a47bc65a79a8fca02a414249d0c8635abb081c7f38c91b6",
"sha256:99b2f3fc51d308286071d0953f92055504a6ffe829a832a9fc7a04318a7683dd",
"sha256:9b6f711b25e01931f1c61ce0115245a23cdc8b80bf8539ac0363bdcf27d649b6",
"sha256:a3105a0eb63eacf98c2ecb0eb4aa03f77f40fbac2bdde22020bb8a536b226bb8",
"sha256:a8eb8b6ea09ec1c2535bf39914377bc8abcab2c7d30fa9225eb4fe412024e427",
"sha256:a92d5c414e8ee1249e850789052608f582416e82422502dc0ac8c577808a9067",
"sha256:d3d6958d53ad307df5e8469cc44474a75393a434addf20ecd451f38a72fe29b8",
"sha256:e0a4d5933a88a2c98bbe19c0c722f5483dc628d7a38338ac2cb64a7dbd34064b",
"sha256:e3bf558c6aeb49afa9f0c06cee7fb5947ee5a1ff3bd794b653d39926b49077fa",
"sha256:e61e363d9a5d7916f3a4ce984a929514c0df3daf3b1b2eb5e6edbb131ee771cf",
"sha256:f977cdf725b20f6b8229b0c87acb98c7717e742ef9f46b113985303ae12a99da",
"sha256:fc7489a50323a0df02378bc2fff86eb69d94cc5639914346c736be981c6a02e7"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==3.10.1"
}, },
"pytz": { "pytz": {
"hashes": [ "hashes": [
@@ -185,11 +316,11 @@
}, },
"requests": { "requests": {
"hashes": [ "hashes": [
"sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
"sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
], ],
"index": "pypi", "index": "pypi",
"version": "==2.25.1" "version": "==2.26.0"
}, },
"rjsmin": { "rjsmin": {
"hashes": [ "hashes": [
@@ -211,40 +342,74 @@
}, },
"six": { "six": {
"hashes": [ "hashes": [
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
"sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
], ],
"version": "==1.15.0" "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==1.16.0"
}, },
"sqlparse": { "sqlparse": {
"hashes": [ "hashes": [
"sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0", "sha256:0c00730c74263a94e5a9919ade150dfc3b19c574389985446148402998287dae",
"sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8" "sha256:48719e356bb8b42991bdbb1e8b83223757b93789c00910a616a071910ca4a64d"
], ],
"version": "==0.4.1" "markers": "python_version >= '3.5'",
"version": "==0.4.2"
}, },
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80", "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4",
"sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73" "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"
], ],
"version": "==1.26.3" "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
"version": "==1.26.6"
},
"websockets": {
"hashes": [
"sha256:01db0ecd1a0ca6702d02a5ed40413e18b7d22f94afb3bbe0d323bac86c42c1c8",
"sha256:085bb8a6e780d30eaa1ba48ac7f3a6707f925edea787cfb761ce5a39e77ac09b",
"sha256:1ac35426fe3e7d3d0fac3d63c8965c76ed67a8fd713937be072bf0ce22808539",
"sha256:1f6b814cff6aadc4288297cb3a248614829c6e4ff5556593c44a115e9dd49939",
"sha256:2a43072e434c041a99f2e1eb9b692df0232a38c37c61d00e9f24db79474329e4",
"sha256:5b2600e01c7ca6f840c42c747ffbe0254f319594ed108db847eb3d75f4aacb80",
"sha256:62160772314920397f9d219147f958b33fa27a12c662d4455c9ccbba9a07e474",
"sha256:706e200fc7f03bed99ad0574cd1ea8b0951477dd18cc978ccb190683c69dba76",
"sha256:71358c7816e2762f3e4af3adf0040f268e219f5a38cb3487a9d0fc2e554fef6a",
"sha256:7d2e12e4f901f1bc062dfdf91831712c4106ed18a9a4cdb65e2e5f502124ca37",
"sha256:7f79f02c7f9a8320aff7d3321cd1c7e3a7dbc15d922ac996cca827301ee75238",
"sha256:82b17524b1ce6ae7f7dd93e4d18e9b9474071e28b65dbf1dfe9b5767778db379",
"sha256:82bd921885231f4a30d9bc550552495b3fc36b1235add6d374e7c65c3babd805",
"sha256:8bbf8660c3f833ddc8b1afab90213f2e672a9ddac6eecb3cde968e6b2807c1c7",
"sha256:9a4d889162bd48588e80950e07fa5e039eee9deb76a58092e8c3ece96d7ef537",
"sha256:b4ade7569b6fd17912452f9c3757d96f8e4044016b6d22b3b8391e641ca50456",
"sha256:b8176deb6be540a46695960a765a77c28ac8b2e3ef2ec95d50a4f5df901edb1c",
"sha256:c4fc9a1d242317892590abe5b61a9127f1a61740477bfb121743f290b8054002",
"sha256:c5880442f5fc268f1ef6d37b2c152c114deccca73f48e3a8c48004d2f16f4567",
"sha256:cd8c6f2ec24aedace251017bc7a414525171d4e6578f914acab9349362def4da",
"sha256:d67646ddd17a86117ae21c27005d83c1895c0cef5d7be548b7549646372f868a",
"sha256:e42a1f1e03437b017af341e9bbfdc09252cd48ef32a8c3c3ead769eab3b17368",
"sha256:eb282127e9c136f860c6068a4fba5756eb25e755baffb5940b6f1eae071928b2",
"sha256:fe83b3ec9ef34063d86dfe1029160a85f24a5a94271036e5714a57acfdd089a1",
"sha256:ff59c6bdb87b31f7e2d596f09353d5a38c8c8ff571b0e2238e8ee2d55ad68465"
],
"markers": "python_version >= '3.7'",
"version": "==10.0"
}, },
"whitenoise": { "whitenoise": {
"hashes": [ "hashes": [
"sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7", "sha256:d234b871b52271ae7ed6d9da47ffe857c76568f11dd30e28e18c5869dbd11e12",
"sha256:05d00198c777028d72d8b0bbd234db605ef6d60e9410125124002518a48e515d" "sha256:d963ef25639d1417e8a247be36e6aedd8c7c6f0a08adcb5a89146980a96b577c"
], ],
"index": "pypi", "index": "pypi",
"version": "==5.2.0" "version": "==5.3.0"
}, },
"youtube-dl": { "yt-dlp": {
"hashes": [ "hashes": [
"sha256:b390cddbd4d605bd887d0d4063988cef0fa13f916d2e1e3564badbb22504d754", "sha256:c97716a715261657345176ab8190a19efa51db0e5b174a6629956548750245e1",
"sha256:e7d48cd42f3081e1e0064e69f31f2856508ef31c0fc80eeebd8e70c6a031a24d" "sha256:ca7e77cdb055ba2683df5b0807aab1c1e120cbe02c8f35d9d3293d94dbdaea63"
], ],
"index": "pypi", "index": "pypi",
"version": "==2021.2.10" "version": "==2021.9.2"
} }
}, },
"develop": {} "develop": {}

View File

@@ -12,7 +12,7 @@ media server, then TubeSync is for you. Internally, TubeSync is a web interface
on `youtube-dl` and `ffmpeg` with a task scheduler. on `youtube-dl` and `ffmpeg` with a task scheduler.
There are several other web interfaces to YouTube and `youtube-dl` all with varying There are several other web interfaces to YouTube and `youtube-dl` all with varying
features and implemenations. TubeSync's largest difference is full PVR experience of features and implementations. TubeSync's largest difference is full PVR experience of
updating media servers and better selection of media formats. Additionally, to be as updating media servers and better selection of media formats. Additionally, to be as
hands-free as possible, TubeSync has gradual retrying of failures with back-off timers hands-free as possible, TubeSync has gradual retrying of failures with back-off timers
so media which fails to download will be retried for an extended period making it, so media which fails to download will be retried for an extended period making it,
@@ -22,12 +22,9 @@ hopefully, quite reliable.
# Latest container image # Latest container image
```yaml ```yaml
ghcr.io/meeb/tubesync:v0.9 ghcr.io/meeb/tubesync:latest
``` ```
**NOTE: the `:latest` tag does exist, but will contain in-development commits and may
be broken. Use at your own risk.**
# Screenshots # Screenshots
### Dashboard ### Dashboard
@@ -101,8 +98,8 @@ $ mkdir /some/directory/tubesync-downloads
Finally, download and run the container: Finally, download and run the container:
```bash ```bash
# Pull a versioned image # Pull image
$ docker pull ghcr.io/meeb/tubesync:v0.9 $ docker pull ghcr.io/meeb/tubesync:latest
# Start the container using your user ID and group ID # Start the container using your user ID and group ID
$ docker run \ $ docker run \
-d \ -d \
@@ -113,7 +110,7 @@ $ docker run \
-v /some/directory/tubesync-config:/config \ -v /some/directory/tubesync-config:/config \
-v /some/directory/tubesync-downloads:/downloads \ -v /some/directory/tubesync-downloads:/downloads \
-p 4848:4848 \ -p 4848:4848 \
ghcr.io/meeb/tubesync:v0.9 ghcr.io/meeb/tubesync:latest
``` ```
Once running, open `http://localhost:4848` in your browser and you should see the Once running, open `http://localhost:4848` in your browser and you should see the
@@ -125,7 +122,7 @@ Alternatively, for Docker Compose, you can use something like:
```yaml ```yaml
tubesync: tubesync:
image: ghcr.io/meeb/tubesync:v0.9 image: ghcr.io/meeb/tubesync:latest
container_name: tubesync container_name: tubesync
restart: unless-stopped restart: unless-stopped
ports: ports:
@@ -139,6 +136,41 @@ Alternatively, for Docker Compose, you can use something like:
- PGID=1000 - PGID=1000
``` ```
## Optional authentication
Available in `v1.0` (or `:latest`)and later. If you want to enable a basic username and
password to be required to access the TubeSync dashboard you can set them with the
following environment variables:
```bash
HTTP_USER
HTTP_PASS
```
For example in the `docker run ...` line add in:
```bash
...
-e HTTP_USER=some-username \
-e HTTP_PASS=some-secure-password \
...
```
Or in your Docker Compose file you would add in:
```yaml
...
environment:
- HTTP_USER=some-username
- HTTP_PASS=some-secure-password
...
```
When BOTH `HTTP_USER` and `HTTP_PASS` are set then basic HTTP authentication will be
enabled.
# Updating # Updating
To update, you can just pull a new version of the container image as they are released. To update, you can just pull a new version of the container image as they are released.
@@ -205,6 +237,10 @@ and less common features:
![Sync or create missing metadata files](https://github.com/meeb/tubesync/blob/main/docs/create-missing-metadata.md) ![Sync or create missing metadata files](https://github.com/meeb/tubesync/blob/main/docs/create-missing-metadata.md)
![Reset tasks from the command line](https://github.com/meeb/tubesync/blob/main/docs/reset-tasks.md)
![Using PostgreSQL, MySQL or MariaDB as database backends](https://github.com/meeb/tubesync/blob/main/docs/other-database-backends.md)
# Warnings # Warnings
@@ -300,7 +336,9 @@ can log in at http://localhost:4848/admin
### Are there user accounts or multi-user support? ### Are there user accounts or multi-user support?
No not at the moment. This could be added later if there is demand for it. There is support for basic HTTP authentication by setting the `HTTP_USER` and
`HTTP_PASS` environment variables. There is not support for multi-user or user
management.
### Does TubeSync support HTTPS? ### Does TubeSync support HTTPS?
@@ -319,7 +357,7 @@ There are a number of other environment variables you can set. These are, mostly
useful if you are manually installing TubeSync in some other environment. These are: useful if you are manually installing TubeSync in some other environment. These are:
| Name | What | Example | | Name | What | Example |
| ------------------------ | ------------------------------------------------------------ | ---------------------------------- | | ------------------------ | ------------------------------------------------------------ | ------------------------------------ |
| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | | DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l |
| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME | /somepath | | DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME | /somepath |
| TUBESYNC_DEBUG | Enable debugging | True | | TUBESYNC_DEBUG | Enable debugging | True |
@@ -328,6 +366,9 @@ useful if you are manually installing TubeSync in some other environment. These
| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | | GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 |
| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | | LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 |
| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | | LISTEN_PORT | Port number for gunicorn to listen on | 8080 |
| HTTP_USER | Sets the username for HTTP basic authentication | some-username |
| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password |
| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database |
# Manual, non-containerised, installation # Manual, non-containerised, installation
@@ -344,7 +385,7 @@ installing and running WSGI-based Python web applications before attempting this
`tubesync/tubesync/local_settings.py` and edit it as appropriate `tubesync/tubesync/local_settings.py` and edit it as appropriate
5. Run migrations with `./manage.py migrate` 5. Run migrations with `./manage.py migrate`
6. Collect static files with `./manage.py collectstatic` 6. Collect static files with `./manage.py collectstatic`
6. Set up your prefered WSGI server, such as `gunicorn` poiting it to the application 6. Set up your prefered WSGI server, such as `gunicorn` pointing it to the application
in `tubesync/tubesync/wsgi.py` in `tubesync/tubesync/wsgi.py`
7. Set up your proxy server such as `nginx` and forward it to the WSGI server 7. Set up your proxy server such as `nginx` and forward it to the WSGI server
8. Check the web interface is working 8. Check the web interface is working

View File

@@ -5,5 +5,20 @@ umask "$UMASK_SET"
cd /app || exit cd /app || exit
PIDFILE=/run/app/gunicorn.pid
if [ -f "${PIDFILE}" ]
then
PID=$(cat $PIDFILE)
echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
if kill -0 $PID
then
echo "Killing old gunicorn process with PID: ${PID}"
kill -9 $PID
fi
echo "Removing stale PID file: ${PIDFILE}"
rm ${PIDFILE}
fi
exec s6-setuidgid app \ exec s6-setuidgid app \
/usr/local/bin/gunicorn -c /app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application /usr/local/bin/gunicorn -c /app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application

View File

@@ -0,0 +1,80 @@
# TubeSync
## Advanced usage guide - using other database backends
This is a new feature in v1.0 of TubeSync and later. It allows you to use a custom
existing external database server instead of the default SQLite database. You may want
to use this if you encounter performance issues with adding very large or a large
number of channels and database write contention (as shown by errors in the log)
become an issue.
## Requirements
TubeSync supports SQLite (the automatic default) as well as PostgreSQL, MySQL and
MariaDB. For MariaDB just follow the MySQL instructions as the driver is the same.
You should a blank install of TubeSync. Migrating to a new database will reset your
database. If you are comfortable with Django you can export and re-import existing
database data with:
```bash
$ docker exec -ti tubesync python3 /app/manage.py dumpdata > some-file.json
```
Then change you database backend over, then use
```bash
$ cat some-file.json | docker exec -ti tubesync python3 /app/manage.py loaddata --format=json -
```
As detailed in the Django documentation:
https://docs.djangoproject.com/en/3.1/ref/django-admin/#dumpdata
and:
https://docs.djangoproject.com/en/3.1/ref/django-admin/#loaddata
Further instructions are beyond the scope of TubeSync documenation and you should refer
to Django documentation for more details.
If you are not comfortable with the above, then skip the `dumpdata` steps, however
remember you will start again with a completely new database.
## Steps
### 1. Create a database in your external database server
You need to create a database and a user with permissions to access the database in
your chosen external database server. Steps vary between PostgreSQL, MySQL and MariaDB
so this is up to you to work out.
### 2. Set the database connection string environment variable
You need to provide the database connection details to TubeSync via an environment
variable. The environment variable name is `DATABASE_CONNECTION` and the format is the
standard URL-style string. Examples are:
`postgresql://tubesync:password@localhost:5432/tubesync`
and
`mysql://tubesync:password@localhost:3306/tubesync`
*Important note:* For MySQL databases make SURE you create the tubesync database with
`utf8mb4` encoding, like:
`CREATE DATABASE tubesync CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;`
Without `utf8mb4` encoding things like emojis in video titles (or any extended UTF8
characters) can cause issues.
### 3. Start TubeSync and check the logs
Once you start TubeSync with the new database connection you should see the folling log
entry in the container or stdout logs:
`2021-04-04 22:42:17,912 [tubesync/INFO] Using database connection: django.db.backends.postgresql://tubesync:[hidden]@localhost:5432/tubesync`
If you see a line similar to the above and the web interface loads, congratulations,
you are now using an external database server for your TubeSync data!

33
docs/reset-tasks.md Normal file
View File

@@ -0,0 +1,33 @@
# TubeSync
## Advanced usage guide - reset tasks from the command line
This is a new feature in v1.0 of TubeSync and later. It allows you to reset all
scheduled tasks from the command line as well as the "reset tasks" button in the
"tasks" tab of the dashboard.
This is useful for TubeSync installations where you may have a lot of media and
sources added and the "reset tasks" button may take too long to the extent where
the page times out (with a 502 error or similar issue).
## Requirements
You have added some sources and media
## Steps
### 1. Run the reset tasks command
Execute the following Django command:
`./manage.py reset-tasks`
When deploying TubeSync inside a container, you can execute this with:
`docker exec -ti tubesync python3 /app/manage.py reset-tasks`
This command will log what its doing to the terminal when you run it.
When this is run, new tasks will be immediately created so all your sources will be
indexed again straight away, any missing information such as thumbnails will be
redownloaded, etc.

View File

@@ -1,10 +1,10 @@
from django.conf import settings from django.conf import settings
from .third_party_versions import youtube_dl_version, ffmpeg_version from .third_party_versions import yt_dlp_version, ffmpeg_version
def app_details(request): def app_details(request):
return { return {
'app_version': str(settings.VERSION), 'app_version': str(settings.VERSION),
'youtube_dl_version': youtube_dl_version, 'yt_dlp_version': yt_dlp_version,
'ffmpeg_version': ffmpeg_version, 'ffmpeg_version': ffmpeg_version,
} }

View File

@@ -20,3 +20,10 @@ class DownloadFailedException(Exception):
exist. exist.
''' '''
pass pass
class DatabaseConnectionError(Exception):
'''
Raised when parsing or initially connecting to a database.
'''
pass

View File

@@ -1,4 +1,6 @@
from django.conf import settings
from django.forms import BaseForm from django.forms import BaseForm
from basicauth.middleware import BasicAuthMiddleware as BaseBasicAuthMiddleware
class MaterializeDefaultFieldsMiddleware: class MaterializeDefaultFieldsMiddleware:
@@ -19,3 +21,12 @@ class MaterializeDefaultFieldsMiddleware:
for _, field in v.fields.items(): for _, field in v.fields.items():
field.widget.attrs.update({'class':'browser-default'}) field.widget.attrs.update({'class':'browser-default'})
return response return response
class BasicAuthMiddleware(BaseBasicAuthMiddleware):
def process_request(self, request):
bypass_uris = getattr(settings, 'BASICAUTH_ALWAYS_ALLOW_URIS', [])
if request.path in bypass_uris:
return None
return super().process_request(request)

View File

@@ -181,8 +181,10 @@ main {
display: inline-block; display: inline-block;
font-weight: bold; font-weight: bold;
text-decoration: none; text-decoration: none;
padding: 5px 10px 5px 10px; padding: 5px 8px 4px 8px;
margin: 0 3px 0 3px; margin: 0 3px 6px 3px;
min-width: 40px;
min-height: 40px;
background-color: $pagination-background-colour; background-color: $pagination-background-colour;
color: $pagination-text-colour; color: $pagination-text-colour;
border: 2px $pagination-border-colour solid; border: 2px $pagination-border-colour solid;

View File

@@ -57,7 +57,7 @@
</p> </p>
<p> <p>
<a href="https://github.com/meeb/tubesync" class="nowrap" target="_blank"><i class="fab fa-github"></i> TubeSync</a> version <strong>{{ app_version }}</strong> with <a href="https://github.com/meeb/tubesync" class="nowrap" target="_blank"><i class="fab fa-github"></i> TubeSync</a> version <strong>{{ app_version }}</strong> with
<a href="https://yt-dl.org/" class="nowrap" target="_blank"><i class="fas fa-link"></i> youtube-dl</a> version <strong>{{ youtube_dl_version }}</strong> and <a href="https://github.com/yt-dlp/yt-dlp" class="nowrap" target="_blank"><i class="fas fa-link"></i> yt-dlp</a> version <strong>{{ yt_dlp_version }}</strong> and
<a href="https://ffmpeg.org/" class="nowrap" target="_blank"><i class="fas fa-link"></i> FFmpeg</a> version <strong>{{ ffmpeg_version }}</strong>. <a href="https://ffmpeg.org/" class="nowrap" target="_blank"><i class="fas fa-link"></i> FFmpeg</a> version <strong>{{ ffmpeg_version }}</strong>.
</p> </p>
</div> </div>

View File

@@ -3,7 +3,7 @@
<div class="col s12"> <div class="col s12">
<div class="pagination"> <div class="pagination">
{% for i in paginator.page_range %} {% for i in paginator.page_range %}
<a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}">{{ i }}</a> <a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}{% if show_skipped %}&show_skipped=yes{% endif %}">{{ i }}</a>
{% endfor %} {% endfor %}
</div> </div>
</div> </div>

View File

@@ -2,6 +2,8 @@ import os.path
from django.conf import settings from django.conf import settings
from django.test import TestCase, Client from django.test import TestCase, Client
from .testutils import prevent_request_warnings from .testutils import prevent_request_warnings
from .utils import parse_database_connection_string
from .errors import DatabaseConnectionError
class ErrorPageTestCase(TestCase): class ErrorPageTestCase(TestCase):
@@ -61,3 +63,66 @@ class CommonStaticTestCase(TestCase):
favicon_real_path = os.path.join(os.sep.join(root_parts), favicon_real_path = os.path.join(os.sep.join(root_parts),
os.sep.join(url_parts)) os.sep.join(url_parts))
self.assertTrue(os.path.exists(favicon_real_path)) self.assertTrue(os.path.exists(favicon_real_path))
class DatabaseConnectionTestCase(TestCase):
def test_parse_database_connection_string(self):
database_dict = parse_database_connection_string(
'postgresql://tubesync:password@localhost:5432/tubesync')
self.assertEqual(database_dict,
{
'DRIVER': 'postgresql',
'ENGINE': 'django.db.backends.postgresql',
'USER': 'tubesync',
'PASSWORD': 'password',
'HOST': 'localhost',
'PORT': 5432,
'NAME': 'tubesync',
'CONN_MAX_AGE': 300,
'OPTIONS': {},
}
)
database_dict = parse_database_connection_string(
'mysql://tubesync:password@localhost:3306/tubesync')
self.assertEqual(database_dict,
{
'DRIVER': 'mysql',
'ENGINE': 'django.db.backends.mysql',
'USER': 'tubesync',
'PASSWORD': 'password',
'HOST': 'localhost',
'PORT': 3306,
'NAME': 'tubesync',
'CONN_MAX_AGE': 300,
'OPTIONS': {'charset': 'utf8mb4'}
}
)
# Invalid driver
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'test://tubesync:password@localhost:5432/tubesync')
# No username
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'postgresql://password@localhost:5432/tubesync')
# No database name
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'postgresql://tubesync:password@5432')
# Invalid port
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'postgresql://tubesync:password@localhost:test/tubesync')
# Invalid port
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'postgresql://tubesync:password@localhost:65537/tubesync')
# Invalid username or password
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'postgresql://tubesync:password:test@localhost:5432/tubesync')
# Invalid database name
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'postgresql://tubesync:password@localhost:5432/tubesync/test')

View File

@@ -1,7 +1,7 @@
from youtube_dl import version as yt_version from yt_dlp import version as yt_dlp_version
youtube_dl_version = str(yt_version.__version__) yt_dlp_version = str(yt_dlp_version.__version__)
ffmpeg_version = '(shared install)' ffmpeg_version = '(shared install)'

View File

@@ -1,4 +1,95 @@
from urllib.parse import urlunsplit, urlencode from datetime import datetime
from urllib.parse import urlunsplit, urlencode, urlparse
from yt_dlp.utils import LazyList
from .errors import DatabaseConnectionError
def parse_database_connection_string(database_connection_string):
'''
Parses a connection string in a URL style format, such as:
postgresql://tubesync:password@localhost:5432/tubesync
mysql://someuser:somepassword@localhost:3306/tubesync
into a Django-compatible settings.DATABASES dict format.
'''
valid_drivers = ('postgresql', 'mysql')
default_ports = {
'postgresql': 5432,
'mysql': 3306,
}
django_backends = {
'postgresql': 'django.db.backends.postgresql',
'mysql': 'django.db.backends.mysql',
}
backend_options = {
'postgresql': {},
'mysql': {
'charset': 'utf8mb4',
}
}
try:
parts = urlparse(str(database_connection_string))
except Exception as e:
raise DatabaseConnectionError(f'Failed to parse "{database_connection_string}" '
f'as a database connection string: {e}') from e
driver = parts.scheme
user_pass_host_port = parts.netloc
database = parts.path
if driver not in valid_drivers:
raise DatabaseConnectionError(f'Database connection string '
f'"{database_connection_string}" specified an '
f'invalid driver, must be one of {valid_drivers}')
django_driver = django_backends.get(driver)
host_parts = user_pass_host_port.split('@')
if len(host_parts) != 2:
raise DatabaseConnectionError(f'Database connection string netloc must be in '
f'the format of user:pass@host')
user_pass, host_port = host_parts
user_pass_parts = user_pass.split(':')
if len(user_pass_parts) != 2:
raise DatabaseConnectionError(f'Database connection string netloc must be in '
f'the format of user:pass@host')
username, password = user_pass_parts
host_port_parts = host_port.split(':')
if len(host_port_parts) == 1:
# No port number, assign a default port
hostname = host_port_parts[0]
port = default_ports.get(driver)
elif len(host_port_parts) == 2:
# Host name and port number
hostname, port = host_port_parts
try:
port = int(port)
except (ValueError, TypeError) as e:
raise DatabaseConnectionError(f'Database connection string contained an '
f'invalid port, ports must be integers: '
f'{e}') from e
if not 0 < port < 63336:
raise DatabaseConnectionError(f'Database connection string contained an '
f'invalid port, ports must be between 1 and '
f'65535, got {port}')
else:
# Malformed
raise DatabaseConnectionError(f'Database connection host must be a hostname or '
f'a hostname:port combination')
if database.startswith('/'):
database = database[1:]
if not database:
raise DatabaseConnectionError(f'Database connection string path must be a '
f'string in the format of /databasename')
if '/' in database:
raise DatabaseConnectionError(f'Database connection string path can only '
f'contain a single string name, got: {database}')
return {
'DRIVER': driver,
'ENGINE': django_driver,
'NAME': database,
'USER': username,
'PASSWORD': password,
'HOST': hostname,
'PORT': port,
'CONN_MAX_AGE': 300,
'OPTIONS': backend_options.get(driver),
}
def get_client_ip(request): def get_client_ip(request):
@@ -24,3 +115,11 @@ def clean_filename(filename):
filename = filename.replace(char, '') filename = filename.replace(char, '')
filename = ''.join([c for c in filename if ord(c) > 30]) filename = ''.join([c for c in filename if ord(c) > 30])
return ' '.join(filename.split()) return ' '.join(filename.split())
def json_serial(obj):
if isinstance(obj, datetime):
return obj.isoformat()
if isinstance(obj, LazyList):
return list(obj)
raise TypeError(f'Type {type(obj)} is not json_serial()-able')

View File

@@ -0,0 +1,51 @@
import os
import uuid
from django.utils.translation import gettext_lazy as _
from django.core.management.base import BaseCommand, CommandError
from django.db.models import signals
from common.logger import log
from sync.models import Source, Media, MediaServer
from sync.signals import media_post_delete
from sync.tasks import rescan_media_server
class Command(BaseCommand):
help = ('Deletes a source by UUID')
def add_arguments(self, parser):
parser.add_argument('--source', action='store', required=True, help='Source UUID')
def handle(self, *args, **options):
source_uuid_str = options.get('source', '')
try:
source_uuid = uuid.UUID(source_uuid_str)
except Exception as e:
raise CommandError(f'Failed to parse source UUID: {e}')
log.info(f'Deleting source with UUID: {source_uuid}')
# Fetch the source by UUID
try:
source = Source.objects.get(uuid=source_uuid)
except Source.DoesNotExist:
raise CommandError(f'Source does not exist with '
f'UUID: {source_uuid}')
# Detach post-delete signal for Media so we don't spam media servers
signals.post_delete.disconnect(media_post_delete, sender=Media)
# Delete the source, triggering pre-delete signals for each media item
log.info(f'Found source with UUID "{source.uuid}" with name '
f'"{source.name}" and deleting it, this may take some time!')
source.delete()
# Update any media servers
for mediaserver in MediaServer.objects.all():
log.info(f'Scheduling media server updates')
verbose_name = _('Request media server rescan for "{}"')
rescan_media_server(
str(mediaserver.pk),
priority=0,
verbose_name=verbose_name.format(mediaserver),
remove_existing_tasks=True
)
# Re-attach signals
signals.post_delete.connect(media_post_delete, sender=Media)
# All done
log.info('Done')

View File

@@ -0,0 +1,15 @@
import os
from django.core.management.base import BaseCommand, CommandError
from common.logger import log
from sync.models import Source, Media, MediaServer
class Command(BaseCommand):
help = ('Lists sources')
def handle(self, *args, **options):
log.info('Listing sources...')
for source in Source.objects.all():
log.info(f' - {source.uuid}: {source.name}')
log.info('Done')

View File

@@ -0,0 +1,33 @@
from django.core.management.base import BaseCommand, CommandError
from django.utils.translation import gettext_lazy as _
from background_task.models import Task
from sync.models import Source
from sync.tasks import index_source_task
from common.logger import log
class Command(BaseCommand):
help = 'Resets all tasks'
def handle(self, *args, **options):
log.info('Resettings all tasks...')
# Delete all tasks
Task.objects.all().delete()
# Iter all tasks
for source in Source.objects.all():
# Recreate the initial indexing task
log.info(f'Resetting tasks for source: {source}')
verbose_name = _('Index media from source "{}"')
index_source_task(
str(source.pk),
repeat=source.index_schedule,
queue=str(source.pk),
priority=5,
verbose_name=verbose_name.format(source.name)
)
# This also chains down to call each Media objects .save() as well
source.save()
log.info('Done')

View File

@@ -158,6 +158,9 @@ class Source(models.Model):
EVERY_6_HOURS = 21600, _('Every 6 hours') EVERY_6_HOURS = 21600, _('Every 6 hours')
EVERY_12_HOURS = 43200, _('Every 12 hours') EVERY_12_HOURS = 43200, _('Every 12 hours')
EVERY_24_HOURS = 86400, _('Every 24 hours') EVERY_24_HOURS = 86400, _('Every 24 hours')
EVERY_3_DAYS = 259200, _('Every 3 days')
EVERY_7_DAYS = 604800, _('Every 7 days')
NEVER = 0, _('Never')
uuid = models.UUIDField( uuid = models.UUIDField(
_('uuid'), _('uuid'),
@@ -218,7 +221,7 @@ class Source(models.Model):
_('index schedule'), _('index schedule'),
choices=IndexSchedule.choices, choices=IndexSchedule.choices,
db_index=True, db_index=True,
default=IndexSchedule.EVERY_6_HOURS, default=IndexSchedule.EVERY_24_HOURS,
help_text=_('Schedule of how often to index the source for new media') help_text=_('Schedule of how often to index the source for new media')
) )
download_media = models.BooleanField( download_media = models.BooleanField(
@@ -437,7 +440,6 @@ class Source(models.Model):
'title_full': 'Some Media Title Name', 'title_full': 'Some Media Title Name',
'key': 'SoMeUnIqUiD', 'key': 'SoMeUnIqUiD',
'format': '-'.join(fmt), 'format': '-'.join(fmt),
'playlist_index': 1,
'playlist_title': 'Some Playlist Title', 'playlist_title': 'Some Playlist Title',
'ext': self.extension, 'ext': self.extension,
'resolution': self.source_resolution if self.source_resolution else '', 'resolution': self.source_resolution if self.source_resolution else '',
@@ -559,11 +561,6 @@ class Media(models.Model):
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count',
}, },
'playlist_index': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_index',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_index',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'playlist_index',
},
'playlist_title': { 'playlist_title': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title', Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title',
@@ -815,6 +812,20 @@ class Media(models.Model):
hdr = '' hdr = ''
# If the download has completed use existing values # If the download has completed use existing values
if self.downloaded: if self.downloaded:
# Check if there's any stored meta data at all
if (not self.downloaded_video_codec and \
not self.downloaded_audio_codec):
# Marked as downloaded but no metadata, imported?
return {
'resolution': resolution,
'height': height,
'width': width,
'vcodec': vcodec,
'acodec': acodec,
'fps': fps,
'hdr': hdr,
'format': tuple(fmt),
}
resolution = f'{self.downloaded_height}p' resolution = f'{self.downloaded_height}p'
if self.downloaded_format != 'audio': if self.downloaded_format != 'audio':
vcodec = self.downloaded_video_codec.lower() vcodec = self.downloaded_video_codec.lower()
@@ -912,7 +923,6 @@ class Media(models.Model):
'title_full': clean_filename(self.title), 'title_full': clean_filename(self.title),
'key': self.key, 'key': self.key,
'format': '-'.join(display_format['format']), 'format': '-'.join(display_format['format']),
'playlist_index': self.playlist_index,
'playlist_title': self.playlist_title, 'playlist_title': self.playlist_title,
'ext': self.source.extension, 'ext': self.source.extension,
'resolution': display_format['resolution'], 'resolution': display_format['resolution'],
@@ -931,7 +941,10 @@ class Media(models.Model):
@property @property
def loaded_metadata(self): def loaded_metadata(self):
try: try:
return json.loads(self.metadata) data = json.loads(self.metadata)
if not isinstance(data, dict):
return {}
return data
except Exception as e: except Exception as e:
return {} return {}
@@ -968,7 +981,10 @@ class Media(models.Model):
@property @property
def upload_date(self): def upload_date(self):
field = self.get_metadata_field('upload_date') field = self.get_metadata_field('upload_date')
try:
upload_date_str = self.loaded_metadata.get(field, '').strip() upload_date_str = self.loaded_metadata.get(field, '').strip()
except (AttributeError, ValueError) as e:
return None
try: try:
return datetime.strptime(upload_date_str, '%Y%m%d') return datetime.strptime(upload_date_str, '%Y%m%d')
except (AttributeError, ValueError) as e: except (AttributeError, ValueError) as e:
@@ -1023,11 +1039,6 @@ class Media(models.Model):
field = self.get_metadata_field('formats') field = self.get_metadata_field('formats')
return self.loaded_metadata.get(field, []) return self.loaded_metadata.get(field, [])
@property
def playlist_index(self):
field = self.get_metadata_field('playlist_index')
return self.loaded_metadata.get(field, 0)
@property @property
def playlist_title(self): def playlist_title(self):
field = self.get_metadata_field('playlist_title') field = self.get_metadata_field('playlist_title')

View File

@@ -47,6 +47,7 @@ def source_post_save(sender, instance, created, **kwargs):
priority=0, priority=0,
verbose_name=verbose_name.format(instance.name) verbose_name=verbose_name.format(instance.name)
) )
if instance.index_schedule > 0:
delete_task_by_source('sync.tasks.index_source_task', instance.pk) delete_task_by_source('sync.tasks.index_source_task', instance.pk)
log.info(f'Scheduling media indexing for source: {instance.name}') log.info(f'Scheduling media indexing for source: {instance.name}')
verbose_name = _('Index media from source "{}"') verbose_name = _('Index media from source "{}"')
@@ -92,17 +93,57 @@ def task_task_failed(sender, task_id, completed_task, **kwargs):
@receiver(post_save, sender=Media) @receiver(post_save, sender=Media)
def media_post_save(sender, instance, created, **kwargs): def media_post_save(sender, instance, created, **kwargs):
# Triggered after media is saved, Recalculate the "can_download" flag, this may # Triggered after media is saved
cap_changed = False
can_download_changed = False
# Reset the skip flag if the download cap has changed if the media has not
# already been downloaded
if not instance.downloaded:
max_cap_age = instance.source.download_cap_date
published = instance.published
if not published:
if not instance.skip:
log.warn(f'Media: {instance.source} / {instance} has no published date '
f'set, marking to be skipped')
instance.skip = True
cap_changed = True
else:
log.debug(f'Media: {instance.source} / {instance} has no published date '
f'set but is already marked to be skipped')
else:
if max_cap_age:
if published > max_cap_age and instance.skip:
# Media was published after the cap date but is set to be skipped
log.info(f'Media: {instance.source} / {instance} has a valid '
f'publishing date, marking to be unskipped')
instance.skip = False
cap_changed = True
elif published <= max_cap_age and not instance.skip:
log.info(f'Media: {instance.source} / {instance} is too old for '
f'the download cap date, marking to be skipped')
instance.skip = True
cap_changed = True
else:
if instance.skip:
# Media marked to be skipped but source download cap removed
log.info(f'Media: {instance.source} / {instance} has a valid '
f'publishing date, marking to be unskipped')
instance.skip = False
cap_changed = True
# Recalculate the "can_download" flag, this may
# need to change if the source specifications have been changed # need to change if the source specifications have been changed
if instance.metadata: if instance.metadata:
post_save.disconnect(media_post_save, sender=Media)
if instance.get_format_str(): if instance.get_format_str():
if not instance.can_download: if not instance.can_download:
instance.can_download = True instance.can_download = True
instance.save() can_download_changed = True
else: else:
if instance.can_download: if instance.can_download:
instance.can_download = False instance.can_download = False
can_download_changed = True
# Save the instance if any changes were required
if cap_changed or can_download_changed:
post_save.disconnect(media_post_save, sender=Media)
instance.save() instance.save()
post_save.connect(media_post_save, sender=Media) post_save.connect(media_post_save, sender=Media)
# If the media is missing metadata schedule it to be downloaded # If the media is missing metadata schedule it to be downloaded

View File

@@ -10,7 +10,7 @@ import math
import uuid import uuid
from io import BytesIO from io import BytesIO
from hashlib import sha1 from hashlib import sha1
from datetime import timedelta from datetime import timedelta, datetime
from shutil import copyfile from shutil import copyfile
from PIL import Image from PIL import Image
from django.conf import settings from django.conf import settings
@@ -22,6 +22,7 @@ from background_task import background
from background_task.models import Task, CompletedTask from background_task.models import Task, CompletedTask
from common.logger import log from common.logger import log
from common.errors import NoMediaException, DownloadFailedException from common.errors import NoMediaException, DownloadFailedException
from common.utils import json_serial
from .models import Source, Media, MediaServer from .models import Source, Media, MediaServer
from .utils import (get_remote_image, resize_image_to_height, delete_file, from .utils import (get_remote_image, resize_image_to_height, delete_file,
write_text_file) write_text_file)
@@ -175,7 +176,7 @@ def index_source_task(source_id):
# Video has no unique key (ID), it can't be indexed # Video has no unique key (ID), it can't be indexed
continue continue
try: try:
media = Media.objects.get(key=key) media = Media.objects.get(key=key, source=source)
except Media.DoesNotExist: except Media.DoesNotExist:
media = Media(key=key) media = Media(key=key)
media.source = source media.source = source
@@ -224,7 +225,7 @@ def download_media_metadata(media_id):
return return
source = media.source source = media.source
metadata = media.index_metadata() metadata = media.index_metadata()
media.metadata = json.dumps(metadata) media.metadata = json.dumps(metadata, default=json_serial)
upload_date = media.upload_date upload_date = media.upload_date
# Media must have a valid upload date # Media must have a valid upload date
if upload_date: if upload_date:
@@ -242,6 +243,11 @@ def download_media_metadata(media_id):
media.skip = True media.skip = True
# If the source has a cut-off check the upload date is within the allowed delta # If the source has a cut-off check the upload date is within the allowed delta
if source.delete_old_media and source.days_to_keep > 0: if source.delete_old_media and source.days_to_keep > 0:
if not isinstance(media.published, datetime):
# Media has no known published date or incomplete metadata
log.warn(f'Media: {source} / {media} has no published date, skipping')
media.skip = True
else:
delta = timezone.now() - timedelta(days=source.days_to_keep) delta = timezone.now() - timedelta(days=source.days_to_keep)
if media.published < delta: if media.published < delta:
# Media was published after the cutoff date, skip it # Media was published after the cutoff date, skip it
@@ -305,18 +311,26 @@ def download_media(media_id):
return return
if media.skip: if media.skip:
# Media was toggled to be skipped after the task was scheduled # Media was toggled to be skipped after the task was scheduled
log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but it ' log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
f'is now marked to be skipped, not downloading') f'it is now marked to be skipped, not downloading')
return return
if media.downloaded and media.media_file: if media.downloaded and media.media_file:
# Media has been marked as downloaded before the download_media task was fired, # Media has been marked as downloaded before the download_media task was fired,
# skip it # skip it
log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but it ' log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
f'has already been marked as downloaded, not downloading again') f'it has already been marked as downloaded, not downloading again')
return return
if not media.source.download_media: if not media.source.download_media:
log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but the ' log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
f'source {media.source} has since been marked to not download media, ' f'the source {media.source} has since been marked to not download, '
f'not downloading')
return
max_cap_age = media.source.download_cap_date
published = media.published
if max_cap_age and published:
if published <= max_cap_age:
log.warn(f'Download task triggered media: {media} (UUID: {media.pk}) but '
f'the source has a download cap and the media is now too old, '
f'not downloading') f'not downloading')
return return
filepath = media.filepath filepath = media.filepath

View File

@@ -63,11 +63,6 @@
<td>Media format string</td> <td>Media format string</td>
<td>720p-avc1-mp4a</td> <td>720p-avc1-mp4a</td>
</tr> </tr>
<tr>
<td>{playlist_index}</td>
<td>Playlist index of media, if it's in a playlist</td>
<td>12</td>
</tr>
<tr> <tr>
<td>{playlist_title}</td> <td>{playlist_title}</td>
<td>Playlist title of media, if it's in a playlist</td> <td>Playlist title of media, if it's in a playlist</td>

View File

@@ -123,6 +123,10 @@
<td class="hide-on-small-only">Downloads directory</td> <td class="hide-on-small-only">Downloads directory</td>
<td><span class="hide-on-med-and-up">Downloads directory<br></span><strong>{{ downloads_dir }}</strong></td> <td><span class="hide-on-med-and-up">Downloads directory<br></span><strong>{{ downloads_dir }}</strong></td>
</tr> </tr>
<tr title="Database connection used by TubeSync">
<td class="hide-on-small-only">Database</td>
<td><span class="hide-on-med-and-up">Database<br></span><strong>{{ database_connection }}</strong></td>
</tr>
</table> </table>
</div> </div>
</div> </div>

View File

@@ -4,9 +4,16 @@
{% block content %} {% block content %}
<div class="row"> <div class="row">
<div class="col s12"> <div class="col s12 m9">
<h1 class="truncate">Media</h1> <h1 class="truncate">Media</h1>
</div> </div>
<div class="col s12 m3">
{% if show_skipped %}
<a href="{% url 'sync:media' %}{% if source %}?filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye-slash"></i> Hide skipped media</a>
{% else %}
<a href="{% url 'sync:media' %}?show_skipped=yes{% if source %}&filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye"></i> Show skipped media</a>
{% endif %}
</div>
</div> </div>
{% include 'infobox.html' with message=message %} {% include 'infobox.html' with message=message %}
<div class="row no-margin-bottom"> <div class="row no-margin-bottom">
@@ -48,5 +55,5 @@
</div> </div>
{% endfor %} {% endfor %}
</div> </div>
{% include 'pagination.html' with pagination=sources.paginator filter=source.pk %} {% include 'pagination.html' with pagination=sources.paginator filter=source.pk show_skipped=show_skipped %}
{% endblock %} {% endblock %}

View File

@@ -30,7 +30,7 @@
{% if source.has_failed %} {% if source.has_failed %}
<span class="error-text"><i class="fas fa-exclamation-triangle"></i> <strong>Source has permanent failures</strong></span> <span class="error-text"><i class="fas fa-exclamation-triangle"></i> <strong>Source has permanent failures</strong></span>
{% else %} {% else %}
<strong>{{ source.media_count }}</strong> media items{% if source.delete_old_media and source.days_to_keep > 0 %}, keep {{ source.days_to_keep }} days of media{% endif %} <strong>{{ source.media_count }}</strong> media items, <strong>{{ source.downloaded_count }}</strong> downloaded{% if source.delete_old_media and source.days_to_keep > 0 %}, keeping {{ source.days_to_keep }} days of media{% endif %}
{% endif %} {% endif %}
</a> </a>
{% empty %} {% empty %}

View File

@@ -9,7 +9,6 @@
"average_rating": 1.2345, "average_rating": 1.2345,
"dislike_count": 123, "dislike_count": 123,
"like_count": 456, "like_count": 456,
"playlist_index": 789,
"playlist_title": "test playlist", "playlist_title": "test playlist",
"uploader": "test uploader", "uploader": "test uploader",
"categories":[ "categories":[

View File

@@ -344,21 +344,25 @@ class FrontEndTestCase(TestCase):
}] }]
} }
''' '''
past_date = timezone.make_aware(datetime(year=2000, month=1, day=1))
test_media1 = Media.objects.create( test_media1 = Media.objects.create(
key='mediakey1', key='mediakey1',
source=test_source, source=test_source,
published=past_date,
metadata=test_minimal_metadata metadata=test_minimal_metadata
) )
test_media1_pk = str(test_media1.pk) test_media1_pk = str(test_media1.pk)
test_media2 = Media.objects.create( test_media2 = Media.objects.create(
key='mediakey2', key='mediakey2',
source=test_source, source=test_source,
published=past_date,
metadata=test_minimal_metadata metadata=test_minimal_metadata
) )
test_media2_pk = str(test_media2.pk) test_media2_pk = str(test_media2.pk)
test_media3 = Media.objects.create( test_media3 = Media.objects.create(
key='mediakey3', key='mediakey3',
source=test_source, source=test_source,
published=past_date,
metadata=test_minimal_metadata metadata=test_minimal_metadata
) )
test_media3_pk = str(test_media3.pk) test_media3_pk = str(test_media3.pk)
@@ -535,9 +539,6 @@ class FilepathTestCase(TestCase):
self.source.media_format = 'test-{format}' self.source.media_format = 'test-{format}'
self.assertEqual(self.source.get_example_media_format(), self.assertEqual(self.source.get_example_media_format(),
'test-1080p-vp9-opus') 'test-1080p-vp9-opus')
self.source.media_format = 'test-{playlist_index}'
self.assertEqual(self.source.get_example_media_format(),
'test-1')
self.source.media_format = 'test-{playlist_title}' self.source.media_format = 'test-{playlist_title}'
self.assertEqual(self.source.get_example_media_format(), self.assertEqual(self.source.get_example_media_format(),
'test-Some Playlist Title') 'test-Some Playlist Title')
@@ -1161,14 +1162,14 @@ class FormatMatchingTestCase(TestCase):
('2160p', 'VP9', False, True): (True, '337'), # Exact match, hdr ('2160p', 'VP9', False, True): (True, '337'), # Exact match, hdr
('2160p', 'VP9', True, False): (True, '315'), # Exact match, 60fps ('2160p', 'VP9', True, False): (True, '315'), # Exact match, 60fps
('2160p', 'VP9', True, True): (True, '337'), # Exact match, 60fps+hdr ('2160p', 'VP9', True, True): (True, '337'), # Exact match, 60fps+hdr
('4320P', 'AVC1', False, False): (False, False), ('4320p', 'AVC1', False, False): (False, False),
('4320P', 'AVC1', False, True): (False, False), ('4320p', 'AVC1', False, True): (False, False),
('4320P', 'AVC1', True, False): (False, False), ('4320p', 'AVC1', True, False): (False, False),
('4320P', 'AVC1', True, True): (False, False), ('4320p', 'AVC1', True, True): (False, False),
('4320P', 'VP9', False, False): (False, False), ('4320p', 'VP9', False, False): (False, False),
('4320P', 'VP9', False, True): (False, False), ('4320p', 'VP9', False, True): (False, False),
('4320P', 'VP9', True, False): (True, '272'), # Exact match, 60fps ('4320p', 'VP9', True, False): (True, '272'), # Exact match, 60fps
('4320P', 'VP9', True, True): (False, False), ('4320p', 'VP9', True, True): (False, False),
} }
for params, expected in expected_matches.items(): for params, expected in expected_matches.items():
resolution, vcodec, prefer_60fps, prefer_hdr = params resolution, vcodec, prefer_60fps, prefer_hdr = params
@@ -1367,14 +1368,14 @@ class FormatMatchingTestCase(TestCase):
('2160p', 'VP9', False, True): (True, '337'), # Exact match, hdr ('2160p', 'VP9', False, True): (True, '337'), # Exact match, hdr
('2160p', 'VP9', True, False): (True, '315'), # Exact match, 60fps ('2160p', 'VP9', True, False): (True, '315'), # Exact match, 60fps
('2160p', 'VP9', True, True): (True, '337'), # Exact match, 60fps+hdr ('2160p', 'VP9', True, True): (True, '337'), # Exact match, 60fps+hdr
('4320P', 'AVC1', False, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams) ('4320p', 'AVC1', False, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
('4320P', 'AVC1', False, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams) ('4320p', 'AVC1', False, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
('4320P', 'AVC1', True, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams) ('4320p', 'AVC1', True, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
('4320P', 'AVC1', True, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams) ('4320p', 'AVC1', True, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
('4320P', 'VP9', False, False): (False, '272'), # Fallback, 60fps (no other 8k streams) ('4320p', 'VP9', False, False): (False, '272'), # Fallback, 60fps (no other 8k streams)
('4320P', 'VP9', False, True): (False, '272'), # Fallback, 60fps (no other 8k streams) ('4320p', 'VP9', False, True): (False, '272'), # Fallback, 60fps (no other 8k streams)
('4320P', 'VP9', True, False): (True, '272'), # Exact match, 60fps ('4320p', 'VP9', True, False): (True, '272'), # Exact match, 60fps
('4320P', 'VP9', True, True): (False, '272'), # Fallback, 60fps (no other 8k streams) ('4320p', 'VP9', True, True): (False, '272'), # Fallback, 60fps (no other 8k streams)
} }
for params, expected in expected_matches.items(): for params, expected in expected_matches.items():
resolution, vcodec, prefer_60fps, prefer_hdr = params resolution, vcodec, prefer_60fps, prefer_hdr = params

View File

@@ -10,7 +10,7 @@ from django.views.generic.detail import SingleObjectMixin
from django.http import HttpResponse from django.http import HttpResponse
from django.urls import reverse_lazy from django.urls import reverse_lazy
from django.db import IntegrityError from django.db import IntegrityError
from django.db.models import Q, Count, Sum from django.db.models import Q, Count, Sum, When, Case
from django.forms import ValidationError from django.forms import ValidationError
from django.utils.text import slugify from django.utils.text import slugify
from django.utils import timezone from django.utils import timezone
@@ -78,6 +78,7 @@ class DashboardView(TemplateView):
# Config and download locations # Config and download locations
data['config_dir'] = str(settings.CONFIG_BASE_DIR) data['config_dir'] = str(settings.CONFIG_BASE_DIR)
data['downloads_dir'] = str(settings.DOWNLOAD_ROOT) data['downloads_dir'] = str(settings.DOWNLOAD_ROOT)
data['database_connection'] = settings.DATABASE_CONNECTION_STR
return data return data
@@ -104,7 +105,10 @@ class SourcesView(ListView):
def get_queryset(self): def get_queryset(self):
all_sources = Source.objects.all().order_by('name') all_sources = Source.objects.all().order_by('name')
return all_sources.annotate(media_count=Count('media_source')) return all_sources.annotate(
media_count=Count('media_source'),
downloaded_count=Count(Case(When(media_source__downloaded=True, then=1)))
)
def get_context_data(self, *args, **kwargs): def get_context_data(self, *args, **kwargs):
data = super().get_context_data(*args, **kwargs) data = super().get_context_data(*args, **kwargs)
@@ -438,6 +442,7 @@ class MediaView(ListView):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.filter_source = None self.filter_source = None
self.show_skipped = False
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def dispatch(self, request, *args, **kwargs): def dispatch(self, request, *args, **kwargs):
@@ -447,13 +452,22 @@ class MediaView(ListView):
self.filter_source = Source.objects.get(pk=filter_by) self.filter_source = Source.objects.get(pk=filter_by)
except Source.DoesNotExist: except Source.DoesNotExist:
self.filter_source = None self.filter_source = None
show_skipped = request.GET.get('show_skipped', '').strip()
if show_skipped == 'yes':
self.show_skipped = True
return super().dispatch(request, *args, **kwargs) return super().dispatch(request, *args, **kwargs)
def get_queryset(self): def get_queryset(self):
if self.filter_source: if self.filter_source:
if self.show_skipped:
q = Media.objects.filter(source=self.filter_source) q = Media.objects.filter(source=self.filter_source)
else: else:
q = Media.objects.filter(source=self.filter_source, skip=False)
else:
if self.show_skipped:
q = Media.objects.all() q = Media.objects.all()
else:
q = Media.objects.filter(skip=False)
return q.order_by('-published', '-created') return q.order_by('-published', '-created')
def get_context_data(self, *args, **kwargs): def get_context_data(self, *args, **kwargs):
@@ -464,6 +478,7 @@ class MediaView(ListView):
message = str(self.messages.get('filter', '')) message = str(self.messages.get('filter', ''))
data['message'] = message.format(name=self.filter_source.name) data['message'] = message.format(name=self.filter_source.name)
data['source'] = self.filter_source data['source'] = self.filter_source
data['show_skipped'] = self.show_skipped
return data return data

View File

@@ -8,7 +8,7 @@ import os
from django.conf import settings from django.conf import settings
from copy import copy from copy import copy
from common.logger import log from common.logger import log
import youtube_dl import yt_dlp
_youtubedl_cachedir = getattr(settings, 'YOUTUBE_DL_CACHEDIR', None) _youtubedl_cachedir = getattr(settings, 'YOUTUBE_DL_CACHEDIR', None)
@@ -19,7 +19,7 @@ if _youtubedl_cachedir:
class YouTubeError(youtube_dl.utils.DownloadError): class YouTubeError(yt_dlp.utils.DownloadError):
''' '''
Generic wrapped error for all errors that could be raised by youtube-dl. Generic wrapped error for all errors that could be raised by youtube-dl.
''' '''
@@ -41,11 +41,16 @@ def get_media_info(url):
'extract_flat': True, 'extract_flat': True,
}) })
response = {} response = {}
with youtube_dl.YoutubeDL(opts) as y: with yt_dlp.YoutubeDL(opts) as y:
try: try:
response = y.extract_info(url, download=False) response = y.extract_info(url, download=False)
except youtube_dl.utils.DownloadError as e: except yt_dlp.utils.DownloadError as e:
raise YouTubeError(f'Failed to extract_info for "{url}": {e}') from e raise YouTubeError(f'Failed to extract_info for "{url}": {e}') from e
if not response:
raise YouTubeError(f'Failed to extract_info for "{url}": No metadata was '
f'returned by youtube-dl, check for error messages in the '
f'logs above. This task will be retried later with an '
f'exponential backoff.')
return response return response
@@ -94,9 +99,9 @@ def download_media(url, media_format, extension, output_file):
'quiet': True, 'quiet': True,
'progress_hooks': [hook], 'progress_hooks': [hook],
}) })
with youtube_dl.YoutubeDL(opts) as y: with yt_dlp.YoutubeDL(opts) as y:
try: try:
return y.download([url]) return y.download([url])
except youtube_dl.utils.DownloadError as e: except yt_dlp.utils.DownloadError as e:
raise YouTubeError(f'Failed to download for "{url}": {e}') from e raise YouTubeError(f'Failed to download for "{url}": {e}') from e
return False return False

View File

@@ -0,0 +1,22 @@
import importlib
from django.conf import settings
from django.db.backends.utils import CursorWrapper
def patch_ensure_connection():
for name, config in settings.DATABASES.items():
module = importlib.import_module(config['ENGINE'] + '.base')
def ensure_connection(self):
if self.connection is not None:
try:
with CursorWrapper(self.create_cursor(), self) as cursor:
cursor.execute('SELECT 1;')
return
except Exception:
pass
with self.wrap_database_errors:
self.connect()
module.DatabaseWrapper.ensure_connection = ensure_connection

View File

@@ -1,5 +1,7 @@
import os import os
from pathlib import Path from pathlib import Path
from common.logger import log
from common.utils import parse_database_connection_string
BASE_DIR = Path(__file__).resolve().parent.parent BASE_DIR = Path(__file__).resolve().parent.parent
@@ -21,12 +23,31 @@ FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', None)
TIME_ZONE = os.getenv('TZ', 'UTC') TIME_ZONE = os.getenv('TZ', 'UTC')
database_dict = {}
database_connection_env = os.getenv('DATABASE_CONNECTION', '')
if database_connection_env:
database_dict = parse_database_connection_string(database_connection_env)
if database_dict:
log.info(f'Using database connection: {database_dict["ENGINE"]}://'
f'{database_dict["USER"]}:[hidden]@{database_dict["HOST"]}:'
f'{database_dict["PORT"]}/{database_dict["NAME"]}')
DATABASES = {
'default': database_dict,
}
DATABASE_CONNECTION_STR = (f'{database_dict["DRIVER"]} at "{database_dict["HOST"]}:'
f'{database_dict["PORT"]}" database '
f'"{database_dict["NAME"]}"')
else:
DATABASES = { DATABASES = {
'default': { 'default': {
'ENGINE': 'django.db.backends.sqlite3', 'ENGINE': 'django.db.backends.sqlite3',
'NAME': CONFIG_BASE_DIR / 'db.sqlite3', 'NAME': CONFIG_BASE_DIR / 'db.sqlite3',
} }
} }
DATABASE_CONNECTION_STR = f'sqlite at "{DATABASES["default"]["NAME"]}"'
DEFAULT_THREADS = 1 DEFAULT_THREADS = 1
MAX_BACKGROUND_TASK_ASYNC_THREADS = 8 MAX_BACKGROUND_TASK_ASYNC_THREADS = 8
@@ -38,3 +59,15 @@ if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS:
MEDIA_ROOT = CONFIG_BASE_DIR / 'media' MEDIA_ROOT = CONFIG_BASE_DIR / 'media'
DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR
YOUTUBE_DL_CACHEDIR = CONFIG_BASE_DIR / 'cache' YOUTUBE_DL_CACHEDIR = CONFIG_BASE_DIR / 'cache'
BASICAUTH_USERNAME = os.getenv('HTTP_USER', '').strip()
BASICAUTH_PASSWORD = os.getenv('HTTP_PASS', '').strip()
if BASICAUTH_USERNAME and BASICAUTH_PASSWORD:
BASICAUTH_DISABLE = False
BASICAUTH_USERS = {
BASICAUTH_USERNAME: BASICAUTH_PASSWORD,
}
else:
BASICAUTH_DISABLE = True
BASICAUTH_USERS = {}

View File

@@ -16,6 +16,7 @@ DATABASES = {
'NAME': CONFIG_BASE_DIR / 'db.sqlite3', 'NAME': CONFIG_BASE_DIR / 'db.sqlite3',
} }
} }
DATABASE_CONNECTION_STR = f'sqlite at "{DATABASES["default"]["NAME"]}"'
DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR / 'downloads' DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR / 'downloads'

View File

@@ -6,7 +6,7 @@ CONFIG_BASE_DIR = BASE_DIR
DOWNLOADS_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR
VERSION = 0.9 VERSION = '0.10.0'
SECRET_KEY = '' SECRET_KEY = ''
DEBUG = False DEBUG = False
ALLOWED_HOSTS = [] ALLOWED_HOSTS = []
@@ -37,6 +37,7 @@ MIDDLEWARE = [
'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware',
'whitenoise.middleware.WhiteNoiseMiddleware', 'whitenoise.middleware.WhiteNoiseMiddleware',
'common.middleware.MaterializeDefaultFieldsMiddleware', 'common.middleware.MaterializeDefaultFieldsMiddleware',
'common.middleware.BasicAuthMiddleware',
] ]
@@ -75,6 +76,9 @@ WSGI_APPLICATION = 'tubesync.wsgi.application'
DATABASES = {} DATABASES = {}
DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
AUTH_PASSWORD_VALIDATORS = [ AUTH_PASSWORD_VALIDATORS = [
{ {
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
@@ -117,11 +121,17 @@ Disallow: /
X_FRAME_OPTIONS = 'SAMEORIGIN' X_FRAME_OPTIONS = 'SAMEORIGIN'
BASICAUTH_DISABLE = True
BASICAUTH_REALM = 'Authenticate to TubeSync'
BASICAUTH_ALWAYS_ALLOW_URIS = ('/healthcheck',)
BASICAUTH_USERS = {}
HEALTHCHECK_FIREWALL = True HEALTHCHECK_FIREWALL = True
HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',) HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',)
MAX_ATTEMPTS = 10 # Number of times tasks will be retried MAX_ATTEMPTS = 15 # Number of times tasks will be retried
MAX_RUN_TIME = 1800 # Maximum amount of time in seconds a task can run MAX_RUN_TIME = 1800 # Maximum amount of time in seconds a task can run
BACKGROUND_TASK_RUN_ASYNC = True # Run tasks async in the background BACKGROUND_TASK_RUN_ASYNC = True # Run tasks async in the background
BACKGROUND_TASK_ASYNC_THREADS = 1 # Number of async tasks to run at once BACKGROUND_TASK_ASYNC_THREADS = 1 # Number of async tasks to run at once
@@ -131,7 +141,7 @@ COMPLETED_TASKS_DAYS_TO_KEEP = 7 # Number of days to keep completed t
SOURCES_PER_PAGE = 100 SOURCES_PER_PAGE = 100
MEDIA_PER_PAGE = 72 MEDIA_PER_PAGE = 144
TASKS_PER_PAGE = 100 TASKS_PER_PAGE = 100
@@ -149,6 +159,7 @@ YOUTUBE_DEFAULTS = {
'age_limit': 99, # 'Age in years' to spoof 'age_limit': 99, # 'Age in years' to spoof
'ignoreerrors': True, # Skip on errors (such as unavailable videos in playlists) 'ignoreerrors': True, # Skip on errors (such as unavailable videos in playlists)
'cachedir': False, # Disable on-disk caching 'cachedir': False, # Disable on-disk caching
'addmetadata': True, # Embed metadata during postprocessing where available
} }
@@ -161,3 +172,7 @@ except ImportError as e:
import sys import sys
sys.stderr.write(f'Unable to import local_settings: {e}\n') sys.stderr.write(f'Unable to import local_settings: {e}\n')
sys.exit(1) sys.exit(1)
from .dbutils import patch_ensure_connection
patch_ensure_connection()