135 Commits

Author SHA1 Message Date
meeb
e8d75a79c5 fix release build 2021-09-20 17:27:39 +10:00
meeb
ff4be7cfa0 fix release build 2021-09-20 17:14:59 +10:00
meeb
c1cb19259e bump to 0.10.0 2021-09-20 17:08:22 +10:00
meeb
837b6c3107 switch to yt-dlp with required library updates, resolves #145 2021-09-20 16:49:03 +10:00
meeb
ced6314a62 bump libs 2021-09-20 16:25:01 +10:00
meeb
bb6c195ae7 bump libs 2021-09-20 00:18:52 +10:00
meeb
c280b76777 bump libs 2021-09-11 14:58:35 +10:00
meeb
248da767b0 bump libs 2021-09-08 15:19:02 +10:00
meeb
1069b87295 remove outdated image version from readme 2021-09-04 20:09:39 +10:00
meeb
3525a65cd6 add webp support, part of Pillow lib upgrade, discussed in #150 2021-09-04 19:06:23 +10:00
meeb
c51a5bb365 bump libs 2021-09-04 19:00:31 +10:00
meeb
7f4b9aff14 add libjpeg to container build, resolves #149 2021-09-04 17:13:21 +10:00
meeb
a59e7fe65f remove broken playlist_index support in media formatting options, resolves #142 2021-09-04 15:22:45 +10:00
meeb
3e0a71f2ef add cli commands to list and delete sources, useful for deleting massive sources that currently time out as discussed in #148 2021-09-04 15:00:23 +10:00
meeb
3dfbca2af4 update container build to support pillow>=3, add webp image support 2021-09-03 14:46:13 +10:00
meeb
0c256f59d8 bump libs 2021-09-03 14:32:31 +10:00
meeb
dbbae72c25 add yt-dlp to build 2021-09-03 14:31:17 +10:00
meeb
b1b852d82c add check for stale PID file or old processes to gracefully handle hard container stop/starts, discussed in #144 2021-08-27 17:13:25 +10:00
meeb
437bb17f75 bump libs 2021-08-27 04:06:29 +10:00
meeb
fdfcb5fd33 bump libs 2021-08-14 01:16:21 +10:00
meeb
ff35f791f6 handle media having no metadata at all gracefully, resolves #138 2021-08-07 03:01:44 +10:00
meeb
b2ea37ffec bump libs 2021-08-03 15:53:41 +10:00
meeb
d89530d5b8 Merge pull request #136 from hangrymuppet/patch-1
Add libpq5 to docker image
2021-08-03 15:45:56 +10:00
H Bajaj
f00050008b Add libpq5 to docker image 2021-08-02 22:29:10 -07:00
meeb
68604d19c7 bump libs 2021-07-22 18:09:27 +10:00
meeb
55e5b5632f bump libs 2021-07-11 17:11:17 +10:00
meeb
5e18cb92dd bump libs 2021-07-04 16:05:55 +10:00
meeb
6178e0baa0 bump libs 2021-06-28 17:39:34 +10:00
meeb
8050bac507 add libpq-dev to container build 2021-06-19 15:36:39 +10:00
meeb
6dcdac1647 add postgresql-common to container build for new postgresql-binary 2021-06-19 15:30:59 +10:00
meeb
763f6b89ef bump libs 2021-06-19 15:22:19 +10:00
meeb
6c28292918 bump youtube-dl 2021-06-11 16:21:24 +10:00
meeb
574fc55a5e bump django 2021-06-04 14:03:39 +10:00
meeb
c8fd74b3a4 Merge branch 'main' of github.com:meeb/tubesync into main 2021-06-02 15:13:00 +10:00
meeb
6622e17a5a bump libs 2021-06-02 15:12:50 +10:00
meeb
ea05bd0b13 Merge pull request #125 from Bad3r/main
README.md - Line: 15 - Spelling Error
2021-06-02 15:08:30 +10:00
Bad3r
019c98dc76 Spelling Error
implemenations --> implementations
2021-06-01 18:26:08 -04:00
meeb
72dfe51a46 use env vars in image names and tags 2021-05-27 20:49:48 +10:00
meeb
22cebba8ac remove explicit Dockerfile path 2021-05-27 20:33:52 +10:00
meeb
d51d198f94 remove $ in yaml docker tags 2021-05-27 20:28:50 +10:00
meeb
ed0c2d7dd3 switch to actions v2 format 2021-05-27 20:26:11 +10:00
meeb
5ced901ae8 streamline multi-arch build, part of #81 2021-05-27 20:12:02 +10:00
meeb
afda481046 custom builder just building amd64 for now pending multi-arch ffmpeg releases being available, part of #81 2021-05-27 20:01:29 +10:00
meeb
a986864f77 add custom docker builder, part of #81 2021-05-27 19:56:21 +10:00
meeb
ad1c4ecbc9 add multi-platform container image builds to latest image build process, part of #81 2021-05-27 19:49:41 +10:00
meeb
54b7de4442 add multi-platform container image builds to ci process, part of #81 2021-05-27 19:48:11 +10:00
meeb
d1996aee80 add addmetadata as a default youtube-dl flag, part of #111 2021-05-27 19:22:56 +10:00
meeb
326cefbec1 add hack to force database connection to stay alive by hooking into DatabaseWrapper.ensure_connection, resolves #121 2021-05-27 19:12:29 +10:00
meeb
d6e81c6af7 tweak can_download evaluation logic so post-metadata checks trigger a save if required, part of #108 2021-05-27 18:57:43 +10:00
meeb
a000f8f2c0 update libs 2021-05-27 18:43:24 +10:00
meeb
cbab09e931 bump libs 2021-05-22 03:01:29 +10:00
meeb
414fca08ca bump libs 2021-05-21 01:44:27 +10:00
meeb
874c71b7aa add mysql utf8mb4 encoding note to docs, resolves #117 2021-05-17 15:28:43 +10:00
meeb
5b101825f5 update tests 2021-05-17 01:30:04 +10:00
meeb
0db8db4351 force mysql to use utf8mb4 for connections, related to #117 2021-05-17 01:27:50 +10:00
meeb
d4fd148089 update django 2021-05-15 15:00:20 +10:00
meeb
c739d594d8 update libs 2021-05-06 23:10:45 +10:00
meeb
05e8ad8e89 update libs 2021-05-05 15:49:59 +10:00
meeb
024ab72e5f update libs 2021-05-04 14:25:09 +10:00
meeb
66ec3a29ec update tests to match changes in #115 2021-04-30 14:03:36 +10:00
meeb
28a565737f set database connections CONN_MAX_AGE to a default of 300, resolves #115 2021-04-30 14:00:04 +10:00
meeb
2c7116f6ba fix AutoField warning in django >= 3.2, part of #115 2021-04-30 13:55:33 +10:00
meeb
9ccb9db6de add missing libmariadb3 package, update gunicorn package hash, resolves #113 2021-04-28 23:20:21 +10:00
meeb
2d992cbb90 update libs 2021-04-26 13:17:05 +10:00
meeb
302a3614cf update libs 2021-04-25 13:32:48 +10:00
meeb
ea546013de bump libs 2021-04-21 15:43:08 +10:00
meeb
fb18610893 bump libs 2021-04-07 21:00:38 +10:00
meeb
2364432088 bump libs 2021-04-06 11:31:36 +10:00
meeb
655bed14fd fix loaddata import example in docs 2021-04-05 23:59:48 +10:00
meeb
721399f665 fix container build deps for python mysql client, related to #72 2021-04-05 00:31:50 +10:00
meeb
694ed5c581 update example database settings 2021-04-04 23:09:05 +10:00
meeb
a98f2462ed tweak docs 2021-04-04 23:06:44 +10:00
meeb
5461a5357d check dicts output of parse_database_connection_string() in tests, part of #72 2021-04-04 23:04:51 +10:00
meeb
20df9f4044 support external postgresql, mysql and mariadb databases, resolves #72 2021-04-04 23:01:15 +10:00
meeb
3ec4f7c525 fix test running in Makefile 2021-04-04 22:16:22 +10:00
meeb
443fb827d0 update libs 2021-04-02 21:28:14 +11:00
meeb
a810303f52 bump libs 2021-03-30 11:54:01 +11:00
meeb
9370a481f9 Merge pull request #100 from micahmo/main
Related to #88: Fix 4320p in test data.
2021-03-18 12:19:45 +11:00
Micah Morrison
1478c95d59 Related to #88: Fix 4320p in test data.
RESOLUTION_MAP expects lowercase p in resolution name.
2021-03-17 17:19:15 -04:00
meeb
f69fa747af fix download cap date comparison check in signal, resolves #97 2021-03-15 01:13:16 +11:00
meeb
a29a92893f Merge pull request #95 from micahmo/main
Fix meeb/tubesync#90: Match media on source and id
2021-03-13 23:05:45 +11:00
Micah Morrison
7d471056c1 Fix meeb/tubesync#90: Match media on source and id 2021-03-11 20:37:44 -05:00
meeb
119493c181 disable warning on skipping an already skipped video with no publish date, related to #77 2021-03-08 13:26:34 +11:00
meeb
02a0f924b4 typo 2021-03-08 13:08:38 +11:00
meeb
38665eb00d add a secondary check when download tasks are triggered for download caps, related to #77 2021-03-08 12:20:44 +11:00
meeb
c32358bcef media requires a published date, update tests to match, related to #77 2021-03-08 12:08:10 +11:00
meeb
df9316bede handle media which may not have a published date set for some erroneous reason, related to #77 2021-03-08 11:55:23 +11:00
meeb
8525d920a0 update ffmpeg to 4.3.2, resolves #75 2021-03-07 17:38:35 +11:00
meeb
a6e08d9a10 update s6-init to 2.2.0.3, resolves #76 2021-03-07 17:12:40 +11:00
meeb
2e0d0385b0 recalculate media skip flag when source download cap is updated, resolves #77 2021-03-07 14:03:19 +11:00
meeb
972c184c70 add longer source indexing options and a never option, resolves #68 2021-03-07 13:39:49 +11:00
meeb
adeafbfcb4 update libs 2021-03-07 13:10:10 +11:00
meeb
2c1c45e829 bump to 0.9.1 2021-03-05 14:20:23 +11:00
meeb
c64f54bcb4 bump to 0.9.1 2021-03-05 14:16:17 +11:00
meeb
6ce55b0337 bump libs 2021-03-04 15:39:51 +11:00
meeb
d06c4beae0 bump upstream libs 2021-02-22 19:22:30 +11:00
meeb
db651e16b9 raise a manual exception when youtube-dl extract_info returns no data to trigger backoffs, increase backoff retry and timers, resolves #66 2021-02-22 13:24:11 +11:00
meeb
86068790ed missing import 2021-02-21 11:52:14 +11:00
meeb
ea72671351 add version details for http basic auth instructions 2021-02-21 11:45:56 +11:00
meeb
96b9eddf43 add a reset tasks cli command 2021-02-21 11:44:52 +11:00
meeb
bceefc8b01 skip media which has no publish date in locally stored metadata 2021-02-21 11:15:57 +11:00
meeb
820cc69937 typo 2021-02-19 14:40:41 +11:00
meeb
1e8711be51 add media items downloaded counter to sources list overview 2021-02-19 14:37:56 +11:00
meeb
e3423bc2d2 preserve media filter when toggling skipped media 2021-02-19 14:26:01 +11:00
meeb
6fbf72d0e7 optional basic HTTP authentication, resolves #62 2021-02-19 12:58:34 +11:00
meeb
d6852bf828 account for metadata loading as None for upload_date, resolves #59 2021-02-18 20:50:13 +11:00
meeb
f6f4f244d7 hide skipped media by default and add a show skipped media button 2021-02-18 19:34:44 +11:00
meeb
df35aa2a5f increase media per page, tweak pagination button layout 2021-02-18 19:21:41 +11:00
meeb
799c0fce39 bump to v0.9 2021-02-18 17:29:03 +11:00
meeb
2f324f28a9 add sync-missing-metadata command with docs, resolves #25 2021-02-18 17:24:14 +11:00
meeb
895bfe6f87 scrub % in titles, resolves #55 2021-02-18 16:35:10 +11:00
meeb
e0669b107d change default filename date prefix from YYYYMMDD to YYYY-MM-DD 2021-02-18 16:31:27 +11:00
meeb
0dc201b293 guide tweak 2021-02-18 16:26:58 +11:00
meeb
82fa0f6bce add sync.Source.download_media master flag, add manual import existing media command with docs, resolves #24 2021-02-18 16:24:24 +11:00
meeb
8b93cb4a59 handle malformed responses passively 2021-02-18 14:55:40 +11:00
meeb
647254d7f7 set X-Frame-Options to SAMEORIGIN by default, resolves #51 2021-02-17 22:25:13 +11:00
meeb
3567e20600 fix typing issue when videos have no votes and their vote count is None not 0 in metadata, resolves #50 2021-02-17 22:14:29 +11:00
meeb
5348e25303 update libs 2021-02-17 20:08:06 +11:00
meeb
749df3f7bb switch to using flat indexing of media, only index metadata for media once, resolves #38 and dramatically reduces crawl requests to youtube 2021-02-02 17:24:19 +11:00
meeb
2c2f53e5b2 update packages 2021-02-02 15:05:53 +11:00
meeb
06cfafb803 assert the healthcheck.py exec permissions are preserved, should resolve #41 2021-01-25 13:35:42 +11:00
meeb
f5a37f2e86 update deps 2021-01-25 13:15:43 +11:00
meeb
36747a47e0 create FUNDING.yml 2021-01-22 15:40:26 +11:00
meeb
ffd69e8d40 bump to v0.8 2021-01-20 18:13:28 +11:00
meeb
eebef3371f fix form label padding from overlapping field, resolves #37 2021-01-20 18:03:24 +11:00
meeb
4cd6701c8a fix bug handling audio fallback detection, resolves #31 2021-01-20 18:00:28 +11:00
meeb
4ebe6f2a37 pin footer at the bottom of the viewport, resolves #26 2021-01-20 17:42:16 +11:00
meeb
d553d58fde update youtube-dl 2021-01-20 17:34:50 +11:00
meeb
df40a1367a sanitise youtube video titles for use in sane filenames, resolves #35 2021-01-20 17:34:19 +11:00
meeb
607ee77e70 update deps 2021-01-16 03:50:16 +11:00
meeb
9af493aa8a update upstream library versions 2021-01-03 15:18:31 +11:00
meeb
f0c94ff789 Merge pull request #29 from ltomes/main
Added format descriptions.
2020-12-30 15:40:41 +11:00
Levi Tomes
39c7799831 Changed PR intent
Changed PR intent
2020-12-29 22:36:43 -06:00
Levi Tomes
da7371f830 Added format descriptions
Added format descriptions to the media format help_text.
2020-12-29 21:50:04 -06:00
meeb
387cfefc8f issues with dupe background async worker threads, drop defaults back down to 1 worker 2020-12-21 03:04:54 +11:00
51 changed files with 1552 additions and 1529 deletions

1
.github/FUNDING.yml vendored Normal file
View File

@@ -0,0 +1 @@
github: [meeb]

View File

@@ -35,13 +35,19 @@ jobs:
containerise:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Build the container image
run: docker build . --tag $IMAGE_NAME
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Log into GitHub Container Registry
run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
- name: Push image to GitHub Container Registry
run: |
LATEST_TAG=ghcr.io/meeb/$IMAGE_NAME:latest
docker tag $IMAGE_NAME $LATEST_TAG
docker push $LATEST_TAG
- name: Build and push
uses: docker/build-push-action@v2
with:
platforms: linux/amd64
push: true
tags: ghcr.io/meeb/${{ env.IMAGE_NAME }}:latest
cache-from: type=registry,ref=ghcr.io/meeb/${{ env.IMAGE_NAME }}:latest
cache-to: type=inline
build-args: |
IMAGE_NAME=${{ env.IMAGE_NAME }}

View File

@@ -11,18 +11,23 @@ jobs:
containerise:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Get tag
id: vars
run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
- name: Build the container image
run: docker build . --tag $IMAGE_NAME
id: tag
uses: dawidd6/action-get-tag@v1
- uses: docker/build-push-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Log into GitHub Container Registry
run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
- name: Push image to GitHub Container Registry
env:
RELEASE_TAG: ${{ steps.vars.outputs.tag }}
run: |
REF_TAG=ghcr.io/meeb/$IMAGE_NAME:$RELEASE_TAG
docker tag $IMAGE_NAME $REF_TAG
docker push $REF_TAG
- name: Build and push
uses: docker/build-push-action@v2
with:
platforms: linux/amd64
push: true
tags: ghcr.io/meeb/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
cache-from: type=registry,ref=ghcr.io/meeb/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
cache-to: type=inline
build-args: |
IMAGE_NAME=${{ env.IMAGE_NAME }}

View File

@@ -1,8 +1,8 @@
FROM debian:buster-slim
ARG ARCH="amd64"
ARG S6_VERSION="2.1.0.2"
ARG FFMPEG_VERSION="4.3.1"
ARG S6_VERSION="2.2.0.3"
ARG FFMPEG_VERSION="4.3.2"
ENV DEBIAN_FRONTEND="noninteractive" \
HOME="/root" \
@@ -10,9 +10,9 @@ ENV DEBIAN_FRONTEND="noninteractive" \
LANG="en_US.UTF-8" \
LC_ALL="en_US.UTF-8" \
TERM="xterm" \
S6_EXPECTED_SHA256="52460473413601ff7a84ae690b161a074217ddc734990c2cdee9847166cf669e" \
S6_EXPECTED_SHA256="a7076cf205b331e9f8479bbb09d9df77dbb5cd8f7d12e9b74920902e0c16dd98" \
S6_DOWNLOAD="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${ARCH}.tar.gz" \
FFMPEG_EXPECTED_SHA256="47d95c0129fba27d051748a442a44a73ce1bd38d1e3f9fe1e9dd7258c7581fa5" \
FFMPEG_EXPECTED_SHA256="34bffcd0b58695e3ee5eba2573b37f06cb5088050733ca96265815f58bd61d35" \
FFMPEG_DOWNLOAD="https://tubesync.sfo2.digitaloceanspaces.com/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static.tar.xz"
@@ -57,11 +57,28 @@ WORKDIR /app
# Set up the app
RUN set -x && \
apt-get update && \
# Install required distro packages
apt-get -y install nginx-light && \
apt-get -y --no-install-recommends install python3 python3-setuptools python3-pip python3-dev gcc make && \
apt-get -y --no-install-recommends install \
python3 \
python3-setuptools \
python3-pip \
python3-dev \
gcc \
make \
default-libmysqlclient-dev \
libmariadb3 \
postgresql-common \
libpq-dev \
libpq5 \
libjpeg62-turbo \
libwebp6 \
libjpeg-dev \
zlib1g-dev \
libwebp-dev && \
# Install pipenv
pip3 --disable-pip-version-check install pipenv && \
pip3 --disable-pip-version-check install wheel pipenv && \
# Create a 'app' user which the application will run as
groupadd app && \
useradd -M -d /app -s /bin/false -g app app && \
@@ -82,7 +99,17 @@ RUN set -x && \
rm /app/Pipfile.lock && \
pipenv --clear && \
pip3 --disable-pip-version-check uninstall -y pipenv wheel virtualenv && \
apt-get -y autoremove --purge python3-pip python3-dev gcc make && \
apt-get -y autoremove --purge \
python3-pip \
python3-dev \
gcc \
make \
default-libmysqlclient-dev \
postgresql-common \
libpq-dev \
libjpeg-dev \
zlib1g-dev \
libwebp-dev && \
apt-get -y autoremove && \
apt-get -y autoclean && \
rm -rf /var/lib/apt/lists/* && \

View File

@@ -30,4 +30,4 @@ runcontainer:
test:
$(python) app/manage.py test --verbosity=2
cd tubesync && $(python) manage.py test --verbosity=2 && cd ..

View File

@@ -14,9 +14,12 @@ whitenoise = "*"
gunicorn = "*"
django-compressor = "*"
httptools = "*"
youtube-dl = "*"
django-background-tasks = "*"
requests = "*"
django-basicauth = "*"
psycopg2-binary = "*"
mysqlclient = "*"
yt-dlp = "*"
[requires]
python_version = "3"

375
Pipfile.lock generated
View File

@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "a4bb556fc61ee4583f9588980450b071814298ee4d1a1023fad149c14d14aaba"
"sha256": "ac12e45a1719945b2e19d4a12b03136225f1f5e81affd1adf44a7b3c8dd36b8a"
},
"pipfile-spec": 6,
"requires": {
@@ -18,32 +18,34 @@
"default": {
"asgiref": {
"hashes": [
"sha256:5ee950735509d04eb673bd7f7120f8fa1c9e2df495394992c73234d526907e17",
"sha256:7162a3cb30ab0609f1a4c95938fd73e8604f63bdba516a7f7d64b83ff09478f0"
"sha256:4ef1ab46b484e3c706329cedeff284a5d40824200638503f5768edb6de7d58e9",
"sha256:ffc141aa908e6f175673e7b1b3b7af4fdb0ecb738fc5c8b88f69f055c2415214"
],
"version": "==3.3.1"
"markers": "python_version >= '3.6'",
"version": "==3.4.1"
},
"certifi": {
"hashes": [
"sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
"sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
"sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
"sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
],
"version": "==2020.12.5"
"version": "==2021.5.30"
},
"chardet": {
"charset-normalizer": {
"hashes": [
"sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
"sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
"sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6",
"sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f"
],
"version": "==4.0.0"
"markers": "python_version >= '3'",
"version": "==2.0.6"
},
"django": {
"hashes": [
"sha256:5c866205f15e7a7123f1eec6ab939d22d5bde1416635cab259684af66d8e48a2",
"sha256:edb10b5c45e7e9c0fb1dc00b76ec7449aca258a39ffd613dbd078c51d19c9f03"
"sha256:95b318319d6997bac3595517101ad9cc83fe5672ac498ba48d1a410f47afecd2",
"sha256:e93c93565005b37ddebf2396b4dc4b6913c1838baa82efdfb79acedd5816c240"
],
"index": "pypi",
"version": "==3.1.4"
"version": "==3.2.7"
},
"django-appconf": {
"hashes": [
@@ -59,6 +61,14 @@
"index": "pypi",
"version": "==1.2.5"
},
"django-basicauth": {
"hashes": [
"sha256:15e9e366f698f53c71b1e794dafea060f990a2ac556bae6b7330dd25324a091c",
"sha256:e5e47d1acdc1943bedcc1bf673059d6c15e257dfe9eef67a22fb824f79546c0d"
],
"index": "pypi",
"version": "==0.5.3"
},
"django-compat": {
"hashes": [
"sha256:3ac9a3bedc56b9365d9eb241bc5157d0c193769bf995f9a78dc1bc24e7c2331b"
@@ -67,111 +77,236 @@
},
"django-compressor": {
"hashes": [
"sha256:57ac0a696d061e5fc6fbc55381d2050f353b973fb97eee5593f39247bc0f30af",
"sha256:d2ed1c6137ddaac5536233ec0a819e14009553fee0a869bea65d03e5285ba74f"
"sha256:3358077605c146fdcca5f9eaffb50aa5dbe15f238f8854679115ebf31c0415e0",
"sha256:f8313f59d5e65712fc28787d084fe834997c9dfa92d064a1a3ec3d3366594d04"
],
"index": "pypi",
"version": "==2.4"
"version": "==2.4.1"
},
"django-sass-processor": {
"hashes": [
"sha256:9b46a12ca8bdcb397d46fbcc49e6a926ff9f76a93c5efeb23b495419fd01fc7a"
"sha256:1f043180c47754018e803a77da003377f5ea6558de57cd6946eb27a32e9c16a2",
"sha256:dcaad47c591a2d52689c1bd209259e922e902d886293f0d5c9e0d1a4eb85eda2"
],
"index": "pypi",
"version": "==0.8.2"
"version": "==1.0.1"
},
"gunicorn": {
"hashes": [
"sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626",
"sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"
"sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e",
"sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"
],
"index": "pypi",
"version": "==20.0.4"
"version": "==20.1.0"
},
"httptools": {
"hashes": [
"sha256:0a4b1b2012b28e68306575ad14ad5e9120b34fccd02a81eb08838d7e3bbb48be",
"sha256:3592e854424ec94bd17dc3e0c96a64e459ec4147e6d53c0a42d0ebcef9cb9c5d",
"sha256:41b573cf33f64a8f8f3400d0a7faf48e1888582b6f6e02b82b9bd4f0bf7497ce",
"sha256:56b6393c6ac7abe632f2294da53f30d279130a92e8ae39d8d14ee2e1b05ad1f2",
"sha256:86c6acd66765a934e8730bf0e9dfaac6fdcf2a4334212bd4a0a1c78f16475ca6",
"sha256:96da81e1992be8ac2fd5597bf0283d832287e20cb3cfde8996d2b00356d4e17f",
"sha256:96eb359252aeed57ea5c7b3d79839aaa0382c9d3149f7d24dd7172b1bcecb009",
"sha256:a2719e1d7a84bb131c4f1e0cb79705034b48de6ae486eb5297a139d6a3296dce",
"sha256:ac0aa11e99454b6a66989aa2d44bca41d4e0f968e395a0a8f164b401fefe359a",
"sha256:bc3114b9edbca5a1eb7ae7db698c669eb53eb8afbbebdde116c174925260849c",
"sha256:fa3cd71e31436911a44620473e873a256851e1f53dee56669dae403ba41756a4",
"sha256:fea04e126014169384dee76a153d4573d90d0cbd1d12185da089f73c78390437"
"sha256:04114db99605c9b56ea22a8ec4d7b1485b908128ed4f4a8f6438489c428da794",
"sha256:074afd8afdeec0fa6786cd4a1676e0c0be23dc9a017a86647efa6b695168104f",
"sha256:113816f9af7dcfc4aa71ebb5354d77365f666ecf96ac7ff2aa1d24b6bca44165",
"sha256:1a8f26327023fa1a947d36e60a0582149e182fbbc949c8a65ec8665754dbbe69",
"sha256:2119fa619a4c53311f594f25c0205d619350fcb32140ec5057f861952e9b2b4f",
"sha256:21e948034f70e47c8abfa2d5e6f1a5661f87a2cddc7bcc70f61579cc87897c70",
"sha256:32a10a5903b5bc0eb647d01cd1e95bec3bb614a9bf53f0af1e01360b2debdf81",
"sha256:3787c1f46e9722ef7f07ea5c76b0103037483d1b12e34a02c53ceca5afa4e09a",
"sha256:3f82eb106e1474c63dba36a176067e65b48385f4cecddf3616411aa5d1fbdfec",
"sha256:3f9b4856d46ba1f0c850f4e84b264a9a8b4460acb20e865ec00978ad9fbaa4cf",
"sha256:4137137de8976511a392e27bfdcf231bd926ac13d375e0414e927b08217d779e",
"sha256:4687dfc116a9f1eb22a7d797f0dc6f6e17190d406ca4e729634b38aa98044b17",
"sha256:47dba2345aaa01b87e4981e8756af441349340708d5b60712c98c55a4d28f4af",
"sha256:5a836bd85ae1fb4304f674808488dae403e136d274aa5bafd0e6ee456f11c371",
"sha256:6e676bc3bb911b11f3d7e2144b9a53600bf6b9b21e0e4437aa308e1eef094d97",
"sha256:72ee0e3fb9c6437ab3ae34e9abee67fcee6876f4f58504e3f613dd5882aafdb7",
"sha256:79717080dc3f8b1eeb7f820b9b81528acbc04be6041f323fdd97550da2062575",
"sha256:8ac842df4fc3952efa7820b277961ea55e068bbc54cb59a0820400de7ae358d8",
"sha256:9f475b642c48b1b78584bdd12a5143e2c512485664331eade9c29ef769a17598",
"sha256:b8ac7dee63af4346e02b1e6d32202e3b5b3706a9928bec6da6d7a5b066217422",
"sha256:c0ac2e0ce6733c55858932e7d37fcc7b67ba6bb23e9648593c55f663de031b93",
"sha256:c14576b737d9e6e4f2a86af04918dbe9b62f57ce8102a8695c9a382dbe405c7f",
"sha256:cdc3975db86c29817e6d13df14e037c931fc893a710fb71097777a4147090068",
"sha256:eda95634027200f4b2a6d499e7c2e7fa9b8ee57e045dfda26958ea0af27c070b"
],
"index": "pypi",
"version": "==0.1.1"
"version": "==0.3.0"
},
"idna": {
"hashes": [
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
"sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a",
"sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"
],
"version": "==2.10"
"markers": "python_version >= '3'",
"version": "==3.2"
},
"libsass": {
"hashes": [
"sha256:1521d2a8d4b397c6ec90640a1f6b5529077035efc48ef1c2e53095544e713d1b",
"sha256:1b2d415bbf6fa7da33ef46e549db1418498267b459978eff8357e5e823962d35",
"sha256:25ebc2085f5eee574761ccc8d9cd29a9b436fc970546d5ef08c6fa41eb57dff1",
"sha256:2ae806427b28bc1bb7cb0258666d854fcf92ba52a04656b0b17ba5e190fb48a9",
"sha256:4a246e4b88fd279abef8b669206228c92534d96ddcd0770d7012088c408dff23",
"sha256:553e5096414a8d4fb48d0a48f5a038d3411abe254d79deac5e008516c019e63a",
"sha256:697f0f9fa8a1367ca9ec6869437cb235b1c537fc8519983d1d890178614a8903",
"sha256:a8fd4af9f853e8bf42b1425c5e48dd90b504fa2e70d7dac5ac80b8c0a5a5fe85",
"sha256:c9411fec76f480ffbacc97d8188322e02a5abca6fc78e70b86a2a2b421eae8a2",
"sha256:daa98a51086d92aa7e9c8871cf1a8258124b90e2abf4697852a3dca619838618",
"sha256:e0e60836eccbf2d9e24ec978a805cd6642fa92515fbd95e3493fee276af76f8a",
"sha256:e64ae2587f1a683e831409aad03ba547c245ef997e1329fffadf7a866d2510b8",
"sha256:f6852828e9e104d2ce0358b73c550d26dd86cc3a69439438c3b618811b9584f5"
"sha256:06c8776417fe930714bdc930a3d7e795ae3d72be6ac883ff72a1b8f7c49e5ffb",
"sha256:12f39712de38689a8b785b7db41d3ba2ea1d46f9379d81ea4595802d91fa6529",
"sha256:1e25dd9047a9392d3c59a0b869e0404f2b325a03871ee45285ee33b3664f5613",
"sha256:659ae41af8708681fa3ec73f47b9735a6725e71c3b66ff570bfce78952f2314e",
"sha256:6b984510ed94993708c0d697b4fef2d118929bbfffc3b90037be0f5ccadf55e7",
"sha256:a005f298f64624f313a3ac618ab03f844c71d84ae4f4a4aec4b68d2a4ffe75eb",
"sha256:abc29357ee540849faf1383e1746d40d69ed5cb6d4c346df276b258f5aa8977a",
"sha256:d5ba529d9ce668be9380563279f3ffe988f27bc5b299c5a28453df2e0b0fbaf2",
"sha256:e2b1a7d093f2e76dc694c17c0c285e846d0b0deb0e8b21dc852ba1a3a4e2f1d6"
],
"index": "pypi",
"version": "==0.20.1"
"version": "==0.21.0"
},
"mutagen": {
"hashes": [
"sha256:6397602efb3c2d7baebd2166ed85731ae1c1d475abca22090b7141ff5034b3e1",
"sha256:9c9f243fcec7f410f138cb12c21c84c64fde4195481a30c9bfb05b5f003adfed"
],
"markers": "python_version >= '3.5' and python_version < '4'",
"version": "==1.45.1"
},
"mysqlclient": {
"hashes": [
"sha256:0ac0dd759c4ca02c35a9fedc24bc982cf75171651e8187c2495ec957a87dfff7",
"sha256:3381ca1a4f37ff1155fcfde20836b46416d66531add8843f6aa6d968982731c3",
"sha256:71c4b330cf2313bbda0307fc858cc9055e64493ba9bf28454d25cf8b3ee8d7f5",
"sha256:f6ebea7c008f155baeefe16c56cd3ee6239f7a5a9ae42396c2f1860f08a7c432",
"sha256:fc575093cf81b6605bed84653e48b277318b880dc9becf42dd47fa11ffd3e2b6"
],
"index": "pypi",
"version": "==2.0.3"
},
"pillow": {
"hashes": [
"sha256:006de60d7580d81f4a1a7e9f0173dc90a932e3905cc4d47ea909bc946302311a",
"sha256:0a2e8d03787ec7ad71dc18aec9367c946ef8ef50e1e78c71f743bc3a770f9fae",
"sha256:0eeeae397e5a79dc088d8297a4c2c6f901f8fb30db47795113a4a605d0f1e5ce",
"sha256:11c5c6e9b02c9dac08af04f093eb5a2f84857df70a7d4a6a6ad461aca803fb9e",
"sha256:2fb113757a369a6cdb189f8df3226e995acfed0a8919a72416626af1a0a71140",
"sha256:4b0ef2470c4979e345e4e0cc1bbac65fda11d0d7b789dbac035e4c6ce3f98adb",
"sha256:59e903ca800c8cfd1ebe482349ec7c35687b95e98cefae213e271c8c7fffa021",
"sha256:5abd653a23c35d980b332bc0431d39663b1709d64142e3652890df4c9b6970f6",
"sha256:5f9403af9c790cc18411ea398a6950ee2def2a830ad0cfe6dc9122e6d528b302",
"sha256:6b4a8fd632b4ebee28282a9fef4c341835a1aa8671e2770b6f89adc8e8c2703c",
"sha256:6c1aca8231625115104a06e4389fcd9ec88f0c9befbabd80dc206c35561be271",
"sha256:795e91a60f291e75de2e20e6bdd67770f793c8605b553cb6e4387ce0cb302e09",
"sha256:7ba0ba61252ab23052e642abdb17fd08fdcfdbbf3b74c969a30c58ac1ade7cd3",
"sha256:7c9401e68730d6c4245b8e361d3d13e1035cbc94db86b49dc7da8bec235d0015",
"sha256:81f812d8f5e8a09b246515fac141e9d10113229bc33ea073fec11403b016bcf3",
"sha256:895d54c0ddc78a478c80f9c438579ac15f3e27bf442c2a9aa74d41d0e4d12544",
"sha256:8de332053707c80963b589b22f8e0229f1be1f3ca862a932c1bcd48dafb18dd8",
"sha256:92c882b70a40c79de9f5294dc99390671e07fc0b0113d472cbea3fde15db1792",
"sha256:95edb1ed513e68bddc2aee3de66ceaf743590bf16c023fb9977adc4be15bd3f0",
"sha256:b63d4ff734263ae4ce6593798bcfee6dbfb00523c82753a3a03cbc05555a9cc3",
"sha256:bd7bf289e05470b1bc74889d1466d9ad4a56d201f24397557b6f65c24a6844b8",
"sha256:cc3ea6b23954da84dbee8025c616040d9aa5eaf34ea6895a0a762ee9d3e12e11",
"sha256:cc9ec588c6ef3a1325fa032ec14d97b7309db493782ea8c304666fb10c3bd9a7",
"sha256:d3d07c86d4efa1facdf32aa878bd508c0dc4f87c48125cc16b937baa4e5b5e11",
"sha256:d8a96747df78cda35980905bf26e72960cba6d355ace4780d4bdde3b217cdf1e",
"sha256:e38d58d9138ef972fceb7aeec4be02e3f01d383723965bfcef14d174c8ccd039",
"sha256:eb472586374dc66b31e36e14720747595c2b265ae962987261f044e5cce644b5",
"sha256:fbd922f702582cb0d71ef94442bfca57624352622d75e3be7a1e7e9360b07e72"
"sha256:0412516dcc9de9b0a1e0ae25a280015809de8270f134cc2c1e32c4eeb397cf30",
"sha256:04835e68ef12904bc3e1fd002b33eea0779320d4346082bd5b24bec12ad9c3e9",
"sha256:06d1adaa284696785375fa80a6a8eb309be722cf4ef8949518beb34487a3df71",
"sha256:085a90a99404b859a4b6c3daa42afde17cb3ad3115e44a75f0d7b4a32f06a6c9",
"sha256:0b9911ec70731711c3b6ebcde26caea620cbdd9dcb73c67b0730c8817f24711b",
"sha256:10e00f7336780ca7d3653cf3ac26f068fa11b5a96894ea29a64d3dc4b810d630",
"sha256:11c27e74bab423eb3c9232d97553111cc0be81b74b47165f07ebfdd29d825875",
"sha256:11eb7f98165d56042545c9e6db3ce394ed8b45089a67124298f0473b29cb60b2",
"sha256:13654b521fb98abdecec105ea3fb5ba863d1548c9b58831dd5105bb3873569f1",
"sha256:15ccb81a6ffc57ea0137f9f3ac2737ffa1d11f786244d719639df17476d399a7",
"sha256:18a07a683805d32826c09acfce44a90bf474e6a66ce482b1c7fcd3757d588df3",
"sha256:19ec4cfe4b961edc249b0e04b5618666c23a83bc35842dea2bfd5dfa0157f81b",
"sha256:1c3ff00110835bdda2b1e2b07f4a2548a39744bb7de5946dc8e95517c4fb2ca6",
"sha256:27a330bf7014ee034046db43ccbb05c766aa9e70b8d6c5260bfc38d73103b0ba",
"sha256:2b11c9d310a3522b0fd3c35667914271f570576a0e387701f370eb39d45f08a4",
"sha256:2c661542c6f71dfd9dc82d9d29a8386287e82813b0375b3a02983feac69ef864",
"sha256:2cde7a4d3687f21cffdf5bb171172070bb95e02af448c4c8b2f223d783214056",
"sha256:2d5e9dc0bf1b5d9048a94c48d0813b6c96fccfa4ccf276d9c36308840f40c228",
"sha256:2f23b2d3079522fdf3c09de6517f625f7a964f916c956527bed805ac043799b8",
"sha256:35d27687f027ad25a8d0ef45dd5208ef044c588003cdcedf05afb00dbc5c2deb",
"sha256:35d409030bf3bd05fa66fb5fdedc39c521b397f61ad04309c90444e893d05f7d",
"sha256:4326ea1e2722f3dc00ed77c36d3b5354b8fb7399fb59230249ea6d59cbed90da",
"sha256:4abc247b31a98f29e5224f2d31ef15f86a71f79c7f4d2ac345a5d551d6393073",
"sha256:4d89a2e9219a526401015153c0e9dd48319ea6ab9fe3b066a20aa9aee23d9fd3",
"sha256:4e59e99fd680e2b8b11bbd463f3c9450ab799305d5f2bafb74fefba6ac058616",
"sha256:548794f99ff52a73a156771a0402f5e1c35285bd981046a502d7e4793e8facaa",
"sha256:56fd98c8294f57636084f4b076b75f86c57b2a63a8410c0cd172bc93695ee979",
"sha256:59697568a0455764a094585b2551fd76bfd6b959c9f92d4bdec9d0e14616303a",
"sha256:6bff50ba9891be0a004ef48828e012babaaf7da204d81ab9be37480b9020a82b",
"sha256:6cb3dd7f23b044b0737317f892d399f9e2f0b3a02b22b2c692851fb8120d82c6",
"sha256:7dbfbc0020aa1d9bc1b0b8bcf255a7d73f4ad0336f8fd2533fcc54a4ccfb9441",
"sha256:838eb85de6d9307c19c655c726f8d13b8b646f144ca6b3771fa62b711ebf7624",
"sha256:8b68f565a4175e12e68ca900af8910e8fe48aaa48fd3ca853494f384e11c8bcd",
"sha256:8f284dc1695caf71a74f24993b7c7473d77bc760be45f776a2c2f4e04c170550",
"sha256:963ebdc5365d748185fdb06daf2ac758116deecb2277ec5ae98139f93844bc09",
"sha256:a048dad5ed6ad1fad338c02c609b862dfaa921fcd065d747194a6805f91f2196",
"sha256:a1bd983c565f92779be456ece2479840ec39d386007cd4ae83382646293d681b",
"sha256:a66566f8a22561fc1a88dc87606c69b84fa9ce724f99522cf922c801ec68f5c1",
"sha256:bcb04ff12e79b28be6c9988f275e7ab69f01cc2ba319fb3114f87817bb7c74b6",
"sha256:bd24054aaf21e70a51e2a2a5ed1183560d3a69e6f9594a4bfe360a46f94eba83",
"sha256:be25cb93442c6d2f8702c599b51184bd3ccd83adebd08886b682173e09ef0c3f",
"sha256:c691b26283c3a31594683217d746f1dad59a7ae1d4cfc24626d7a064a11197d4",
"sha256:cc9d0dec711c914ed500f1d0d3822868760954dce98dfb0b7382a854aee55d19",
"sha256:ce2e5e04bb86da6187f96d7bab3f93a7877830981b37f0287dd6479e27a10341",
"sha256:ce651ca46d0202c302a535d3047c55a0131a720cf554a578fc1b8a2aff0e7d96",
"sha256:d0c8ebbfd439c37624db98f3877d9ed12c137cadd99dde2d2eae0dab0bbfc355",
"sha256:d675a876b295afa114ca8bf42d7f86b5fb1298e1b6bb9a24405a3f6c8338811c",
"sha256:dde3f3ed8d00c72631bc19cbfff8ad3b6215062a5eed402381ad365f82f0c18c",
"sha256:e5a31c07cea5edbaeb4bdba6f2b87db7d3dc0f446f379d907e51cc70ea375629",
"sha256:f514c2717012859ccb349c97862568fdc0479aad85b0270d6b5a6509dbc142e2",
"sha256:fc0db32f7223b094964e71729c0361f93db43664dd1ec86d3df217853cedda87",
"sha256:fd4fd83aa912d7b89b4b4a1580d30e2a4242f3936882a3f433586e5ab97ed0d5",
"sha256:feb5db446e96bfecfec078b943cc07744cc759893cef045aa8b8b6d6aaa8274e"
],
"index": "pypi",
"version": "==8.0.1"
"version": "==8.3.2"
},
"psycopg2-binary": {
"hashes": [
"sha256:0b7dae87f0b729922e06f85f667de7bf16455d411971b2043bbd9577af9d1975",
"sha256:0f2e04bd2a2ab54fa44ee67fe2d002bb90cee1c0f1cc0ebc3148af7b02034cbd",
"sha256:123c3fb684e9abfc47218d3784c7b4c47c8587951ea4dd5bc38b6636ac57f616",
"sha256:1473c0215b0613dd938db54a653f68251a45a78b05f6fc21af4326f40e8360a2",
"sha256:14db1752acdd2187d99cb2ca0a1a6dfe57fc65c3281e0f20e597aac8d2a5bd90",
"sha256:1e3a362790edc0a365385b1ac4cc0acc429a0c0d662d829a50b6ce743ae61b5a",
"sha256:1e85b74cbbb3056e3656f1cc4781294df03383127a8114cbc6531e8b8367bf1e",
"sha256:20f1ab44d8c352074e2d7ca67dc00843067788791be373e67a0911998787ce7d",
"sha256:2f62c207d1740b0bde5c4e949f857b044818f734a3d57f1d0d0edc65050532ed",
"sha256:3242b9619de955ab44581a03a64bdd7d5e470cc4183e8fcadd85ab9d3756ce7a",
"sha256:35c4310f8febe41f442d3c65066ca93cccefd75013df3d8c736c5b93ec288140",
"sha256:4235f9d5ddcab0b8dbd723dca56ea2922b485ea00e1dafacf33b0c7e840b3d32",
"sha256:5ced67f1e34e1a450cdb48eb53ca73b60aa0af21c46b9b35ac3e581cf9f00e31",
"sha256:7360647ea04db2e7dff1648d1da825c8cf68dc5fbd80b8fb5b3ee9f068dcd21a",
"sha256:8c13d72ed6af7fd2c8acbd95661cf9477f94e381fce0792c04981a8283b52917",
"sha256:988b47ac70d204aed01589ed342303da7c4d84b56c2f4c4b8b00deda123372bf",
"sha256:995fc41ebda5a7a663a254a1dcac52638c3e847f48307b5416ee373da15075d7",
"sha256:a36c7eb6152ba5467fb264d73844877be8b0847874d4822b7cf2d3c0cb8cdcb0",
"sha256:aed4a9a7e3221b3e252c39d0bf794c438dc5453bc2963e8befe9d4cd324dff72",
"sha256:aef9aee84ec78af51107181d02fe8773b100b01c5dfde351184ad9223eab3698",
"sha256:b0221ca5a9837e040ebf61f48899926b5783668b7807419e4adae8175a31f773",
"sha256:b4d7679a08fea64573c969f6994a2631908bb2c0e69a7235648642f3d2e39a68",
"sha256:c250a7ec489b652c892e4f0a5d122cc14c3780f9f643e1a326754aedf82d9a76",
"sha256:ca86db5b561b894f9e5f115d6a159fff2a2570a652e07889d8a383b5fae66eb4",
"sha256:cfc523edecddaef56f6740d7de1ce24a2fdf94fd5e704091856a201872e37f9f",
"sha256:da113b70f6ec40e7d81b43d1b139b9db6a05727ab8be1ee559f3a69854a69d34",
"sha256:f6fac64a38f6768e7bc7b035b9e10d8a538a9fadce06b983fb3e6fa55ac5f5ce",
"sha256:f8559617b1fcf59a9aedba2c9838b5b6aa211ffedecabca412b92a1ff75aac1a",
"sha256:fbb42a541b1093385a2d8c7eec94d26d30437d0e77c1d25dae1dcc46741a385e"
],
"index": "pypi",
"version": "==2.9.1"
},
"pycryptodome": {
"hashes": [
"sha256:09c1555a3fa450e7eaca41ea11cd00afe7c91fef52353488e65663777d8524e0",
"sha256:12222a5edc9ca4a29de15fbd5339099c4c26c56e13c2ceddf0b920794f26165d",
"sha256:1723ebee5561628ce96748501cdaa7afaa67329d753933296321f0be55358dce",
"sha256:1c5e1ca507de2ad93474be5cfe2bfa76b7cf039a1a32fc196f40935944871a06",
"sha256:2603c98ae04aac675fefcf71a6c87dc4bb74a75e9071ae3923bbc91a59f08d35",
"sha256:2dea65df54349cdfa43d6b2e8edb83f5f8d6861e5cf7b1fbc3e34c5694c85e27",
"sha256:31c1df17b3dc5f39600a4057d7db53ac372f492c955b9b75dd439f5d8b460129",
"sha256:38661348ecb71476037f1e1f553159b80d256c00f6c0b00502acac891f7116d9",
"sha256:3e2e3a06580c5f190df843cdb90ea28d61099cf4924334d5297a995de68e4673",
"sha256:3f840c49d38986f6e17dbc0673d37947c88bc9d2d9dba1c01b979b36f8447db1",
"sha256:501ab36aae360e31d0ec370cf5ce8ace6cb4112060d099b993bc02b36ac83fb6",
"sha256:60386d1d4cfaad299803b45a5bc2089696eaf6cdd56f9fc17479a6f89595cfc8",
"sha256:6260e24d41149268122dd39d4ebd5941e9d107f49463f7e071fd397e29923b0c",
"sha256:6bbf7fee7b7948b29d7e71fcacf48bac0c57fb41332007061a933f2d996f9713",
"sha256:6d2df5223b12437e644ce0a3be7809471ffa71de44ccd28b02180401982594a6",
"sha256:758949ca62690b1540dfb24ad773c6da9cd0e425189e83e39c038bbd52b8e438",
"sha256:77997519d8eb8a4adcd9a47b9cec18f9b323e296986528186c0e9a7a15d6a07e",
"sha256:7fd519b89585abf57bf47d90166903ec7b43af4fe23c92273ea09e6336af5c07",
"sha256:98213ac2b18dc1969a47bc65a79a8fca02a414249d0c8635abb081c7f38c91b6",
"sha256:99b2f3fc51d308286071d0953f92055504a6ffe829a832a9fc7a04318a7683dd",
"sha256:9b6f711b25e01931f1c61ce0115245a23cdc8b80bf8539ac0363bdcf27d649b6",
"sha256:a3105a0eb63eacf98c2ecb0eb4aa03f77f40fbac2bdde22020bb8a536b226bb8",
"sha256:a8eb8b6ea09ec1c2535bf39914377bc8abcab2c7d30fa9225eb4fe412024e427",
"sha256:a92d5c414e8ee1249e850789052608f582416e82422502dc0ac8c577808a9067",
"sha256:d3d6958d53ad307df5e8469cc44474a75393a434addf20ecd451f38a72fe29b8",
"sha256:e0a4d5933a88a2c98bbe19c0c722f5483dc628d7a38338ac2cb64a7dbd34064b",
"sha256:e3bf558c6aeb49afa9f0c06cee7fb5947ee5a1ff3bd794b653d39926b49077fa",
"sha256:e61e363d9a5d7916f3a4ce984a929514c0df3daf3b1b2eb5e6edbb131ee771cf",
"sha256:f977cdf725b20f6b8229b0c87acb98c7717e742ef9f46b113985303ae12a99da",
"sha256:fc7489a50323a0df02378bc2fff86eb69d94cc5639914346c736be981c6a02e7"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==3.10.1"
},
"pytz": {
"hashes": [
"sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",
"sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"
"sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
"sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
],
"version": "==2020.4"
"version": "==2021.1"
},
"rcssmin": {
"hashes": [
@@ -181,11 +316,11 @@
},
"requests": {
"hashes": [
"sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
"sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
"sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
"sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
],
"index": "pypi",
"version": "==2.25.1"
"version": "==2.26.0"
},
"rjsmin": {
"hashes": [
@@ -207,40 +342,74 @@
},
"six": {
"hashes": [
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
"sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
"sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
],
"version": "==1.15.0"
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==1.16.0"
},
"sqlparse": {
"hashes": [
"sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0",
"sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8"
"sha256:0c00730c74263a94e5a9919ade150dfc3b19c574389985446148402998287dae",
"sha256:48719e356bb8b42991bdbb1e8b83223757b93789c00910a616a071910ca4a64d"
],
"version": "==0.4.1"
"markers": "python_version >= '3.5'",
"version": "==0.4.2"
},
"urllib3": {
"hashes": [
"sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
"sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
"sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4",
"sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"
],
"version": "==1.26.2"
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
"version": "==1.26.6"
},
"websockets": {
"hashes": [
"sha256:01db0ecd1a0ca6702d02a5ed40413e18b7d22f94afb3bbe0d323bac86c42c1c8",
"sha256:085bb8a6e780d30eaa1ba48ac7f3a6707f925edea787cfb761ce5a39e77ac09b",
"sha256:1ac35426fe3e7d3d0fac3d63c8965c76ed67a8fd713937be072bf0ce22808539",
"sha256:1f6b814cff6aadc4288297cb3a248614829c6e4ff5556593c44a115e9dd49939",
"sha256:2a43072e434c041a99f2e1eb9b692df0232a38c37c61d00e9f24db79474329e4",
"sha256:5b2600e01c7ca6f840c42c747ffbe0254f319594ed108db847eb3d75f4aacb80",
"sha256:62160772314920397f9d219147f958b33fa27a12c662d4455c9ccbba9a07e474",
"sha256:706e200fc7f03bed99ad0574cd1ea8b0951477dd18cc978ccb190683c69dba76",
"sha256:71358c7816e2762f3e4af3adf0040f268e219f5a38cb3487a9d0fc2e554fef6a",
"sha256:7d2e12e4f901f1bc062dfdf91831712c4106ed18a9a4cdb65e2e5f502124ca37",
"sha256:7f79f02c7f9a8320aff7d3321cd1c7e3a7dbc15d922ac996cca827301ee75238",
"sha256:82b17524b1ce6ae7f7dd93e4d18e9b9474071e28b65dbf1dfe9b5767778db379",
"sha256:82bd921885231f4a30d9bc550552495b3fc36b1235add6d374e7c65c3babd805",
"sha256:8bbf8660c3f833ddc8b1afab90213f2e672a9ddac6eecb3cde968e6b2807c1c7",
"sha256:9a4d889162bd48588e80950e07fa5e039eee9deb76a58092e8c3ece96d7ef537",
"sha256:b4ade7569b6fd17912452f9c3757d96f8e4044016b6d22b3b8391e641ca50456",
"sha256:b8176deb6be540a46695960a765a77c28ac8b2e3ef2ec95d50a4f5df901edb1c",
"sha256:c4fc9a1d242317892590abe5b61a9127f1a61740477bfb121743f290b8054002",
"sha256:c5880442f5fc268f1ef6d37b2c152c114deccca73f48e3a8c48004d2f16f4567",
"sha256:cd8c6f2ec24aedace251017bc7a414525171d4e6578f914acab9349362def4da",
"sha256:d67646ddd17a86117ae21c27005d83c1895c0cef5d7be548b7549646372f868a",
"sha256:e42a1f1e03437b017af341e9bbfdc09252cd48ef32a8c3c3ead769eab3b17368",
"sha256:eb282127e9c136f860c6068a4fba5756eb25e755baffb5940b6f1eae071928b2",
"sha256:fe83b3ec9ef34063d86dfe1029160a85f24a5a94271036e5714a57acfdd089a1",
"sha256:ff59c6bdb87b31f7e2d596f09353d5a38c8c8ff571b0e2238e8ee2d55ad68465"
],
"markers": "python_version >= '3.7'",
"version": "==10.0"
},
"whitenoise": {
"hashes": [
"sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7",
"sha256:05d00198c777028d72d8b0bbd234db605ef6d60e9410125124002518a48e515d"
"sha256:d234b871b52271ae7ed6d9da47ffe857c76568f11dd30e28e18c5869dbd11e12",
"sha256:d963ef25639d1417e8a247be36e6aedd8c7c6f0a08adcb5a89146980a96b577c"
],
"index": "pypi",
"version": "==5.2.0"
"version": "==5.3.0"
},
"youtube-dl": {
"yt-dlp": {
"hashes": [
"sha256:65968065e66966955dc79fad9251565fcc982566118756da624bd21467f3a04c",
"sha256:eaa859f15b6897bec21474b7787dc958118c8088e1f24d4ef1d58eab13188958"
"sha256:c97716a715261657345176ab8190a19efa51db0e5b174a6629956548750245e1",
"sha256:ca7e77cdb055ba2683df5b0807aab1c1e120cbe02c8f35d9d3293d94dbdaea63"
],
"index": "pypi",
"version": "==2020.12.14"
"version": "==2021.9.2"
}
},
"develop": {}

View File

@@ -12,7 +12,7 @@ media server, then TubeSync is for you. Internally, TubeSync is a web interface
on `youtube-dl` and `ffmpeg` with a task scheduler.
There are several other web interfaces to YouTube and `youtube-dl` all with varying
features and implemenations. TubeSync's largest difference is full PVR experience of
features and implementations. TubeSync's largest difference is full PVR experience of
updating media servers and better selection of media formats. Additionally, to be as
hands-free as possible, TubeSync has gradual retrying of failures with back-off timers
so media which fails to download will be retried for an extended period making it,
@@ -22,12 +22,9 @@ hopefully, quite reliable.
# Latest container image
```yaml
ghcr.io/meeb/tubesync:v0.7
ghcr.io/meeb/tubesync:latest
```
**NOTE: the `:latest` tag does exist, but will contain in-development commits and may
be broken. Use at your own risk.**
# Screenshots
### Dashboard
@@ -101,8 +98,8 @@ $ mkdir /some/directory/tubesync-downloads
Finally, download and run the container:
```bash
# Pull a versioned image
$ docker pull ghcr.io/meeb/tubesync:v0.7
# Pull image
$ docker pull ghcr.io/meeb/tubesync:latest
# Start the container using your user ID and group ID
$ docker run \
-d \
@@ -113,7 +110,7 @@ $ docker run \
-v /some/directory/tubesync-config:/config \
-v /some/directory/tubesync-downloads:/downloads \
-p 4848:4848 \
ghcr.io/meeb/tubesync:v0.7
ghcr.io/meeb/tubesync:latest
```
Once running, open `http://localhost:4848` in your browser and you should see the
@@ -125,7 +122,7 @@ Alternatively, for Docker Compose, you can use something like:
```yaml
tubesync:
image: ghcr.io/meeb/tubesync:v0.7
image: ghcr.io/meeb/tubesync:latest
container_name: tubesync
restart: unless-stopped
ports:
@@ -139,6 +136,41 @@ Alternatively, for Docker Compose, you can use something like:
- PGID=1000
```
## Optional authentication
Available in `v1.0` (or `:latest`)and later. If you want to enable a basic username and
password to be required to access the TubeSync dashboard you can set them with the
following environment variables:
```bash
HTTP_USER
HTTP_PASS
```
For example in the `docker run ...` line add in:
```bash
...
-e HTTP_USER=some-username \
-e HTTP_PASS=some-secure-password \
...
```
Or in your Docker Compose file you would add in:
```yaml
...
environment:
- HTTP_USER=some-username
- HTTP_PASS=some-secure-password
...
```
When BOTH `HTTP_USER` and `HTTP_PASS` are set then basic HTTP authentication will be
enabled.
# Updating
To update, you can just pull a new version of the container image as they are released.
@@ -196,6 +228,20 @@ $ docker logs --follow tubesync
```
# Advanced usage guides
Once you're happy using TubeSync there are some advanced usage guides for more complex
and less common features:
![Import existing media into TubeSync](https://github.com/meeb/tubesync/blob/main/docs/import-existing-media.md)
![Sync or create missing metadata files](https://github.com/meeb/tubesync/blob/main/docs/create-missing-metadata.md)
![Reset tasks from the command line](https://github.com/meeb/tubesync/blob/main/docs/reset-tasks.md)
![Using PostgreSQL, MySQL or MariaDB as database backends](https://github.com/meeb/tubesync/blob/main/docs/other-database-backends.md)
# Warnings
### 1. Index frequency
@@ -290,7 +336,9 @@ can log in at http://localhost:4848/admin
### Are there user accounts or multi-user support?
No not at the moment. This could be added later if there is demand for it.
There is support for basic HTTP authentication by setting the `HTTP_USER` and
`HTTP_PASS` environment variables. There is not support for multi-user or user
management.
### Does TubeSync support HTTPS?
@@ -309,7 +357,7 @@ There are a number of other environment variables you can set. These are, mostly
useful if you are manually installing TubeSync in some other environment. These are:
| Name | What | Example |
| ------------------------ | ------------------------------------------------------------ | ---------------------------------- |
| ------------------------ | ------------------------------------------------------------ | ------------------------------------ |
| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l |
| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME | /somepath |
| TUBESYNC_DEBUG | Enable debugging | True |
@@ -318,6 +366,9 @@ useful if you are manually installing TubeSync in some other environment. These
| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 |
| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 |
| LISTEN_PORT | Port number for gunicorn to listen on | 8080 |
| HTTP_USER | Sets the username for HTTP basic authentication | some-username |
| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password |
| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database |
# Manual, non-containerised, installation
@@ -334,7 +385,7 @@ installing and running WSGI-based Python web applications before attempting this
`tubesync/tubesync/local_settings.py` and edit it as appropriate
5. Run migrations with `./manage.py migrate`
6. Collect static files with `./manage.py collectstatic`
6. Set up your prefered WSGI server, such as `gunicorn` poiting it to the application
6. Set up your prefered WSGI server, such as `gunicorn` pointing it to the application
in `tubesync/tubesync/wsgi.py`
7. Set up your proxy server such as `nginx` and forward it to the WSGI server
8. Check the web interface is working

View File

@@ -19,8 +19,8 @@ chown -R app:app /app/common/static && \
chmod -R 0750 /app/common/static && \
chown -R app:app /app/static && \
chmod -R 0750 /app/static && \
find /app -type f -exec chmod 640 {} \; && \
chmod +x /app/healthcheck.py
find /app -type f ! -iname healthcheck.py -exec chmod 640 {} \; && \
chmod 0755 /app/healthcheck.py
# Run migrations
exec s6-setuidgid app \

View File

@@ -5,5 +5,20 @@ umask "$UMASK_SET"
cd /app || exit
PIDFILE=/run/app/gunicorn.pid
if [ -f "${PIDFILE}" ]
then
PID=$(cat $PIDFILE)
echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
if kill -0 $PID
then
echo "Killing old gunicorn process with PID: ${PID}"
kill -9 $PID
fi
echo "Removing stale PID file: ${PIDFILE}"
rm ${PIDFILE}
fi
exec s6-setuidgid app \
/usr/local/bin/gunicorn -c /app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application

View File

@@ -0,0 +1,37 @@
# TubeSync
## Advanced usage guide - creating missing metadata
This is a new feature in v0.9 of TubeSync and later. It allows you to create or
re-create missing metadata in your TubeSync download directories for missing `nfo`
files and thumbnails.
If you add a source with "write NFO files" or "copy thumbnails" disabled, download
some media and then update the source to write NFO files or copy thumbnails then
TubeSync will not automatically retroactively attempt to copy or create your missing
metadata files. You can use a special one-off command to manually write missing
metadata files to the correct locations.
## Requirements
You have added a source without metadata writing enabled, downloaded some media, then
updated the source to enable metadata writing.
## Steps
### 1. Run the batch metadata sync command
Execute the following Django command:
`./manage.py sync-missing-metadata`
When deploying TubeSync inside a container, you can execute this with:
`docker exec -ti tubesync python3 /app/manage.py sync-missing-metadata`
This command will log what its doing to the terminal when you run it.
Internally, this command loops over all your sources which have been saved with
"write NFO files" or "copy thumbnails" enabled. Then, loops over all media saved to
that source and confirms that the appropriate thumbnail files have been copied over and
the NFO file has been written if enabled.

View File

@@ -0,0 +1,81 @@
# TubeSync
## Advanced usage guide - importing existing media
This is a new feature in v0.9 of TubeSync and later. It allows you to mark existing
downloaded media as "downloaded" in TubeSync. You can use this feature if, for example,
you already have an extensive catalogue of downloaded media which you want to mark
as downloaded into TubeSync so TubeSync doesn't re-download media you already have.
## Requirements
Your existing downloaded media MUST contain the unique ID. For YouTube videos, this is
means the YouTube video ID MUST be in the filename.
Supported extensions to be imported are .m4a, .ogg, .mkv, .mp3, .mp4 and .avi. Your
media you want to import must end in one of these file extensions.
## Caveats
As TubeSync does not probe media and your existing media may be re-encoded or in
different formats to what is available in the current media metadata there is no way
for TubeSync to know what codecs, resolution, bitrate etc. your imported media is in.
Any manually imported existing local media will display blank boxes for this
information on the TubeSync interface as it's unavailable.
## Steps
### 1. Add your source to TubeSync
Add your source to TubeSync, such as a YouTube channel. **Make sure you untick the
"download media" checkbox.**
This will allow TubeSync to index all the available media on your source, but won't
start downloading any media.
### 2. Wait
Wait for all the media on your source to be indexed. This may take some time.
### 3. Move your existing media into TubeSync
You now need to move your existing media into TubeSync. You need to move the media
files into the correct download directories created by TubeSync. For example, if you
have downloaded videos for a YouTube channel "TestChannel", you would have added this
as a source called TestChannel and in a directory called test-channel in Tubesync. It
would have a download directory created on disk at:
`/path/to/downloads/test-channel`
You would move all of your pre-existing videos you downloaded outside of TubeSync for
this channel into this directory.
In short, your existing media needs to be moved into the correct TubeSync source
directory to be detected.
This is required so TubeSync can known which Source to link the media to.
### 4. Run the batch import command
Execute the following Django command:
`./manage.py import-existing-media`
When deploying TubeSync inside a container, you can execute this with:
`docker exec -ti tubesync python3 /app/manage.py import-existing-media`
This command will log what its doing to the terminal when you run it.
Internally, `import-existing-media` looks for the unique media key (for YouTube, this
is the YouTube video ID) in the filename and detects the source to link it to based
on the directory the media file is inside.
### 5. Re-enable downloading at the source
Edit your source and re-enable / tick the "download media" option. This will allow
TubeSync to download any missing media you did not manually import.
Note that TubeSync will still get screenshots write `nfo` files etc. for files you
manually import if enabled at the source level.

View File

@@ -0,0 +1,80 @@
# TubeSync
## Advanced usage guide - using other database backends
This is a new feature in v1.0 of TubeSync and later. It allows you to use a custom
existing external database server instead of the default SQLite database. You may want
to use this if you encounter performance issues with adding very large or a large
number of channels and database write contention (as shown by errors in the log)
become an issue.
## Requirements
TubeSync supports SQLite (the automatic default) as well as PostgreSQL, MySQL and
MariaDB. For MariaDB just follow the MySQL instructions as the driver is the same.
You should a blank install of TubeSync. Migrating to a new database will reset your
database. If you are comfortable with Django you can export and re-import existing
database data with:
```bash
$ docker exec -ti tubesync python3 /app/manage.py dumpdata > some-file.json
```
Then change you database backend over, then use
```bash
$ cat some-file.json | docker exec -ti tubesync python3 /app/manage.py loaddata --format=json -
```
As detailed in the Django documentation:
https://docs.djangoproject.com/en/3.1/ref/django-admin/#dumpdata
and:
https://docs.djangoproject.com/en/3.1/ref/django-admin/#loaddata
Further instructions are beyond the scope of TubeSync documenation and you should refer
to Django documentation for more details.
If you are not comfortable with the above, then skip the `dumpdata` steps, however
remember you will start again with a completely new database.
## Steps
### 1. Create a database in your external database server
You need to create a database and a user with permissions to access the database in
your chosen external database server. Steps vary between PostgreSQL, MySQL and MariaDB
so this is up to you to work out.
### 2. Set the database connection string environment variable
You need to provide the database connection details to TubeSync via an environment
variable. The environment variable name is `DATABASE_CONNECTION` and the format is the
standard URL-style string. Examples are:
`postgresql://tubesync:password@localhost:5432/tubesync`
and
`mysql://tubesync:password@localhost:3306/tubesync`
*Important note:* For MySQL databases make SURE you create the tubesync database with
`utf8mb4` encoding, like:
`CREATE DATABASE tubesync CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;`
Without `utf8mb4` encoding things like emojis in video titles (or any extended UTF8
characters) can cause issues.
### 3. Start TubeSync and check the logs
Once you start TubeSync with the new database connection you should see the folling log
entry in the container or stdout logs:
`2021-04-04 22:42:17,912 [tubesync/INFO] Using database connection: django.db.backends.postgresql://tubesync:[hidden]@localhost:5432/tubesync`
If you see a line similar to the above and the web interface loads, congratulations,
you are now using an external database server for your TubeSync data!

33
docs/reset-tasks.md Normal file
View File

@@ -0,0 +1,33 @@
# TubeSync
## Advanced usage guide - reset tasks from the command line
This is a new feature in v1.0 of TubeSync and later. It allows you to reset all
scheduled tasks from the command line as well as the "reset tasks" button in the
"tasks" tab of the dashboard.
This is useful for TubeSync installations where you may have a lot of media and
sources added and the "reset tasks" button may take too long to the extent where
the page times out (with a 502 error or similar issue).
## Requirements
You have added some sources and media
## Steps
### 1. Run the reset tasks command
Execute the following Django command:
`./manage.py reset-tasks`
When deploying TubeSync inside a container, you can execute this with:
`docker exec -ti tubesync python3 /app/manage.py reset-tasks`
This command will log what its doing to the terminal when you run it.
When this is run, new tasks will be immediately created so all your sources will be
indexed again straight away, any missing information such as thumbnails will be
redownloaded, etc.

View File

@@ -1,10 +1,10 @@
from django.conf import settings
from .third_party_versions import youtube_dl_version, ffmpeg_version
from .third_party_versions import yt_dlp_version, ffmpeg_version
def app_details(request):
return {
'app_version': str(settings.VERSION),
'youtube_dl_version': youtube_dl_version,
'yt_dlp_version': yt_dlp_version,
'ffmpeg_version': ffmpeg_version,
}

View File

@@ -20,3 +20,10 @@ class DownloadFailedException(Exception):
exist.
'''
pass
class DatabaseConnectionError(Exception):
'''
Raised when parsing or initially connecting to a database.
'''
pass

View File

@@ -1,4 +1,6 @@
from django.conf import settings
from django.forms import BaseForm
from basicauth.middleware import BasicAuthMiddleware as BaseBasicAuthMiddleware
class MaterializeDefaultFieldsMiddleware:
@@ -19,3 +21,12 @@ class MaterializeDefaultFieldsMiddleware:
for _, field in v.fields.items():
field.widget.attrs.update({'class':'browser-default'})
return response
class BasicAuthMiddleware(BaseBasicAuthMiddleware):
def process_request(self, request):
bypass_uris = getattr(settings, 'BASICAUTH_ALWAYS_ALLOW_URIS', [])
if request.path in bypass_uris:
return None
return super().process_request(request)

View File

@@ -4,7 +4,7 @@
}
.help-text {
color: $form-help-text-colour;
padding: 1rem 0 1rem 0;
padding-bottom: 1rem;
}
label {
text-transform: uppercase;

View File

@@ -5,6 +5,13 @@ html {
color: $text-colour;
}
body {
display: flex;
min-height: 100vh;
flex-direction: column;
justify-content: space-between;
}
header {
background-color: $header-background-colour;
@@ -174,8 +181,10 @@ main {
display: inline-block;
font-weight: bold;
text-decoration: none;
padding: 5px 10px 5px 10px;
margin: 0 3px 0 3px;
padding: 5px 8px 4px 8px;
margin: 0 3px 6px 3px;
min-width: 40px;
min-height: 40px;
background-color: $pagination-background-colour;
color: $pagination-text-colour;
border: 2px $pagination-border-colour solid;

View File

@@ -16,6 +16,8 @@
<body>
<div class="app">
<header>
<div class="container">
<a href="{% url 'sync:dashboard' %}">
@@ -43,6 +45,8 @@
</div>
</main>
</div>
<footer>
<div class="container">
<p>
@@ -53,7 +57,7 @@
</p>
<p>
<a href="https://github.com/meeb/tubesync" class="nowrap" target="_blank"><i class="fab fa-github"></i> TubeSync</a> version <strong>{{ app_version }}</strong> with
<a href="https://yt-dl.org/" class="nowrap" target="_blank"><i class="fas fa-link"></i> youtube-dl</a> version <strong>{{ youtube_dl_version }}</strong> and
<a href="https://github.com/yt-dlp/yt-dlp" class="nowrap" target="_blank"><i class="fas fa-link"></i> yt-dlp</a> version <strong>{{ yt_dlp_version }}</strong> and
<a href="https://ffmpeg.org/" class="nowrap" target="_blank"><i class="fas fa-link"></i> FFmpeg</a> version <strong>{{ ffmpeg_version }}</strong>.
</p>
</div>

View File

@@ -3,7 +3,7 @@
<div class="col s12">
<div class="pagination">
{% for i in paginator.page_range %}
<a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}">{{ i }}</a>
<a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}{% if show_skipped %}&show_skipped=yes{% endif %}">{{ i }}</a>
{% endfor %}
</div>
</div>

View File

@@ -2,6 +2,8 @@ import os.path
from django.conf import settings
from django.test import TestCase, Client
from .testutils import prevent_request_warnings
from .utils import parse_database_connection_string
from .errors import DatabaseConnectionError
class ErrorPageTestCase(TestCase):
@@ -61,3 +63,66 @@ class CommonStaticTestCase(TestCase):
favicon_real_path = os.path.join(os.sep.join(root_parts),
os.sep.join(url_parts))
self.assertTrue(os.path.exists(favicon_real_path))
class DatabaseConnectionTestCase(TestCase):
def test_parse_database_connection_string(self):
database_dict = parse_database_connection_string(
'postgresql://tubesync:password@localhost:5432/tubesync')
self.assertEqual(database_dict,
{
'DRIVER': 'postgresql',
'ENGINE': 'django.db.backends.postgresql',
'USER': 'tubesync',
'PASSWORD': 'password',
'HOST': 'localhost',
'PORT': 5432,
'NAME': 'tubesync',
'CONN_MAX_AGE': 300,
'OPTIONS': {},
}
)
database_dict = parse_database_connection_string(
'mysql://tubesync:password@localhost:3306/tubesync')
self.assertEqual(database_dict,
{
'DRIVER': 'mysql',
'ENGINE': 'django.db.backends.mysql',
'USER': 'tubesync',
'PASSWORD': 'password',
'HOST': 'localhost',
'PORT': 3306,
'NAME': 'tubesync',
'CONN_MAX_AGE': 300,
'OPTIONS': {'charset': 'utf8mb4'}
}
)
# Invalid driver
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'test://tubesync:password@localhost:5432/tubesync')
# No username
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'postgresql://password@localhost:5432/tubesync')
# No database name
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'postgresql://tubesync:password@5432')
# Invalid port
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'postgresql://tubesync:password@localhost:test/tubesync')
# Invalid port
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'postgresql://tubesync:password@localhost:65537/tubesync')
# Invalid username or password
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'postgresql://tubesync:password:test@localhost:5432/tubesync')
# Invalid database name
with self.assertRaises(DatabaseConnectionError):
parse_database_connection_string(
'postgresql://tubesync:password@localhost:5432/tubesync/test')

View File

@@ -1,7 +1,7 @@
from youtube_dl import version as yt_version
from yt_dlp import version as yt_dlp_version
youtube_dl_version = str(yt_version.__version__)
yt_dlp_version = str(yt_dlp_version.__version__)
ffmpeg_version = '(shared install)'

View File

@@ -1,4 +1,95 @@
from urllib.parse import urlunsplit, urlencode
from datetime import datetime
from urllib.parse import urlunsplit, urlencode, urlparse
from yt_dlp.utils import LazyList
from .errors import DatabaseConnectionError
def parse_database_connection_string(database_connection_string):
'''
Parses a connection string in a URL style format, such as:
postgresql://tubesync:password@localhost:5432/tubesync
mysql://someuser:somepassword@localhost:3306/tubesync
into a Django-compatible settings.DATABASES dict format.
'''
valid_drivers = ('postgresql', 'mysql')
default_ports = {
'postgresql': 5432,
'mysql': 3306,
}
django_backends = {
'postgresql': 'django.db.backends.postgresql',
'mysql': 'django.db.backends.mysql',
}
backend_options = {
'postgresql': {},
'mysql': {
'charset': 'utf8mb4',
}
}
try:
parts = urlparse(str(database_connection_string))
except Exception as e:
raise DatabaseConnectionError(f'Failed to parse "{database_connection_string}" '
f'as a database connection string: {e}') from e
driver = parts.scheme
user_pass_host_port = parts.netloc
database = parts.path
if driver not in valid_drivers:
raise DatabaseConnectionError(f'Database connection string '
f'"{database_connection_string}" specified an '
f'invalid driver, must be one of {valid_drivers}')
django_driver = django_backends.get(driver)
host_parts = user_pass_host_port.split('@')
if len(host_parts) != 2:
raise DatabaseConnectionError(f'Database connection string netloc must be in '
f'the format of user:pass@host')
user_pass, host_port = host_parts
user_pass_parts = user_pass.split(':')
if len(user_pass_parts) != 2:
raise DatabaseConnectionError(f'Database connection string netloc must be in '
f'the format of user:pass@host')
username, password = user_pass_parts
host_port_parts = host_port.split(':')
if len(host_port_parts) == 1:
# No port number, assign a default port
hostname = host_port_parts[0]
port = default_ports.get(driver)
elif len(host_port_parts) == 2:
# Host name and port number
hostname, port = host_port_parts
try:
port = int(port)
except (ValueError, TypeError) as e:
raise DatabaseConnectionError(f'Database connection string contained an '
f'invalid port, ports must be integers: '
f'{e}') from e
if not 0 < port < 63336:
raise DatabaseConnectionError(f'Database connection string contained an '
f'invalid port, ports must be between 1 and '
f'65535, got {port}')
else:
# Malformed
raise DatabaseConnectionError(f'Database connection host must be a hostname or '
f'a hostname:port combination')
if database.startswith('/'):
database = database[1:]
if not database:
raise DatabaseConnectionError(f'Database connection string path must be a '
f'string in the format of /databasename')
if '/' in database:
raise DatabaseConnectionError(f'Database connection string path can only '
f'contain a single string name, got: {database}')
return {
'DRIVER': driver,
'ENGINE': django_driver,
'NAME': database,
'USER': username,
'PASSWORD': password,
'HOST': hostname,
'PORT': port,
'CONN_MAX_AGE': 300,
'OPTIONS': backend_options.get(driver),
}
def get_client_ip(request):
@@ -14,3 +105,21 @@ def append_uri_params(uri, params):
uri = str(uri)
qs = urlencode(params)
return urlunsplit(('', '', uri, qs, ''))
def clean_filename(filename):
if not isinstance(filename, str):
raise ValueError(f'filename must be a str, got {type(filename)}')
to_scrub = '<>\/:*?"|%'
for char in to_scrub:
filename = filename.replace(char, '')
filename = ''.join([c for c in filename if ord(c) > 30])
return ' '.join(filename.split())
def json_serial(obj):
if isinstance(obj, datetime):
return obj.isoformat()
if isinstance(obj, LazyList):
return list(obj)
raise TypeError(f'Type {type(obj)} is not json_serial()-able')

0
tubesync/healthcheck.py Normal file → Executable file
View File

File diff suppressed because it is too large Load Diff

View File

@@ -7,7 +7,7 @@ class SourceAdmin(admin.ModelAdmin):
ordering = ('-created',)
list_display = ('uuid', 'name', 'source_type', 'last_crawl',
'has_failed')
'download_media', 'has_failed')
readonly_fields = ('uuid', 'created')
search_fields = ('uuid', 'key', 'name')

View File

@@ -0,0 +1,51 @@
import os
import uuid
from django.utils.translation import gettext_lazy as _
from django.core.management.base import BaseCommand, CommandError
from django.db.models import signals
from common.logger import log
from sync.models import Source, Media, MediaServer
from sync.signals import media_post_delete
from sync.tasks import rescan_media_server
class Command(BaseCommand):
help = ('Deletes a source by UUID')
def add_arguments(self, parser):
parser.add_argument('--source', action='store', required=True, help='Source UUID')
def handle(self, *args, **options):
source_uuid_str = options.get('source', '')
try:
source_uuid = uuid.UUID(source_uuid_str)
except Exception as e:
raise CommandError(f'Failed to parse source UUID: {e}')
log.info(f'Deleting source with UUID: {source_uuid}')
# Fetch the source by UUID
try:
source = Source.objects.get(uuid=source_uuid)
except Source.DoesNotExist:
raise CommandError(f'Source does not exist with '
f'UUID: {source_uuid}')
# Detach post-delete signal for Media so we don't spam media servers
signals.post_delete.disconnect(media_post_delete, sender=Media)
# Delete the source, triggering pre-delete signals for each media item
log.info(f'Found source with UUID "{source.uuid}" with name '
f'"{source.name}" and deleting it, this may take some time!')
source.delete()
# Update any media servers
for mediaserver in MediaServer.objects.all():
log.info(f'Scheduling media server updates')
verbose_name = _('Request media server rescan for "{}"')
rescan_media_server(
str(mediaserver.pk),
priority=0,
verbose_name=verbose_name.format(mediaserver),
remove_existing_tasks=True
)
# Re-attach signals
signals.post_delete.connect(media_post_delete, sender=Media)
# All done
log.info('Done')

View File

@@ -0,0 +1,55 @@
import os
from pathlib import Path
from django.core.management.base import BaseCommand, CommandError
from common.logger import log
from sync.models import Source, Media
class Command(BaseCommand):
help = ('Scans download media directories for media not yet downloaded and ',
'marks them as downloaded')
extra_extensions = ['mp3', 'mp4', 'avi']
def handle(self, *args, **options):
log.info('Building directory to Source map...')
dirmap = {}
for s in Source.objects.all():
dirmap[s.directory_path] = s
log.info(f'Scanning sources...')
file_extensions = list(Source.EXTENSIONS) + self.extra_extensions
for sourceroot, source in dirmap.items():
media = list(Media.objects.filter(source=source, downloaded=False,
skip=False))
if not media:
log.info(f'Source "{source}" has no missing media')
continue
log.info(f'Scanning Source "{source}" directory for media to '
f'import: {sourceroot}, looking for {len(media)} '
f'undownloaded and unskipped items')
on_disk = []
for (root, dirs, files) in os.walk(sourceroot):
rootpath = Path(root)
for filename in files:
filepart, ext = os.path.splitext(filename)
if ext.startswith('.'):
ext = ext[1:]
ext = ext.strip().lower()
if ext not in file_extensions:
continue
on_disk.append(str(rootpath / filename))
filemap = {}
for item in media:
for filepath in on_disk:
if item.key in filepath:
# The unique item key is in the file name on disk, map it to
# the undownloaded media item
filemap[filepath] = item
continue
for filepath, item in filemap.items():
log.info(f'Matched on-disk file: {filepath} '
f'to media item: {item.source} / {item}')
item.media_file.name = filepath
item.downloaded = True
item.save()
log.info('Done')

View File

@@ -0,0 +1,15 @@
import os
from django.core.management.base import BaseCommand, CommandError
from common.logger import log
from sync.models import Source, Media, MediaServer
class Command(BaseCommand):
help = ('Lists sources')
def handle(self, *args, **options):
log.info('Listing sources...')
for source in Source.objects.all():
log.info(f' - {source.uuid}: {source.name}')
log.info('Done')

View File

@@ -0,0 +1,33 @@
from django.core.management.base import BaseCommand, CommandError
from django.utils.translation import gettext_lazy as _
from background_task.models import Task
from sync.models import Source
from sync.tasks import index_source_task
from common.logger import log
class Command(BaseCommand):
help = 'Resets all tasks'
def handle(self, *args, **options):
log.info('Resettings all tasks...')
# Delete all tasks
Task.objects.all().delete()
# Iter all tasks
for source in Source.objects.all():
# Recreate the initial indexing task
log.info(f'Resetting tasks for source: {source}')
verbose_name = _('Index media from source "{}"')
index_source_task(
str(source.pk),
repeat=source.index_schedule,
queue=str(source.pk),
priority=5,
verbose_name=verbose_name.format(source.name)
)
# This also chains down to call each Media objects .save() as well
source.save()
log.info('Done')

View File

@@ -0,0 +1,34 @@
import os
from shutil import copyfile
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q
from common.logger import log
from sync.models import Source, Media
from sync.utils import write_text_file
class Command(BaseCommand):
help = 'Syncs missing metadata (such as nfo files) if source settings are updated'
def handle(self, *args, **options):
log.info('Syncing missing metadata...')
sources = Source.objects.filter(Q(copy_thumbnails=True) | Q(write_nfo=True))
for source in sources.order_by('name'):
log.info(f'Finding media for source: {source}')
for item in Media.objects.filter(source=source, downloaded=True):
log.info(f'Checking media for missing metadata: {source} / {item}')
thumbpath = item.thumbpath
if not thumbpath.is_file():
if item.thumb:
log.info(f'Copying missing thumbnail from: {item.thumb.path} '
f'to: {thumbpath}')
copyfile(item.thumb.path, thumbpath)
else:
log.error(f'Tried to copy missing thumbnail for {item} but '
f'the thumbnail has not been downloaded')
nfopath = item.nfopath
if not nfopath.is_file():
log.info(f'Writing missing NFO file: {nfopath}')
write_text_file(nfopath, item.nfoxml)
log.info('Done')

View File

@@ -66,7 +66,7 @@ def get_best_audio_format(media):
# No codecs matched
if media.source.can_fallback:
# Can fallback, find the next highest bitrate non-matching codec
return False, audio_formats[0]
return False, audio_formats[0]['id']
else:
# Can't fallback
return False, False

View File

@@ -0,0 +1,30 @@
# Generated by Django 3.1.6 on 2021-02-18 04:42
import django.core.files.storage
from django.db import migrations, models
import sync.models
class Migration(migrations.Migration):
dependencies = [
('sync', '0008_source_download_cap'),
]
operations = [
migrations.AddField(
model_name='source',
name='download_media',
field=models.BooleanField(default=True, help_text='Download media from this source, if not selected the source will only be indexed', verbose_name='download media'),
),
migrations.AlterField(
model_name='media',
name='media_file',
field=models.FileField(blank=True, help_text='Media file', max_length=200, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'),
),
migrations.AlterField(
model_name='source',
name='media_format',
field=models.CharField(default='{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'),
),
]

View File

@@ -12,6 +12,7 @@ from django.utils.text import slugify
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from common.errors import NoFormatException
from common.utils import clean_filename
from .youtube import (get_media_info as get_youtube_media_info,
download_media as download_youtube_media)
from .utils import seconds_to_timestr, parse_media_format
@@ -100,6 +101,11 @@ class Source(models.Model):
(FALLBACK_NEXT_BEST_HD, _('Get next best resolution but at least HD'))
)
EXTENSION_M4A = 'm4a'
EXTENSION_OGG = 'ogg'
EXTENSION_MKV = 'mkv'
EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
# Fontawesome icons used for the source on the front end
ICONS = {
SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
@@ -112,6 +118,12 @@ class Source(models.Model):
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}',
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
}
# Format used to create indexable URLs
INDEX_URLS = {
SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos',
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos',
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
}
# Callback functions to get a list of media from the source
INDEXERS = {
SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
@@ -146,6 +158,9 @@ class Source(models.Model):
EVERY_6_HOURS = 21600, _('Every 6 hours')
EVERY_12_HOURS = 43200, _('Every 12 hours')
EVERY_24_HOURS = 86400, _('Every 24 hours')
EVERY_3_DAYS = 259200, _('Every 3 days')
EVERY_7_DAYS = 604800, _('Every 7 days')
NEVER = 0, _('Never')
uuid = models.UUIDField(
_('uuid'),
@@ -200,15 +215,20 @@ class Source(models.Model):
_('media format'),
max_length=200,
default=settings.MEDIA_FORMATSTR_DEFAULT,
help_text=_('File format to use for saving files')
help_text=_('File format to use for saving files, detailed options at bottom of page.')
)
index_schedule = models.IntegerField(
_('index schedule'),
choices=IndexSchedule.choices,
db_index=True,
default=IndexSchedule.EVERY_6_HOURS,
default=IndexSchedule.EVERY_24_HOURS,
help_text=_('Schedule of how often to index the source for new media')
)
download_media = models.BooleanField(
_('download media'),
default=True,
help_text=_('Download media from this source, if not selected the source will only be indexed')
)
download_cap = models.IntegerField(
_('download cap'),
choices=CapChoices.choices,
@@ -327,23 +347,32 @@ class Source(models.Model):
'''
if self.is_audio:
if self.source_acodec == self.SOURCE_ACODEC_MP4A:
return 'm4a'
return self.EXTENSION_M4A
elif self.source_acodec == self.SOURCE_ACODEC_OPUS:
return 'ogg'
return self.EXTENSION_OGG
else:
raise ValueError('Unable to choose audio extension, uknown acodec')
else:
return 'mkv'
return self.EXTENSION_MKV
@classmethod
def create_url(obj, source_type, key):
url = obj.URLS.get(source_type)
return url.format(key=key)
@classmethod
def create_index_url(obj, source_type, key):
url = obj.INDEX_URLS.get(source_type)
return url.format(key=key)
@property
def url(self):
return Source.create_url(self.source_type, self.key)
@property
def index_url(self):
return Source.create_index_url(self.source_type, self.key)
@property
def format_summary(self):
if self.source_resolution == Source.SOURCE_RESOLUTION_AUDIO:
@@ -411,7 +440,6 @@ class Source(models.Model):
'title_full': 'Some Media Title Name',
'key': 'SoMeUnIqUiD',
'format': '-'.join(fmt),
'playlist_index': 1,
'playlist_title': 'Some Playlist Title',
'ext': self.extension,
'resolution': self.source_resolution if self.source_resolution else '',
@@ -436,25 +464,10 @@ class Source(models.Model):
indexer = self.INDEXERS.get(self.source_type, None)
if not callable(indexer):
raise Exception(f'Source type f"{self.source_type}" has no indexer')
response = indexer(self.url)
# Account for nested playlists, such as a channel of playlists of playlists
def _recurse_playlists(playlist):
videos = []
if not playlist:
return videos
entries = playlist.get('entries', [])
for entry in entries:
if not entry:
continue
subentries = entry.get('entries', [])
if subentries:
videos = videos + _recurse_playlists(entry)
else:
videos.append(entry)
return videos
return _recurse_playlists(response)
response = indexer(self.index_url)
if not isinstance(response, dict):
return []
return response.get('entries', [])
def get_media_thumb_path(instance, filename):
@@ -480,6 +493,12 @@ class Media(models.Model):
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/watch?v={key}',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}',
}
# Callback functions to get a list of media from the source
INDEXERS = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
}
# Maps standardised names to names used in source metdata
METADATA_FIELDS = {
'upload_date': {
@@ -542,11 +561,6 @@ class Media(models.Model):
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count',
},
'playlist_index': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_index',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_index',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'playlist_index',
},
'playlist_title': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title',
@@ -557,14 +571,18 @@ class Media(models.Model):
STATE_SCHEDULED = 'scheduled'
STATE_DOWNLOADING = 'downloading'
STATE_DOWNLOADED = 'downloaded'
STATE_SKIPPED = 'skipped'
STATE_DISABLED_AT_SOURCE = 'source-disabled'
STATE_ERROR = 'error'
STATES = (STATE_UNKNOWN, STATE_SCHEDULED, STATE_DOWNLOADING, STATE_DOWNLOADED,
STATE_ERROR)
STATE_SKIPPED, STATE_DISABLED_AT_SOURCE, STATE_ERROR)
STATE_ICONS = {
STATE_UNKNOWN: '<i class="far fa-question-circle" title="Unknown download state"></i>',
STATE_SCHEDULED: '<i class="far fa-clock" title="Scheduled to download"></i>',
STATE_DOWNLOADING: '<i class="fas fa-download" title="Downloading now"></i>',
STATE_DOWNLOADED: '<i class="far fa-check-circle" title="Downloaded"></i>',
STATE_SKIPPED: '<i class="fas fa-exclamation-circle" title="Skipped"></i>',
STATE_DISABLED_AT_SOURCE: '<i class="fas fa-stop-circle" title="Media downloading disabled at source"></i>',
STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>',
}
@@ -794,6 +812,20 @@ class Media(models.Model):
hdr = ''
# If the download has completed use existing values
if self.downloaded:
# Check if there's any stored meta data at all
if (not self.downloaded_video_codec and \
not self.downloaded_audio_codec):
# Marked as downloaded but no metadata, imported?
return {
'resolution': resolution,
'height': height,
'width': width,
'vcodec': vcodec,
'acodec': acodec,
'fps': fps,
'hdr': hdr,
'format': tuple(fmt),
}
resolution = f'{self.downloaded_height}p'
if self.downloaded_format != 'audio':
vcodec = self.downloaded_video_codec.lower()
@@ -838,6 +870,7 @@ class Media(models.Model):
fmt.append(resolution)
vcodec = vformat['vcodec'].lower()
fmt.append(vcodec)
if aformat:
acodec = aformat['acodec'].lower()
fmt.append(acodec)
if vformat:
@@ -887,10 +920,9 @@ class Media(models.Model):
'source': self.source.slugname,
'source_full': self.source.name,
'title': self.slugtitle,
'title_full': self.title,
'title_full': clean_filename(self.title),
'key': self.key,
'format': '-'.join(display_format['format']),
'playlist_index': self.playlist_index,
'playlist_title': self.playlist_title,
'ext': self.source.extension,
'resolution': display_format['resolution'],
@@ -902,10 +934,17 @@ class Media(models.Model):
'hdr': display_format['hdr'],
}
@property
def has_metadata(self):
return self.metadata is not None
@property
def loaded_metadata(self):
try:
return json.loads(self.metadata)
data = json.loads(self.metadata)
if not isinstance(data, dict):
return {}
return data
except Exception as e:
return {}
@@ -942,7 +981,10 @@ class Media(models.Model):
@property
def upload_date(self):
field = self.get_metadata_field('upload_date')
try:
upload_date_str = self.loaded_metadata.get(field, '').strip()
except (AttributeError, ValueError) as e:
return None
try:
return datetime.strptime(upload_date_str, '%Y%m%d')
except (AttributeError, ValueError) as e:
@@ -974,8 +1016,12 @@ class Media(models.Model):
def votes(self):
field = self.get_metadata_field('upvotes')
upvotes = self.loaded_metadata.get(field, 0)
if not isinstance(upvotes, int):
upvotes = 0
field = self.get_metadata_field('downvotes')
downvotes = self.loaded_metadata.get(field, 0)
if not isinstance(downvotes, int):
downvotes = 0
return upvotes + downvotes
@property
@@ -993,11 +1039,6 @@ class Media(models.Model):
field = self.get_metadata_field('formats')
return self.loaded_metadata.get(field, [])
@property
def playlist_index(self):
field = self.get_metadata_field('playlist_index')
return self.loaded_metadata.get(field, 0)
@property
def playlist_title(self):
field = self.get_metadata_field('playlist_title')
@@ -1005,7 +1046,7 @@ class Media(models.Model):
@property
def filename(self):
# Otherwise, create a suitable filename from the source media_format
# Create a suitable filename from the source media_format
media_format = str(self.source.media_format)
media_details = self.format_dict
return media_format.format(**media_details)
@@ -1161,6 +1202,10 @@ class Media(models.Model):
return self.STATE_ERROR
else:
return self.STATE_SCHEDULED
if self.skip:
return self.STATE_SKIPPED
if not self.source.download_media:
return self.STATE_DISABLED_AT_SOURCE
return self.STATE_UNKNOWN
def get_download_state_icon(self, task=None):
@@ -1178,6 +1223,16 @@ class Media(models.Model):
# Return the download paramaters
return format_str, self.source.extension
def index_metadata(self):
'''
Index the media metadata returning a dict of info.
'''
indexer = self.INDEXERS.get(self.source.source_type, None)
if not callable(indexer):
raise Exception(f'Meida with source type f"{self.source.source_type}" '
f'has no indexer')
return indexer(self.url)
class MediaServer(models.Model):
'''

View File

@@ -8,8 +8,9 @@ from background_task.models import Task
from common.logger import log
from .models import Source, Media, MediaServer
from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task,
download_media_thumbnail, map_task_to_instance,
check_source_directory_exists, download_media, rescan_media_server)
download_media_thumbnail, download_media_metadata,
map_task_to_instance, check_source_directory_exists,
download_media, rescan_media_server)
from .utils import delete_file
@@ -46,6 +47,7 @@ def source_post_save(sender, instance, created, **kwargs):
priority=0,
verbose_name=verbose_name.format(instance.name)
)
if instance.index_schedule > 0:
delete_task_by_source('sync.tasks.index_source_task', instance.pk)
log.info(f'Scheduling media indexing for source: {instance.name}')
verbose_name = _('Index media from source "{}"')
@@ -91,18 +93,69 @@ def task_task_failed(sender, task_id, completed_task, **kwargs):
@receiver(post_save, sender=Media)
def media_post_save(sender, instance, created, **kwargs):
# Triggered after media is saved, Recalculate the "can_download" flag, this may
# Triggered after media is saved
cap_changed = False
can_download_changed = False
# Reset the skip flag if the download cap has changed if the media has not
# already been downloaded
if not instance.downloaded:
max_cap_age = instance.source.download_cap_date
published = instance.published
if not published:
if not instance.skip:
log.warn(f'Media: {instance.source} / {instance} has no published date '
f'set, marking to be skipped')
instance.skip = True
cap_changed = True
else:
log.debug(f'Media: {instance.source} / {instance} has no published date '
f'set but is already marked to be skipped')
else:
if max_cap_age:
if published > max_cap_age and instance.skip:
# Media was published after the cap date but is set to be skipped
log.info(f'Media: {instance.source} / {instance} has a valid '
f'publishing date, marking to be unskipped')
instance.skip = False
cap_changed = True
elif published <= max_cap_age and not instance.skip:
log.info(f'Media: {instance.source} / {instance} is too old for '
f'the download cap date, marking to be skipped')
instance.skip = True
cap_changed = True
else:
if instance.skip:
# Media marked to be skipped but source download cap removed
log.info(f'Media: {instance.source} / {instance} has a valid '
f'publishing date, marking to be unskipped')
instance.skip = False
cap_changed = True
# Recalculate the "can_download" flag, this may
# need to change if the source specifications have been changed
post_save.disconnect(media_post_save, sender=Media)
if instance.metadata:
if instance.get_format_str():
if not instance.can_download:
instance.can_download = True
instance.save()
can_download_changed = True
else:
if instance.can_download:
instance.can_download = False
can_download_changed = True
# Save the instance if any changes were required
if cap_changed or can_download_changed:
post_save.disconnect(media_post_save, sender=Media)
instance.save()
post_save.connect(media_post_save, sender=Media)
# If the media is missing metadata schedule it to be downloaded
if not instance.metadata:
log.info(f'Scheduling task to download metadata for: {instance.url}')
verbose_name = _('Downloading metadata for "{}"')
download_media_metadata(
str(instance.pk),
priority=10,
verbose_name=verbose_name.format(instance.pk),
remove_existing_tasks=True
)
# If the media is missing a thumbnail schedule it to be downloaded
if not instance.thumb_file_exists:
instance.thumb = None
@@ -124,7 +177,8 @@ def media_post_save(sender, instance, created, **kwargs):
if not instance.media_file_exists:
instance.downloaded = False
instance.media_file = None
if not instance.downloaded and instance.can_download and not instance.skip:
if (not instance.downloaded and instance.can_download and not instance.skip
and instance.source.download_media):
delete_task_by_media('sync.tasks.download_media', (str(instance.pk),))
verbose_name = _('Downloading media for "{}"')
download_media(

View File

@@ -10,7 +10,7 @@ import math
import uuid
from io import BytesIO
from hashlib import sha1
from datetime import timedelta
from datetime import timedelta, datetime
from shutil import copyfile
from PIL import Image
from django.conf import settings
@@ -22,6 +22,7 @@ from background_task import background
from background_task.models import Task, CompletedTask
from common.logger import log
from common.errors import NoMediaException, DownloadFailedException
from common.utils import json_serial
from .models import Source, Media, MediaServer
from .utils import (get_remote_image, resize_image_to_height, delete_file,
write_text_file)
@@ -175,34 +176,10 @@ def index_source_task(source_id):
# Video has no unique key (ID), it can't be indexed
continue
try:
media = Media.objects.get(key=key)
media = Media.objects.get(key=key, source=source)
except Media.DoesNotExist:
media = Media(key=key)
media.source = source
media.metadata = json.dumps(video)
upload_date = media.upload_date
# Media must have a valid upload date
if upload_date:
media.published = timezone.make_aware(upload_date)
else:
log.error(f'Media has no upload date, skipping: {source} / {media}')
continue
# If the source has a download cap date check the upload date is allowed
max_cap_age = source.download_cap_date
if max_cap_age:
if media.published < max_cap_age:
# Media was published after the cap date, skip it
log.warn(f'Media: {source} / {media} is older than cap age '
f'{max_cap_age}, skipping')
continue
# If the source has a cut-off check the upload date is within the allowed delta
if source.delete_old_media and source.days_to_keep > 0:
delta = timezone.now() - timedelta(days=source.days_to_keep)
if media.published < delta:
# Media was published after the cutoff date, skip it
log.warn(f'Media: {source} / {media} is older than '
f'{source.days_to_keep} days, skipping')
continue
try:
media.save()
log.info(f'Indexed media: {source} / {media}')
@@ -234,6 +211,61 @@ def check_source_directory_exists(source_id):
source.make_directory()
@background(schedule=0)
def download_media_metadata(media_id):
'''
Downloads the metadata for a media item.
'''
try:
media = Media.objects.get(pk=media_id)
except Media.DoesNotExist:
# Task triggered but the media no longer exists, do nothing
log.error(f'Task download_media_metadata(pk={media_id}) called but no '
f'media exists with ID: {media_id}')
return
source = media.source
metadata = media.index_metadata()
media.metadata = json.dumps(metadata, default=json_serial)
upload_date = media.upload_date
# Media must have a valid upload date
if upload_date:
media.published = timezone.make_aware(upload_date)
else:
log.error(f'Media has no upload date, skipping: {source} / {media}')
media.skip = True
# If the source has a download cap date check the upload date is allowed
max_cap_age = source.download_cap_date
if max_cap_age:
if media.published < max_cap_age:
# Media was published after the cap date, skip it
log.warn(f'Media: {source} / {media} is older than cap age '
f'{max_cap_age}, skipping')
media.skip = True
# If the source has a cut-off check the upload date is within the allowed delta
if source.delete_old_media and source.days_to_keep > 0:
if not isinstance(media.published, datetime):
# Media has no known published date or incomplete metadata
log.warn(f'Media: {source} / {media} has no published date, skipping')
media.skip = True
else:
delta = timezone.now() - timedelta(days=source.days_to_keep)
if media.published < delta:
# Media was published after the cutoff date, skip it
log.warn(f'Media: {source} / {media} is older than '
f'{source.days_to_keep} days, skipping')
media.skip = True
# Check we can download the media item
if not media.skip:
if media.get_format_str():
media.can_download = True
else:
media.can_download = False
# Save the media
media.save()
log.info(f'Saved {len(media.metadata)} bytes of metadata for: '
f'{source} / {media_id}')
@background(schedule=0)
def download_media_thumbnail(media_id, url):
'''
@@ -279,8 +311,27 @@ def download_media(media_id):
return
if media.skip:
# Media was toggled to be skipped after the task was scheduled
log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but it '
f'is now marked to be skipped, not downloading')
log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
f'it is now marked to be skipped, not downloading')
return
if media.downloaded and media.media_file:
# Media has been marked as downloaded before the download_media task was fired,
# skip it
log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
f'it has already been marked as downloaded, not downloading again')
return
if not media.source.download_media:
log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
f'the source {media.source} has since been marked to not download, '
f'not downloading')
return
max_cap_age = media.source.download_cap_date
published = media.published
if max_cap_age and published:
if published <= max_cap_age:
log.warn(f'Download task triggered media: {media} (UUID: {media.pk}) but '
f'the source has a download cap and the media is now too old, '
f'not downloading')
return
filepath = media.filepath
log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"')

View File

@@ -63,11 +63,6 @@
<td>Media format string</td>
<td>720p-avc1-mp4a</td>
</tr>
<tr>
<td>{playlist_index}</td>
<td>Playlist index of media, if it's in a playlist</td>
<td>12</td>
</tr>
<tr>
<td>{playlist_title}</td>
<td>Playlist title of media, if it's in a playlist</td>

View File

@@ -123,6 +123,10 @@
<td class="hide-on-small-only">Downloads directory</td>
<td><span class="hide-on-med-and-up">Downloads directory<br></span><strong>{{ downloads_dir }}</strong></td>
</tr>
<tr title="Database connection used by TubeSync">
<td class="hide-on-small-only">Database</td>
<td><span class="hide-on-med-and-up">Database<br></span><strong>{{ database_connection }}</strong></td>
</tr>
</table>
</div>
</div>

View File

@@ -64,8 +64,14 @@
<td class="hide-on-small-only">Fallback</td>
<td><span class="hide-on-med-and-up">Fallback<br></span><strong>{{ media.source.get_fallback_display }}</strong></td>
</tr>
{% if not media.source.download_media %}
<tr title="Is media marked to be downloaded at the source?">
<td class="hide-on-small-only">Source download?</td>
<td><span class="hide-on-med-and-up">Source download?<br></span><strong>{% if media.source.download_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr>
{% endif %}
{% if media.skip %}
<tr title="Has the media been downloaded?">
<tr title="Is the media marked to be skipped?">
<td class="hide-on-small-only">Skipping?</td>
<td><span class="hide-on-med-and-up">Skipping?<br></span><strong>{% if media.skip %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr>
@@ -109,7 +115,7 @@
{% else %}
<tr title="Can the media be downloaded?">
<td class="hide-on-small-only">Can download?</td>
<td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if youtube_dl_format %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
<td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if media.can_download %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr>
{% endif %}
<tr title="The available media formats">

View File

@@ -4,9 +4,16 @@
{% block content %}
<div class="row">
<div class="col s12">
<div class="col s12 m9">
<h1 class="truncate">Media</h1>
</div>
<div class="col s12 m3">
{% if show_skipped %}
<a href="{% url 'sync:media' %}{% if source %}?filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye-slash"></i> Hide skipped media</a>
{% else %}
<a href="{% url 'sync:media' %}?show_skipped=yes{% if source %}&filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye"></i> Show skipped media</a>
{% endif %}
</div>
</div>
{% include 'infobox.html' with message=message %}
<div class="row no-margin-bottom">
@@ -24,8 +31,12 @@
{% else %}
{% if m.skip %}
<span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped</span>
{% elif not m.source.download_media %}
<span class="error-text"><i class="fas fa-times" title="Not downloading media for this source"></i> Disabled at source</span>
{% elif not m.has_metadata %}
<i class="far fa-clock" title="Waiting for metadata"></i> Fetching metadata
{% elif m.can_download %}
<i class="far fa-clock" title="Waiting to download or downloading"></i> {{ m.published|date:'Y-m-d' }}
<i class="far fa-clock" title="Waiting to download or downloading"></i> Downloading
{% else %}
<span class="error-text"><i class="fas fa-exclamation-triangle" title="No matching formats to download"></i> No matching formats</span>
{% endif %}
@@ -44,5 +55,5 @@
</div>
{% endfor %}
</div>
{% include 'pagination.html' with pagination=sources.paginator filter=source.pk %}
{% include 'pagination.html' with pagination=sources.paginator filter=source.pk show_skipped=show_skipped %}
{% endblock %}

View File

@@ -61,6 +61,10 @@
<td class="hide-on-small-only">Index schedule</td>
<td><span class="hide-on-med-and-up">Index schedule<br></span><strong>{{ source.get_index_schedule_display }}</strong></td>
</tr>
<tr title="Download media from this source">
<td class="hide-on-small-only">Download media?</td>
<td><span class="hide-on-med-and-up">Download media?<br></span><strong>{% if source.download_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr>
<tr title="When then source was created locally in TubeSync">
<td class="hide-on-small-only">Created</td>
<td><span class="hide-on-med-and-up">Created<br></span><strong>{{ source.created|date:'Y-m-d H:i:s' }}</strong></td>

View File

@@ -30,7 +30,7 @@
{% if source.has_failed %}
<span class="error-text"><i class="fas fa-exclamation-triangle"></i> <strong>Source has permanent failures</strong></span>
{% else %}
<strong>{{ source.media_count }}</strong> media items{% if source.delete_old_media and source.days_to_keep > 0 %}, keep {{ source.days_to_keep }} days of media{% endif %}
<strong>{{ source.media_count }}</strong> media items, <strong>{{ source.downloaded_count }}</strong> downloaded{% if source.delete_old_media and source.days_to_keep > 0 %}, keeping {{ source.days_to_keep }} days of media{% endif %}
{% endif %}
</a>
{% empty %}

View File

@@ -9,7 +9,6 @@
"average_rating": 1.2345,
"dislike_count": 123,
"like_count": 456,
"playlist_index": 789,
"playlist_title": "test playlist",
"uploader": "test uploader",
"categories":[

View File

@@ -344,21 +344,25 @@ class FrontEndTestCase(TestCase):
}]
}
'''
past_date = timezone.make_aware(datetime(year=2000, month=1, day=1))
test_media1 = Media.objects.create(
key='mediakey1',
source=test_source,
published=past_date,
metadata=test_minimal_metadata
)
test_media1_pk = str(test_media1.pk)
test_media2 = Media.objects.create(
key='mediakey2',
source=test_source,
published=past_date,
metadata=test_minimal_metadata
)
test_media2_pk = str(test_media2.pk)
test_media3 = Media.objects.create(
key='mediakey3',
source=test_source,
published=past_date,
metadata=test_minimal_metadata
)
test_media3_pk = str(test_media3.pk)
@@ -535,9 +539,6 @@ class FilepathTestCase(TestCase):
self.source.media_format = 'test-{format}'
self.assertEqual(self.source.get_example_media_format(),
'test-1080p-vp9-opus')
self.source.media_format = 'test-{playlist_index}'
self.assertEqual(self.source.get_example_media_format(),
'test-1')
self.source.media_format = 'test-{playlist_title}'
self.assertEqual(self.source.get_example_media_format(),
'test-Some Playlist Title')
@@ -1161,14 +1162,14 @@ class FormatMatchingTestCase(TestCase):
('2160p', 'VP9', False, True): (True, '337'), # Exact match, hdr
('2160p', 'VP9', True, False): (True, '315'), # Exact match, 60fps
('2160p', 'VP9', True, True): (True, '337'), # Exact match, 60fps+hdr
('4320P', 'AVC1', False, False): (False, False),
('4320P', 'AVC1', False, True): (False, False),
('4320P', 'AVC1', True, False): (False, False),
('4320P', 'AVC1', True, True): (False, False),
('4320P', 'VP9', False, False): (False, False),
('4320P', 'VP9', False, True): (False, False),
('4320P', 'VP9', True, False): (True, '272'), # Exact match, 60fps
('4320P', 'VP9', True, True): (False, False),
('4320p', 'AVC1', False, False): (False, False),
('4320p', 'AVC1', False, True): (False, False),
('4320p', 'AVC1', True, False): (False, False),
('4320p', 'AVC1', True, True): (False, False),
('4320p', 'VP9', False, False): (False, False),
('4320p', 'VP9', False, True): (False, False),
('4320p', 'VP9', True, False): (True, '272'), # Exact match, 60fps
('4320p', 'VP9', True, True): (False, False),
}
for params, expected in expected_matches.items():
resolution, vcodec, prefer_60fps, prefer_hdr = params
@@ -1367,14 +1368,14 @@ class FormatMatchingTestCase(TestCase):
('2160p', 'VP9', False, True): (True, '337'), # Exact match, hdr
('2160p', 'VP9', True, False): (True, '315'), # Exact match, 60fps
('2160p', 'VP9', True, True): (True, '337'), # Exact match, 60fps+hdr
('4320P', 'AVC1', False, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
('4320P', 'AVC1', False, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
('4320P', 'AVC1', True, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
('4320P', 'AVC1', True, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
('4320P', 'VP9', False, False): (False, '272'), # Fallback, 60fps (no other 8k streams)
('4320P', 'VP9', False, True): (False, '272'), # Fallback, 60fps (no other 8k streams)
('4320P', 'VP9', True, False): (True, '272'), # Exact match, 60fps
('4320P', 'VP9', True, True): (False, '272'), # Fallback, 60fps (no other 8k streams)
('4320p', 'AVC1', False, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
('4320p', 'AVC1', False, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
('4320p', 'AVC1', True, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
('4320p', 'AVC1', True, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
('4320p', 'VP9', False, False): (False, '272'), # Fallback, 60fps (no other 8k streams)
('4320p', 'VP9', False, True): (False, '272'), # Fallback, 60fps (no other 8k streams)
('4320p', 'VP9', True, False): (True, '272'), # Exact match, 60fps
('4320p', 'VP9', True, True): (False, '272'), # Fallback, 60fps (no other 8k streams)
}
for params, expected in expected_matches.items():
resolution, vcodec, prefer_60fps, prefer_hdr = params

View File

@@ -10,7 +10,7 @@ from django.views.generic.detail import SingleObjectMixin
from django.http import HttpResponse
from django.urls import reverse_lazy
from django.db import IntegrityError
from django.db.models import Q, Count, Sum
from django.db.models import Q, Count, Sum, When, Case
from django.forms import ValidationError
from django.utils.text import slugify
from django.utils import timezone
@@ -78,6 +78,7 @@ class DashboardView(TemplateView):
# Config and download locations
data['config_dir'] = str(settings.CONFIG_BASE_DIR)
data['downloads_dir'] = str(settings.DOWNLOAD_ROOT)
data['database_connection'] = settings.DATABASE_CONNECTION_STR
return data
@@ -104,7 +105,10 @@ class SourcesView(ListView):
def get_queryset(self):
all_sources = Source.objects.all().order_by('name')
return all_sources.annotate(media_count=Count('media_source'))
return all_sources.annotate(
media_count=Count('media_source'),
downloaded_count=Count(Case(When(media_source__downloaded=True, then=1)))
)
def get_context_data(self, *args, **kwargs):
data = super().get_context_data(*args, **kwargs)
@@ -274,9 +278,9 @@ class AddSourceView(CreateView):
template_name = 'sync/source-add.html'
model = Source
fields = ('source_type', 'key', 'name', 'directory', 'media_format',
'index_schedule', 'download_cap', 'delete_old_media', 'days_to_keep',
'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
errors = {
'invalid_media_format': _('Invalid media format, the media format contains '
'errors or is empty. Check the table at the end of '
@@ -365,9 +369,9 @@ class UpdateSourceView(UpdateView):
template_name = 'sync/source-update.html'
model = Source
fields = ('source_type', 'key', 'name', 'directory', 'media_format',
'index_schedule', 'download_cap', 'delete_old_media', 'days_to_keep',
'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
errors = {
'invalid_media_format': _('Invalid media format, the media format contains '
'errors or is empty. Check the table at the end of '
@@ -438,6 +442,7 @@ class MediaView(ListView):
def __init__(self, *args, **kwargs):
self.filter_source = None
self.show_skipped = False
super().__init__(*args, **kwargs)
def dispatch(self, request, *args, **kwargs):
@@ -447,13 +452,22 @@ class MediaView(ListView):
self.filter_source = Source.objects.get(pk=filter_by)
except Source.DoesNotExist:
self.filter_source = None
show_skipped = request.GET.get('show_skipped', '').strip()
if show_skipped == 'yes':
self.show_skipped = True
return super().dispatch(request, *args, **kwargs)
def get_queryset(self):
if self.filter_source:
if self.show_skipped:
q = Media.objects.filter(source=self.filter_source)
else:
q = Media.objects.filter(source=self.filter_source, skip=False)
else:
if self.show_skipped:
q = Media.objects.all()
else:
q = Media.objects.filter(skip=False)
return q.order_by('-published', '-created')
def get_context_data(self, *args, **kwargs):
@@ -464,6 +478,7 @@ class MediaView(ListView):
message = str(self.messages.get('filter', ''))
data['message'] = message.format(name=self.filter_source.name)
data['source'] = self.filter_source
data['show_skipped'] = self.show_skipped
return data

View File

@@ -8,7 +8,7 @@ import os
from django.conf import settings
from copy import copy
from common.logger import log
import youtube_dl
import yt_dlp
_youtubedl_cachedir = getattr(settings, 'YOUTUBE_DL_CACHEDIR', None)
@@ -19,7 +19,7 @@ if _youtubedl_cachedir:
class YouTubeError(youtube_dl.utils.DownloadError):
class YouTubeError(yt_dlp.utils.DownloadError):
'''
Generic wrapped error for all errors that could be raised by youtube-dl.
'''
@@ -37,14 +37,20 @@ def get_media_info(url):
'skip_download': True,
'forcejson': True,
'simulate': True,
'logger': log
'logger': log,
'extract_flat': True,
})
response = {}
with youtube_dl.YoutubeDL(opts) as y:
with yt_dlp.YoutubeDL(opts) as y:
try:
response = y.extract_info(url, download=False)
except youtube_dl.utils.DownloadError as e:
except yt_dlp.utils.DownloadError as e:
raise YouTubeError(f'Failed to extract_info for "{url}": {e}') from e
if not response:
raise YouTubeError(f'Failed to extract_info for "{url}": No metadata was '
f'returned by youtube-dl, check for error messages in the '
f'logs above. This task will be retried later with an '
f'exponential backoff.')
return response
@@ -93,9 +99,9 @@ def download_media(url, media_format, extension, output_file):
'quiet': True,
'progress_hooks': [hook],
})
with youtube_dl.YoutubeDL(opts) as y:
with yt_dlp.YoutubeDL(opts) as y:
try:
return y.download([url])
except youtube_dl.utils.DownloadError as e:
except yt_dlp.utils.DownloadError as e:
raise YouTubeError(f'Failed to download for "{url}": {e}') from e
return False

View File

@@ -0,0 +1,22 @@
import importlib
from django.conf import settings
from django.db.backends.utils import CursorWrapper
def patch_ensure_connection():
for name, config in settings.DATABASES.items():
module = importlib.import_module(config['ENGINE'] + '.base')
def ensure_connection(self):
if self.connection is not None:
try:
with CursorWrapper(self.create_cursor(), self) as cursor:
cursor.execute('SELECT 1;')
return
except Exception:
pass
with self.wrap_database_errors:
self.connect()
module.DatabaseWrapper.ensure_connection = ensure_connection

View File

@@ -1,5 +1,7 @@
import os
from pathlib import Path
from common.logger import log
from common.utils import parse_database_connection_string
BASE_DIR = Path(__file__).resolve().parent.parent
@@ -21,14 +23,33 @@ FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', None)
TIME_ZONE = os.getenv('TZ', 'UTC')
DATABASES = {
database_dict = {}
database_connection_env = os.getenv('DATABASE_CONNECTION', '')
if database_connection_env:
database_dict = parse_database_connection_string(database_connection_env)
if database_dict:
log.info(f'Using database connection: {database_dict["ENGINE"]}://'
f'{database_dict["USER"]}:[hidden]@{database_dict["HOST"]}:'
f'{database_dict["PORT"]}/{database_dict["NAME"]}')
DATABASES = {
'default': database_dict,
}
DATABASE_CONNECTION_STR = (f'{database_dict["DRIVER"]} at "{database_dict["HOST"]}:'
f'{database_dict["PORT"]}" database '
f'"{database_dict["NAME"]}"')
else:
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': CONFIG_BASE_DIR / 'db.sqlite3',
}
}
}
DATABASE_CONNECTION_STR = f'sqlite at "{DATABASES["default"]["NAME"]}"'
DEFAULT_THREADS = 2
DEFAULT_THREADS = 1
MAX_BACKGROUND_TASK_ASYNC_THREADS = 8
BACKGROUND_TASK_ASYNC_THREADS = int(os.getenv('TUBESYNC_WORKERS', DEFAULT_THREADS))
if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS:
@@ -38,3 +59,15 @@ if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS:
MEDIA_ROOT = CONFIG_BASE_DIR / 'media'
DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR
YOUTUBE_DL_CACHEDIR = CONFIG_BASE_DIR / 'cache'
BASICAUTH_USERNAME = os.getenv('HTTP_USER', '').strip()
BASICAUTH_PASSWORD = os.getenv('HTTP_PASS', '').strip()
if BASICAUTH_USERNAME and BASICAUTH_PASSWORD:
BASICAUTH_DISABLE = False
BASICAUTH_USERS = {
BASICAUTH_USERNAME: BASICAUTH_PASSWORD,
}
else:
BASICAUTH_DISABLE = True
BASICAUTH_USERS = {}

View File

@@ -16,6 +16,7 @@ DATABASES = {
'NAME': CONFIG_BASE_DIR / 'db.sqlite3',
}
}
DATABASE_CONNECTION_STR = f'sqlite at "{DATABASES["default"]["NAME"]}"'
DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR / 'downloads'

View File

@@ -6,7 +6,7 @@ CONFIG_BASE_DIR = BASE_DIR
DOWNLOADS_BASE_DIR = BASE_DIR
VERSION = 0.7
VERSION = '0.10.0'
SECRET_KEY = ''
DEBUG = False
ALLOWED_HOSTS = []
@@ -37,6 +37,7 @@ MIDDLEWARE = [
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'whitenoise.middleware.WhiteNoiseMiddleware',
'common.middleware.MaterializeDefaultFieldsMiddleware',
'common.middleware.BasicAuthMiddleware',
]
@@ -75,6 +76,9 @@ WSGI_APPLICATION = 'tubesync.wsgi.application'
DATABASES = {}
DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
@@ -114,11 +118,20 @@ Disallow: /
'''.strip()
X_FRAME_OPTIONS = 'SAMEORIGIN'
BASICAUTH_DISABLE = True
BASICAUTH_REALM = 'Authenticate to TubeSync'
BASICAUTH_ALWAYS_ALLOW_URIS = ('/healthcheck',)
BASICAUTH_USERS = {}
HEALTHCHECK_FIREWALL = True
HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',)
MAX_ATTEMPTS = 10 # Number of times tasks will be retried
MAX_ATTEMPTS = 15 # Number of times tasks will be retried
MAX_RUN_TIME = 1800 # Maximum amount of time in seconds a task can run
BACKGROUND_TASK_RUN_ASYNC = True # Run tasks async in the background
BACKGROUND_TASK_ASYNC_THREADS = 1 # Number of async tasks to run at once
@@ -128,7 +141,7 @@ COMPLETED_TASKS_DAYS_TO_KEEP = 7 # Number of days to keep completed t
SOURCES_PER_PAGE = 100
MEDIA_PER_PAGE = 72
MEDIA_PER_PAGE = 144
TASKS_PER_PAGE = 100
@@ -146,10 +159,11 @@ YOUTUBE_DEFAULTS = {
'age_limit': 99, # 'Age in years' to spoof
'ignoreerrors': True, # Skip on errors (such as unavailable videos in playlists)
'cachedir': False, # Disable on-disk caching
'addmetadata': True, # Embed metadata during postprocessing where available
}
MEDIA_FORMATSTR_DEFAULT = '{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}'
MEDIA_FORMATSTR_DEFAULT = '{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}'
try:
@@ -158,3 +172,7 @@ except ImportError as e:
import sys
sys.stderr.write(f'Unable to import local_settings: {e}\n')
sys.exit(1)
from .dbutils import patch_ensure_connection
patch_ensure_connection()