bump to 0.12.0

Misc fixes

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [meeb]

.github/workflows/ci.yaml

@@ -27,7 +27,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install pipenv
-          pipenv install --system
+          pipenv install --system --skip-lock
       - name: Set up Django environment
         run: cp tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py
       - name: Run Django tests
@@ -35,13 +35,24 @@ jobs:
   containerise:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Build the container image
-      run: docker build . --tag $IMAGE_NAME
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
       - name: Log into GitHub Container Registry
         run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
-    - name: Push image to GitHub Container Registry
-      run: |
-        LATEST_TAG=ghcr.io/meeb/$IMAGE_NAME:latest
-        docker tag $IMAGE_NAME $LATEST_TAG
-        docker push $LATEST_TAG
+      - name: Lowercase github username for ghcr
+        id: string
+        uses: ASzc/change-string-case-action@v1
+        with:
+          string: ${{ github.actor }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest
+          cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest
+          cache-to: type=inline
+          build-args: |
+            IMAGE_NAME=${{ env.IMAGE_NAME }}

.github/workflows/release.yaml

@@ -11,18 +11,28 @@ jobs:
   containerise:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
       - name: Get tag
-      id: vars
-      run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
-    - name: Build the container image
-      run: docker build . --tag $IMAGE_NAME
+        id: tag
+        uses: dawidd6/action-get-tag@v1
+      - uses: docker/build-push-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
       - name: Log into GitHub Container Registry
         run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
-    - name: Push image to GitHub Container Registry
-      env:
-        RELEASE_TAG: ${{ steps.vars.outputs.tag }}
-      run: |
-        REF_TAG=ghcr.io/meeb/$IMAGE_NAME:$RELEASE_TAG
-        docker tag $IMAGE_NAME $REF_TAG
-        docker push $REF_TAG
+      - name: Lowercase github username for ghcr
+        id: string
+        uses: ASzc/change-string-case-action@v1
+        with:
+          string: ${{ github.actor }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
+          cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
+          cache-to: type=inline
+          build-args: |
+            IMAGE_NAME=${{ env.IMAGE_NAME }}

Dockerfile

@@ -1,72 +1,111 @@
-FROM debian:buster-slim
+FROM debian:bullseye-slim
 
-ARG ARCH="amd64"
-ARG S6_VERSION="2.1.0.2"
-ARG FFMPEG_VERSION="4.3.1"
+ARG TARGETPLATFORM
+ARG S6_VERSION="3.1.2.1"
+ARG FFMPEG_DATE="autobuild-2023-01-03-12-55"
+ARG FFMPEG_VERSION="109474-gc94988a781"
 
 ENV DEBIAN_FRONTEND="noninteractive" \
   HOME="/root" \
   LANGUAGE="en_US.UTF-8" \
   LANG="en_US.UTF-8" \
   LC_ALL="en_US.UTF-8" \
-    TERM="xterm" \
-    S6_EXPECTED_SHA256="52460473413601ff7a84ae690b161a074217ddc734990c2cdee9847166cf669e" \
-    S6_DOWNLOAD="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${ARCH}.tar.gz" \
-    FFMPEG_EXPECTED_SHA256="47d95c0129fba27d051748a442a44a73ce1bd38d1e3f9fe1e9dd7258c7581fa5" \
-    FFMPEG_DOWNLOAD="https://tubesync.sfo2.digitaloceanspaces.com/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static.tar.xz"
-
+  TERM="xterm"
 
 # Install third party software
-RUN set -x && \
+RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
+  "linux/amd64")   echo "amd64"  ;; \
+  "linux/arm64")   echo "aarch64" ;; \
+  *)               echo ""        ;; esac) && \
+  export S6_ARCH_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
+  "linux/amd64")   echo "6019b6b06cfdbb1d1cd572d46b9b158a4904fd19ca59d374de4ddaaa6a3727d5" ;; \
+  "linux/arm64")   echo "e73f9a021b64f88278830742149c14ef8a52331102881ba025bf32a66a0e7c78" ;; \
+  *)               echo ""        ;; esac) && \
+  export S6_DOWNLOAD_ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
+  "linux/amd64")   echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-x86_64.tar.xz"   ;; \
+  "linux/arm64")   echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-aarch64.tar.xz" ;; \
+  *)               echo ""        ;; esac) && \
+  export FFMPEG_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
+  "linux/amd64")   echo "ed9059668e4a6dac9bde122a775f52ad08cbb90df3658f8c1e328477c13c242e" ;; \
+  "linux/arm64")   echo "dd1375bd351d38ea1cc3efd68a998699366e28bd9b90df65d11af2b9121746b7" ;; \
+  *)               echo ""        ;; esac) && \
+  export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \
+  "linux/amd64")   echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linux64-gpl.tar.xz"   ;; \
+  "linux/arm64")   echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linuxarm64-gpl.tar.xz" ;; \
+  *)               echo ""        ;; esac) && \
+  export S6_NOARCH_EXPECTED_SHA256="cee89d3eeabdfe15239b2c5c3581d9352d2197d4fd23bba3f1e64bf916ccf496" && \
+  export S6_DOWNLOAD_NOARCH="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-noarch.tar.xz" && \
+  echo "Building for arch: ${ARCH}|${ARCH44}, downloading S6 from: ${S6_DOWNLOAD}}, expecting S6 SHA256: ${S6_EXPECTED_SHA256}" && \
+  set -x && \
   apt-get update && \
   apt-get -y --no-install-recommends install locales && \
   echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
   locale-gen en_US.UTF-8 && \
   # Install required distro packages
-    apt-get -y --no-install-recommends install curl xz-utils ca-certificates binutils && \
+  apt-get -y --no-install-recommends install curl ca-certificates binutils xz-utils && \
   # Install s6
-    curl -L ${S6_DOWNLOAD} --output /tmp/s6-overlay-${ARCH}.tar.gz && \
-    sha256sum /tmp/s6-overlay-${ARCH}.tar.gz && \
-    echo "${S6_EXPECTED_SHA256}  /tmp/s6-overlay-${ARCH}.tar.gz" | sha256sum -c - && \
-    tar xzf /tmp/s6-overlay-${ARCH}.tar.gz -C / && \
+  curl -L ${S6_DOWNLOAD_NOARCH} --output /tmp/s6-overlay-noarch.tar.xz && \
+  echo "${S6_NOARCH_EXPECTED_SHA256}  /tmp/s6-overlay-noarch.tar.xz" | sha256sum -c - && \
+  tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && \
+  curl -L ${S6_DOWNLOAD_ARCH} --output /tmp/s6-overlay-${ARCH}.tar.xz && \
+  echo "${S6_ARCH_EXPECTED_SHA256}  /tmp/s6-overlay-${ARCH}.tar.xz" | sha256sum -c - && \
+  tar -C / -Jxpf /tmp/s6-overlay-${ARCH}.tar.xz && \
   # Install ffmpeg
-    curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg-${ARCH}-static.tar.xz && \
-    echo "${FFMPEG_EXPECTED_SHA256}  /tmp/ffmpeg-${ARCH}-static.tar.xz" | sha256sum -c - && \
-    xz --decompress /tmp/ffmpeg-${ARCH}-static.tar.xz && \
-    tar -xvf /tmp/ffmpeg-${ARCH}-static.tar -C /tmp && \
-    install -v -s -g root -o root -m 0755 -s /tmp/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static/ffmpeg -t /usr/local/bin && \
+  echo "Building for arch: ${ARCH}|${ARCH44}, downloading FFMPEG from: ${FFMPEG_DOWNLOAD}, expecting FFMPEG SHA256: ${FFMPEG_EXPECTED_SHA256}" && \
+  curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg-${ARCH}.tar.xz && \
+  sha256sum /tmp/ffmpeg-${ARCH}.tar.xz && \
+  echo "${FFMPEG_EXPECTED_SHA256}  /tmp/ffmpeg-${ARCH}.tar.xz" | sha256sum -c - && \
+  tar -xf /tmp/ffmpeg-${ARCH}.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffmpeg" && \
+  tar -xf /tmp/ffmpeg-${ARCH}.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffprobe" && \
   # Clean up
   rm -rf /tmp/s6-overlay-${ARCH}.tar.gz && \
-    rm -rf /tmp/ffmpeg-${ARCH}-static.tar && \
-    rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static && \
-    apt-get -y autoremove --purge curl xz-utils binutils
+  rm -rf /tmp/ffmpeg-${ARCH}.tar.xz && \
+  apt-get -y autoremove --purge curl binutils xz-utils
 
 # Copy app
 COPY tubesync /app
 COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings.py
 
-# Append container bundled software versions
-RUN echo "ffmpeg_version = '${FFMPEG_VERSION}-static'" >> /app/common/third_party_versions.py
+# Copy over pip.conf to use piwheels
+COPY pip.conf /etc/pip.conf
 
 # Add Pipfile
 COPY Pipfile /app/Pipfile
-COPY Pipfile.lock /app/Pipfile.lock
 
 # Switch workdir to the the app
 WORKDIR /app
 
 # Set up the app
 RUN set -x && \
+  apt-get update && \
   # Install required distro packages
   apt-get -y install nginx-light && \
-  apt-get -y --no-install-recommends install python3 python3-setuptools python3-pip python3-dev gcc make && \
+  apt-get -y --no-install-recommends install \
+  python3 \
+  python3-setuptools \
+  python3-pip \
+  python3-dev \
+  gcc \
+  g++ \
+  make \
+  default-libmysqlclient-dev \
+  libmariadb3 \
+  postgresql-common \
+  libpq-dev \
+  libpq5 \
+  libjpeg62-turbo \
+  libwebp6 \
+  libjpeg-dev \
+  zlib1g-dev \
+  libwebp-dev \
+  redis-server && \
   # Install pipenv
-  pip3 --disable-pip-version-check install pipenv && \
+  pip3 --disable-pip-version-check install wheel pipenv && \
   # Create a 'app' user which the application will run as
   groupadd app && \
   useradd -M -d /app -s /bin/false -g app app && \
   # Install non-distro packages
-  pipenv install --system && \
+  pipenv install --system --skip-lock && \
   # Make absolutely sure we didn't accidentally bundle a SQLite dev database
   rm -rf /app/db.sqlite3 && \
   # Run any required app commands
@@ -79,10 +118,20 @@ RUN set -x && \
   mkdir -p /downloads/video && \
   # Clean up
   rm /app/Pipfile && \
-  rm /app/Pipfile.lock && \
   pipenv --clear && \
   pip3 --disable-pip-version-check uninstall -y pipenv wheel virtualenv && \
-  apt-get -y autoremove --purge python3-pip python3-dev gcc make && \
+  apt-get -y autoremove --purge \
+  python3-pip \
+  python3-dev \
+  gcc \
+  g++ \
+  make \
+  default-libmysqlclient-dev \
+  postgresql-common \
+  libpq-dev \
+  libjpeg-dev \
+  zlib1g-dev \
+  libwebp-dev && \
   apt-get -y autoremove && \
   apt-get -y autoclean && \
   rm -rf /var/lib/apt/lists/* && \
@@ -92,7 +141,12 @@ RUN set -x && \
   rm -rf /root && \
   mkdir -p /root && \
   chown root:root /root && \
-  chmod 0700 /root
+  chmod 0755 /root
+
+# Append software versions
+RUN set -x && \
+  FFMPEG_VERSION=$(/usr/local/bin/ffmpeg -version | head -n 1 | awk '{ print $3 }') && \
+  echo "ffmpeg_version = '${FFMPEG_VERSION}'" >> /app/common/third_party_versions.py
 
 # Copy root
 COPY config/root /
@@ -102,7 +156,7 @@ HEALTHCHECK --interval=1m --timeout=10s CMD /app/healthcheck.py http://127.0.0.1
 
 # ENVS and ports
 ENV PYTHONPATH "/app:${PYTHONPATH}"
-EXPOSE 8080
+EXPOSE 4848
 
 # Volumes
 VOLUME ["/config", "/downloads"]

Makefile

@@ -8,17 +8,17 @@ all: clean build
 
 
 dev:
-	$(python) app/manage.py runserver
+	$(python) tubesync/manage.py runserver
 
 
 build:
-	mkdir -p app/media
-	mkdir -p app/static
-	$(python) app/manage.py collectstatic --noinput
+	mkdir -p tubesync/media
+	mkdir -p tubesync/static
+	$(python) tubesync/manage.py collectstatic --noinput
 
 
 clean:
-	rm -rf app/static
+	rm -rf tubesync/static
 
 
 container: clean
@@ -29,5 +29,5 @@ runcontainer:
 	$(docker) run --rm --name $(name) --env-file dev.env --log-opt max-size=50m -ti -p 4848:4848 $(image)
 
 
-test:
-	$(python) app/manage.py test --verbosity=2
+test: build
+	cd tubesync && $(python) manage.py test --verbosity=2 && cd ..

Pipfile

@@ -6,7 +6,7 @@ verify_ssl = true
 [dev-packages]
 
 [packages]
-django = "*"
+django = "~=3.2"
 django-sass-processor = "*"
 libsass = "*"
 pillow = "*"
@@ -14,9 +14,11 @@ whitenoise = "*"
 gunicorn = "*"
 django-compressor = "*"
 httptools = "*"
-youtube-dl = "*"
 django-background-tasks = "*"
 requests = "*"
-
-[requires]
-python_version = "3"
+django-basicauth = "*"
+psycopg2-binary = "*"
+mysqlclient = "*"
+yt-dlp = "*"
+redis = "*"
+hiredis = "*"

Pipfile.lock

@@ -1,247 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "a4bb556fc61ee4583f9588980450b071814298ee4d1a1023fad149c14d14aaba"
-        },
-        "pipfile-spec": 6,
-        "requires": {
-            "python_version": "3"
-        },
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "asgiref": {
-            "hashes": [
-                "sha256:5ee950735509d04eb673bd7f7120f8fa1c9e2df495394992c73234d526907e17",
-                "sha256:7162a3cb30ab0609f1a4c95938fd73e8604f63bdba516a7f7d64b83ff09478f0"
-            ],
-            "version": "==3.3.1"
-        },
-        "certifi": {
-            "hashes": [
-                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
-                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
-            ],
-            "version": "==2020.12.5"
-        },
-        "chardet": {
-            "hashes": [
-                "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
-                "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
-            ],
-            "version": "==4.0.0"
-        },
-        "django": {
-            "hashes": [
-                "sha256:5c866205f15e7a7123f1eec6ab939d22d5bde1416635cab259684af66d8e48a2",
-                "sha256:edb10b5c45e7e9c0fb1dc00b76ec7449aca258a39ffd613dbd078c51d19c9f03"
-            ],
-            "index": "pypi",
-            "version": "==3.1.4"
-        },
-        "django-appconf": {
-            "hashes": [
-                "sha256:1b1d0e1069c843ebe8ae5aa48ec52403b1440402b320c3e3a206a0907e97bb06",
-                "sha256:be58deb54a43d77d2e1621fe59f787681376d3cd0b8bd8e4758ef6c3a6453380"
-            ],
-            "version": "==1.0.4"
-        },
-        "django-background-tasks": {
-            "hashes": [
-                "sha256:e1b19e8d495a276c9d64c5a1ff8b41132f75d2f58e45be71b78650dad59af9de"
-            ],
-            "index": "pypi",
-            "version": "==1.2.5"
-        },
-        "django-compat": {
-            "hashes": [
-                "sha256:3ac9a3bedc56b9365d9eb241bc5157d0c193769bf995f9a78dc1bc24e7c2331b"
-            ],
-            "version": "==1.0.15"
-        },
-        "django-compressor": {
-            "hashes": [
-                "sha256:57ac0a696d061e5fc6fbc55381d2050f353b973fb97eee5593f39247bc0f30af",
-                "sha256:d2ed1c6137ddaac5536233ec0a819e14009553fee0a869bea65d03e5285ba74f"
-            ],
-            "index": "pypi",
-            "version": "==2.4"
-        },
-        "django-sass-processor": {
-            "hashes": [
-                "sha256:9b46a12ca8bdcb397d46fbcc49e6a926ff9f76a93c5efeb23b495419fd01fc7a"
-            ],
-            "index": "pypi",
-            "version": "==0.8.2"
-        },
-        "gunicorn": {
-            "hashes": [
-                "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626",
-                "sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"
-            ],
-            "index": "pypi",
-            "version": "==20.0.4"
-        },
-        "httptools": {
-            "hashes": [
-                "sha256:0a4b1b2012b28e68306575ad14ad5e9120b34fccd02a81eb08838d7e3bbb48be",
-                "sha256:3592e854424ec94bd17dc3e0c96a64e459ec4147e6d53c0a42d0ebcef9cb9c5d",
-                "sha256:41b573cf33f64a8f8f3400d0a7faf48e1888582b6f6e02b82b9bd4f0bf7497ce",
-                "sha256:56b6393c6ac7abe632f2294da53f30d279130a92e8ae39d8d14ee2e1b05ad1f2",
-                "sha256:86c6acd66765a934e8730bf0e9dfaac6fdcf2a4334212bd4a0a1c78f16475ca6",
-                "sha256:96da81e1992be8ac2fd5597bf0283d832287e20cb3cfde8996d2b00356d4e17f",
-                "sha256:96eb359252aeed57ea5c7b3d79839aaa0382c9d3149f7d24dd7172b1bcecb009",
-                "sha256:a2719e1d7a84bb131c4f1e0cb79705034b48de6ae486eb5297a139d6a3296dce",
-                "sha256:ac0aa11e99454b6a66989aa2d44bca41d4e0f968e395a0a8f164b401fefe359a",
-                "sha256:bc3114b9edbca5a1eb7ae7db698c669eb53eb8afbbebdde116c174925260849c",
-                "sha256:fa3cd71e31436911a44620473e873a256851e1f53dee56669dae403ba41756a4",
-                "sha256:fea04e126014169384dee76a153d4573d90d0cbd1d12185da089f73c78390437"
-            ],
-            "index": "pypi",
-            "version": "==0.1.1"
-        },
-        "idna": {
-            "hashes": [
-                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
-                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
-            ],
-            "version": "==2.10"
-        },
-        "libsass": {
-            "hashes": [
-                "sha256:1521d2a8d4b397c6ec90640a1f6b5529077035efc48ef1c2e53095544e713d1b",
-                "sha256:1b2d415bbf6fa7da33ef46e549db1418498267b459978eff8357e5e823962d35",
-                "sha256:25ebc2085f5eee574761ccc8d9cd29a9b436fc970546d5ef08c6fa41eb57dff1",
-                "sha256:2ae806427b28bc1bb7cb0258666d854fcf92ba52a04656b0b17ba5e190fb48a9",
-                "sha256:4a246e4b88fd279abef8b669206228c92534d96ddcd0770d7012088c408dff23",
-                "sha256:553e5096414a8d4fb48d0a48f5a038d3411abe254d79deac5e008516c019e63a",
-                "sha256:697f0f9fa8a1367ca9ec6869437cb235b1c537fc8519983d1d890178614a8903",
-                "sha256:a8fd4af9f853e8bf42b1425c5e48dd90b504fa2e70d7dac5ac80b8c0a5a5fe85",
-                "sha256:c9411fec76f480ffbacc97d8188322e02a5abca6fc78e70b86a2a2b421eae8a2",
-                "sha256:daa98a51086d92aa7e9c8871cf1a8258124b90e2abf4697852a3dca619838618",
-                "sha256:e0e60836eccbf2d9e24ec978a805cd6642fa92515fbd95e3493fee276af76f8a",
-                "sha256:e64ae2587f1a683e831409aad03ba547c245ef997e1329fffadf7a866d2510b8",
-                "sha256:f6852828e9e104d2ce0358b73c550d26dd86cc3a69439438c3b618811b9584f5"
-            ],
-            "index": "pypi",
-            "version": "==0.20.1"
-        },
-        "pillow": {
-            "hashes": [
-                "sha256:006de60d7580d81f4a1a7e9f0173dc90a932e3905cc4d47ea909bc946302311a",
-                "sha256:0a2e8d03787ec7ad71dc18aec9367c946ef8ef50e1e78c71f743bc3a770f9fae",
-                "sha256:0eeeae397e5a79dc088d8297a4c2c6f901f8fb30db47795113a4a605d0f1e5ce",
-                "sha256:11c5c6e9b02c9dac08af04f093eb5a2f84857df70a7d4a6a6ad461aca803fb9e",
-                "sha256:2fb113757a369a6cdb189f8df3226e995acfed0a8919a72416626af1a0a71140",
-                "sha256:4b0ef2470c4979e345e4e0cc1bbac65fda11d0d7b789dbac035e4c6ce3f98adb",
-                "sha256:59e903ca800c8cfd1ebe482349ec7c35687b95e98cefae213e271c8c7fffa021",
-                "sha256:5abd653a23c35d980b332bc0431d39663b1709d64142e3652890df4c9b6970f6",
-                "sha256:5f9403af9c790cc18411ea398a6950ee2def2a830ad0cfe6dc9122e6d528b302",
-                "sha256:6b4a8fd632b4ebee28282a9fef4c341835a1aa8671e2770b6f89adc8e8c2703c",
-                "sha256:6c1aca8231625115104a06e4389fcd9ec88f0c9befbabd80dc206c35561be271",
-                "sha256:795e91a60f291e75de2e20e6bdd67770f793c8605b553cb6e4387ce0cb302e09",
-                "sha256:7ba0ba61252ab23052e642abdb17fd08fdcfdbbf3b74c969a30c58ac1ade7cd3",
-                "sha256:7c9401e68730d6c4245b8e361d3d13e1035cbc94db86b49dc7da8bec235d0015",
-                "sha256:81f812d8f5e8a09b246515fac141e9d10113229bc33ea073fec11403b016bcf3",
-                "sha256:895d54c0ddc78a478c80f9c438579ac15f3e27bf442c2a9aa74d41d0e4d12544",
-                "sha256:8de332053707c80963b589b22f8e0229f1be1f3ca862a932c1bcd48dafb18dd8",
-                "sha256:92c882b70a40c79de9f5294dc99390671e07fc0b0113d472cbea3fde15db1792",
-                "sha256:95edb1ed513e68bddc2aee3de66ceaf743590bf16c023fb9977adc4be15bd3f0",
-                "sha256:b63d4ff734263ae4ce6593798bcfee6dbfb00523c82753a3a03cbc05555a9cc3",
-                "sha256:bd7bf289e05470b1bc74889d1466d9ad4a56d201f24397557b6f65c24a6844b8",
-                "sha256:cc3ea6b23954da84dbee8025c616040d9aa5eaf34ea6895a0a762ee9d3e12e11",
-                "sha256:cc9ec588c6ef3a1325fa032ec14d97b7309db493782ea8c304666fb10c3bd9a7",
-                "sha256:d3d07c86d4efa1facdf32aa878bd508c0dc4f87c48125cc16b937baa4e5b5e11",
-                "sha256:d8a96747df78cda35980905bf26e72960cba6d355ace4780d4bdde3b217cdf1e",
-                "sha256:e38d58d9138ef972fceb7aeec4be02e3f01d383723965bfcef14d174c8ccd039",
-                "sha256:eb472586374dc66b31e36e14720747595c2b265ae962987261f044e5cce644b5",
-                "sha256:fbd922f702582cb0d71ef94442bfca57624352622d75e3be7a1e7e9360b07e72"
-            ],
-            "index": "pypi",
-            "version": "==8.0.1"
-        },
-        "pytz": {
-            "hashes": [
-                "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",
-                "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"
-            ],
-            "version": "==2020.4"
-        },
-        "rcssmin": {
-            "hashes": [
-                "sha256:ca87b695d3d7864157773a61263e5abb96006e9ff0e021eff90cbe0e1ba18270"
-            ],
-            "version": "==1.0.6"
-        },
-        "requests": {
-            "hashes": [
-                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
-                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
-            ],
-            "index": "pypi",
-            "version": "==2.25.1"
-        },
-        "rjsmin": {
-            "hashes": [
-                "sha256:0ab825839125eaca57cc59581d72e596e58a7a56fbc0839996b7528f0343a0a8",
-                "sha256:211c2fe8298951663bbc02acdffbf714f6793df54bfc50e1c6c9e71b3f2559a3",
-                "sha256:466fe70cc5647c7c51b3260c7e2e323a98b2b173564247f9c89e977720a0645f",
-                "sha256:585e75a84d9199b68056fd4a083d9a61e2a92dfd10ff6d4ce5bdb04bc3bdbfaf",
-                "sha256:6044ca86e917cd5bb2f95e6679a4192cef812122f28ee08c677513de019629b3",
-                "sha256:714329db774a90947e0e2086cdddb80d5e8c4ac1c70c9f92436378dedb8ae345",
-                "sha256:799890bd07a048892d8d3deb9042dbc20b7f5d0eb7da91e9483c561033b23ce2",
-                "sha256:975b69754d6a76be47c0bead12367a1ca9220d08e5393f80bab0230d4625d1f4",
-                "sha256:b15dc75c71f65d9493a8c7fa233fdcec823e3f1b88ad84a843ffef49b338ac32",
-                "sha256:dd0f4819df4243ffe4c964995794c79ca43943b5b756de84be92b445a652fb86",
-                "sha256:e3908b21ebb584ce74a6ac233bdb5f29485752c9d3be5e50c5484ed74169232c",
-                "sha256:e487a7783ac4339e79ec610b98228eb9ac72178973e3dee16eba0e3feef25924",
-                "sha256:ecd29f1b3e66a4c0753105baec262b331bcbceefc22fbe6f7e8bcd2067bcb4d7"
-            ],
-            "version": "==1.1.0"
-        },
-        "six": {
-            "hashes": [
-                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
-                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
-            ],
-            "version": "==1.15.0"
-        },
-        "sqlparse": {
-            "hashes": [
-                "sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0",
-                "sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8"
-            ],
-            "version": "==0.4.1"
-        },
-        "urllib3": {
-            "hashes": [
-                "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
-                "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
-            ],
-            "version": "==1.26.2"
-        },
-        "whitenoise": {
-            "hashes": [
-                "sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7",
-                "sha256:05d00198c777028d72d8b0bbd234db605ef6d60e9410125124002518a48e515d"
-            ],
-            "index": "pypi",
-            "version": "==5.2.0"
-        },
-        "youtube-dl": {
-            "hashes": [
-                "sha256:65968065e66966955dc79fad9251565fcc982566118756da624bd21467f3a04c",
-                "sha256:eaa859f15b6897bec21474b7787dc958118c8088e1f24d4ef1d58eab13188958"
-            ],
-            "index": "pypi",
-            "version": "==2020.12.14"
-        }
-    },
-    "develop": {}
-}

README.md

@@ -9,10 +9,10 @@ downloaded.
 
 If you want to watch YouTube videos in particular quality or settings from your local
 media server, then TubeSync is for you. Internally, TubeSync is a web interface wrapper
-on `youtube-dl` and `ffmpeg` with a task scheduler.
+on `yt-dlp` and `ffmpeg` with a task scheduler.
 
-There are several other web interfaces to YouTube and `youtube-dl` all with varying
-features and implemenations. TubeSync's largest difference is full PVR experience of
+There are several other web interfaces to YouTube and `yt-dlp` all with varying
+features and implementations. TubeSync's largest difference is full PVR experience of
 updating media servers and better selection of media formats. Additionally, to be as
 hands-free as possible, TubeSync has gradual retrying of failures with back-off timers
 so media which fails to download will be retried for an extended period making it,
@@ -22,7 +22,7 @@ hopefully, quite reliable.
 # Latest container image
 
 ```yaml
-ghcr.io/meeb/tubesync:v0.6
+ghcr.io/meeb/tubesync:latest
 ```
 
 # Screenshots
@@ -69,11 +69,12 @@ currently just Plex, to complete the PVR experience.
 # Installation
 
 TubeSync is designed to be run in a container, such as via Docker or Podman. It also
-works in a Docker Compose stack. Only `amd64` is initially supported.
+works in a Docker Compose stack. `amd64` (most desktop PCs and servers) and `arm64`
+(modern ARM computers, such as the Rasperry Pi 3 or later) are supported.
 
 Example (with Docker on *nix):
 
-First find your the user ID and group ID you want to run TubeSync as, if you're not
+First find the user ID and group ID you want to run TubeSync as, if you're not
 sure what this is it's probably your current user ID and group ID:
 
 ```bash
@@ -98,8 +99,8 @@ $ mkdir /some/directory/tubesync-downloads
 Finally, download and run the container:
 
 ```bash
-# Pull a versioned image
-$ docker pull ghcr.io/meeb/tubesync:v0.6
+# Pull image
+$ docker pull ghcr.io/meeb/tubesync:latest
 # Start the container using your user ID and group ID
 $ docker run \
   -d \
@@ -110,19 +111,21 @@ $ docker run \
   -v /some/directory/tubesync-config:/config \
   -v /some/directory/tubesync-downloads:/downloads \
   -p 4848:4848 \
-  ghcr.io/meeb/tubesync:v0.6
+  ghcr.io/meeb/tubesync:latest
 ```
 
 Once running, open `http://localhost:4848` in your browser and you should see the
 TubeSync dashboard. If you do, you can proceed to adding some sources (YouTube channels
 and playlists). If not, check `docker logs tubesync` to see what errors might be
-occuring, typical ones are file permission issues.
+occurring, typical ones are file permission issues.
 
 Alternatively, for Docker Compose, you can use something like:
 
-```yaml
+```yml
+version: '3.7'
+services:
   tubesync:
-    image: ghcr.io/meeb/tubesync:v0.6
+    image: ghcr.io/meeb/tubesync:latest
     container_name: tubesync
     restart: unless-stopped
     ports:
@@ -136,6 +139,41 @@ Alternatively, for Docker Compose, you can use something like:
       - PGID=1000
 ```
 
+
+## Optional authentication
+
+Available in `v1.0` (or `:latest`)and later. If you want to enable a basic username and
+password to be required to access the TubeSync dashboard you can set them with the
+following environment variables:
+
+```bash
+HTTP_USER
+HTTP_PASS
+```
+
+For example, in the `docker run ...` line add in:
+
+```bash
+...
+-e HTTP_USER=some-username \
+-e HTTP_PASS=some-secure-password \
+...
+```
+
+Or in your Docker Compose file you would add in:
+
+```yaml
+...
+    environment:
+      - HTTP_USER=some-username
+      - HTTP_PASS=some-secure-password
+...
+```
+
+When BOTH `HTTP_USER` and `HTTP_PASS` are set then basic HTTP authentication will be
+enabled.
+
+
 # Updating
 
 To update, you can just pull a new version of the container image as they are released.
@@ -193,6 +231,18 @@ $ docker logs --follow tubesync
 ```
 
 
+# Advanced usage guides
+
+Once you're happy using TubeSync there are some advanced usage guides for more complex
+and less common features:
+
+ * [Import existing media into TubeSync](https://github.com/meeb/tubesync/blob/main/docs/import-existing-media.md)
+ * [Sync or create missing metadata files](https://github.com/meeb/tubesync/blob/main/docs/create-missing-metadata.md)
+ * [Reset tasks from the command line](https://github.com/meeb/tubesync/blob/main/docs/reset-tasks.md)
+ * [Using PostgreSQL, MySQL or MariaDB as database backends](https://github.com/meeb/tubesync/blob/main/docs/other-database-backends.md)
+ * [Using cookies](https://github.com/meeb/tubesync/blob/main/docs/using-cookies.md)
+
+
 # Warnings
 
 ### 1. Index frequency
@@ -210,6 +260,14 @@ every hour" or similar short interval it's entirely possible your TubeSync insta
 spend its entire time just indexing the massive channel over and over again without
 downloading any media. Check your tasks for the status of your TubeSync install.
 
+If you add a significant amount of "work" due to adding many large channels you may
+need to increase the number of background workers by setting the `TUBESYNC_WORKERS`
+environment variable. Try around ~4 at most, although the absolute maximum allowed is 8.
+
+**Be nice.** it's likely entirely possible your IP address could get throttled by the
+source if you try and crawl extremely large amounts very quickly. **Try and be polite
+with the smallest amount of indexing and concurrent downloads possible for your needs.**
+
 
 # FAQ
 
@@ -224,7 +282,7 @@ automatically.
 ### Does TubeSync support any other video platforms?
 
 At the moment, no. This is a pre-release. The library TubeSync uses that does most
-of the downloading work, `youtube-dl`, supports many hundreds of video sources so it's
+of the downloading work, `yt-dlp`, supports many hundreds of video sources so it's
 likely more will be added to TubeSync if there is demand for it.
 
 ### Is there a progress bar?
@@ -236,27 +294,27 @@ your install is doing check the container logs.
 
 ### Are there alerts when a download is complete?
 
-No, this feature is best served by existing services such as the execelent
+No, this feature is best served by existing services such as the excellent 
 [Tautulli](https://tautulli.com/) which can monitor your Plex server and send alerts
 that way.
 
-### There's errors in my "tasks" tab!
+### There are errors in my "tasks" tab!
 
 You only really need to worry about these if there is a permanent failure. Some errors
-are temproary and will be retried for you automatically, such as a download got
-interrupted and will be tried again later. Sources with permanet errors (such as no
+are temporary and will be retried for you automatically, such as a download got
+interrupted and will be tried again later. Sources with permanent errors (such as no
 media available because you got a channel name wrong) will be shown as errors on the
 "sources" tab.
 
 ### What is TubeSync written in?
 
-Python3 using Django, embedding youtube-dl. It's pretty much glue between other much
+Python3 using Django, embedding yt-dlp. It's pretty much glue between other much
 larger libraries.
 
 Notable libraries and software used:
 
  * [Django](https://www.djangoproject.com/)
- * [youtube-dl](https://yt-dl.org/)
+ * [yt-dlp](https://github.com/yt-dlp/yt-dlp)
  * [ffmpeg](https://ffmpeg.org/)
  * [Django Background Tasks](https://github.com/arteria/django-background-tasks/)
  * [django-sass](https://github.com/coderedcorp/django-sass/)
@@ -266,7 +324,7 @@ See the [Pipefile](https://github.com/meeb/tubesync/blob/main/Pipfile) for a ful
 
 ### Can I get access to the full Django admin?
 
-Yes, although pretty much all operations are available through the front end interface
+Yes, although pretty much all operations are available through the front-end interface
 and you can probably break things by playing in the admin. If you still want to access
 it you can run:
 
@@ -279,7 +337,9 @@ can log in at http://localhost:4848/admin
 
 ### Are there user accounts or multi-user support?
 
-No not at the moment. This could be added later if there is demand for it.
+There is support for basic HTTP authentication by setting the `HTTP_USER` and
+`HTTP_PASS` environment variables. There is not support for multi-user or user
+management.
 
 ### Does TubeSync support HTTPS?
 
@@ -298,20 +358,24 @@ There are a number of other environment variables you can set. These are, mostly
 useful if you are manually installing TubeSync in some other environment. These are:
 
 | Name                     | What                                                         | Example                              |
-| ------------------------ | ------------------------------------- | ---------------------------------- |
+| ------------------------ | ------------------------------------------------------------ | ------------------------------------ |
 | DJANGO_SECRET_KEY        | Django's SECRET_KEY                                          | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l     |
-| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME            | /somepath                          |
+| DJANGO_URL_PREFIX        | Run TubeSync in a sub-URL on the web server                  | /somepath/                           |
 | TUBESYNC_DEBUG           | Enable debugging                                             | True                                 |
-| TUBESYNC_HOSTS           | Django's ALLOWED_HOSTS                | tubesync.example.com,otherhost.com |
+| TUBESYNC_WORKERS         | Number of background workers, default is 2, max allowed is 8 | 2                                    |
+| TUBESYNC_HOSTS           | Django's ALLOWED_HOSTS, defaults to `*`                      | tubesync.example.com,otherhost.com   |
 | GUNICORN_WORKERS         | Number of gunicorn workers to spawn                          | 3                                    |
 | LISTEN_HOST              | IP address for gunicorn to listen on                         | 127.0.0.1                            |
 | LISTEN_PORT              | Port number for gunicorn to listen on                        | 8080                                 |
+| HTTP_USER                | Sets the username for HTTP basic authentication              | some-username                        |
+| HTTP_PASS                | Sets the password for HTTP basic authentication              | some-secure-password                 |
+| DATABASE_CONNECTION      | Optional external database connection details                | mysql://user:pass@host:port/database |
 
 
 # Manual, non-containerised, installation
 
 As a relatively normal Django app you can run TubeSync without the container. Beyond
-following this rough guide you are on your own and should be knowledgeable about
+following this rough guide, you are on your own and should be knowledgeable about
 installing and running WSGI-based Python web applications before attempting this.
 
 1. Clone or download this repo
@@ -322,7 +386,7 @@ installing and running WSGI-based Python web applications before attempting this
    `tubesync/tubesync/local_settings.py` and edit it as appropriate
 5. Run migrations with `./manage.py migrate`
 6. Collect static files with `./manage.py collectstatic`
-6. Set up your prefered WSGI server, such as `gunicorn` poiting it to the application
+6. Set up your prefered WSGI server, such as `gunicorn` pointing it to the application
    in `tubesync/tubesync/wsgi.py`
 7. Set up your proxy server such as `nginx` and forward it to the WSGI server
 8. Check the web interface is working
@@ -334,7 +398,7 @@ installing and running WSGI-based Python web applications before attempting this
 
 # Tests
 
-There is a moderately comprehensive test suite focussing on the custom media format
+There is a moderately comprehensive test suite focusing on the custom media format
 matching logic and that the front-end interface works. You can run it via Django:
 
 ```bash

config/root/etc/cont-init.d/10-tubesync

@@ -1,27 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-# Change runtime user UID and GID
-PUID=${PUID:-911}
-PGID=${PGID:-911}
-groupmod -o -g "$PGID" app
-usermod -o -u "$PUID" app
-
-# Reset permissions
-chown -R app:app /run/app && \
-chmod -R 0700 /run/app && \
-chown -R app:app /config && \
-chmod -R 0755 /config && \
-chown -R app:app /downloads && \
-chmod -R 0755 /downloads && \
-chown -R root:app /app && \
-chmod -R 0750 /app && \
-chown -R app:app /app/common/static && \
-chmod -R 0750 /app/common/static && \
-chown -R app:app /app/static && \
-chmod -R 0750 /app/static && \
-find /app -type f -exec chmod 640 {} \; && \
-chmod +x /app/healthcheck.py
-
-# Run migrations
-exec s6-setuidgid app \
-    /usr/bin/python3 /app/manage.py migrate

config/root/etc/nginx/nginx.conf

@@ -79,6 +79,11 @@ http {
             proxy_connect_timeout 10;
         }
 
+        # File dwnload and streaming
+        location /media-data/ {
+            internal;
+            alias /downloads/;
+        }
     }
 
 }

config/root/etc/redis/redis.conf

@@ -0,0 +1,46 @@
+bind 127.0.0.1
+protected-mode yes
+port 6379
+tcp-backlog 511
+timeout 0
+tcp-keepalive 300
+daemonize no
+supervised no
+loglevel notice
+logfile ""
+databases 1
+always-show-logo no
+save ""
+dir /var/lib/redis
+maxmemory 64mb
+maxmemory-policy noeviction
+lazyfree-lazy-eviction no
+lazyfree-lazy-expire no
+lazyfree-lazy-server-del no
+replica-lazy-flush no
+lazyfree-lazy-user-del no
+oom-score-adj no
+oom-score-adj-values 0 200 800
+appendonly no
+appendfsync no
+lua-time-limit 5000
+slowlog-log-slower-than 10000
+slowlog-max-len 128
+latency-monitor-threshold 0
+notify-keyspace-events ""
+hash-max-ziplist-entries 512
+hash-max-ziplist-value 64
+list-max-ziplist-size -2
+list-compress-depth 0
+set-max-intset-entries 512
+zset-max-ziplist-entries 128
+zset-max-ziplist-value 64
+hll-sparse-max-bytes 3000
+stream-node-max-bytes 4096
+stream-node-max-entries 100
+activerehashing yes
+client-output-buffer-limit normal 0 0 0
+client-output-buffer-limit replica 256mb 64mb 60
+client-output-buffer-limit pubsub 32mb 8mb 60
+hz 10
+dynamic-hz yes

config/root/etc/s6-overlay/s6-rc.d/celery-beat/dependencies

@@ -0,0 +1 @@
+gunicorn

config/root/etc/s6-overlay/s6-rc.d/celery-beat/run

@@ -0,0 +1,25 @@
+#!/usr/bin/with-contenv bash
+
+UMASK_SET=${UMASK_SET:-022}
+umask "$UMASK_SET"
+
+cd /app || exit
+
+PIDFILE=/run/app/celery-beat.pid
+SCHEDULE=/tmp/tubesync-celerybeat-schedule
+
+if [ -f "${PIDFILE}" ]
+then
+    PID=$(cat $PIDFILE)
+    echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
+    if kill -0 $PID
+    then
+        echo "Killing old gunicorn process with PID: ${PID}"
+        kill -9 $PID
+    fi
+    echo "Removing stale PID file: ${PIDFILE}"
+    rm ${PIDFILE}
+fi
+
+#exec s6-setuidgid app \
+#    /usr/local/bin/celery --workdir /app -A tubesync beat --pidfile ${PIDFILE} -s ${SCHEDULE}

config/root/etc/s6-overlay/s6-rc.d/celery-beat/type

@@ -0,0 +1 @@
+longrun

config/root/etc/s6-overlay/s6-rc.d/celery-worker/dependencies

@@ -0,0 +1 @@
+gunicorn

config/root/etc/s6-overlay/s6-rc.d/celery-worker/run

@@ -0,0 +1,24 @@
+#!/usr/bin/with-contenv bash
+
+UMASK_SET=${UMASK_SET:-022}
+umask "$UMASK_SET"
+
+cd /app || exit
+
+PIDFILE=/run/app/celery-worker.pid
+
+if [ -f "${PIDFILE}" ]
+then
+    PID=$(cat $PIDFILE)
+    echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
+    if kill -0 $PID
+    then
+        echo "Killing old gunicorn process with PID: ${PID}"
+        kill -9 $PID
+    fi
+    echo "Removing stale PID file: ${PIDFILE}"
+    rm ${PIDFILE}
+fi
+
+#exec s6-setuidgid app \
+#    /usr/local/bin/celery --workdir /app -A tubesync worker --pidfile ${PIDFILE} -l INFO

config/root/etc/s6-overlay/s6-rc.d/celery-worker/type

@@ -0,0 +1 @@
+longrun

config/root/etc/s6-overlay/s6-rc.d/gunicorn/dependencies

@@ -0,0 +1 @@
+tubesync-init

config/root/etc/s6-overlay/s6-rc.d/gunicorn/run

@@ -0,0 +1,24 @@
+#!/command/with-contenv bash
+
+UMASK_SET=${UMASK_SET:-022}
+umask "$UMASK_SET"
+
+cd /app || exit
+
+PIDFILE=/run/app/gunicorn.pid
+
+if [ -f "${PIDFILE}" ]
+then
+    PID=$(cat $PIDFILE)
+    echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
+    if kill -0 $PID
+    then
+        echo "Killing old gunicorn process with PID: ${PID}"
+        kill -9 $PID
+    fi
+    echo "Removing stale PID file: ${PIDFILE}"
+    rm ${PIDFILE}
+fi
+
+exec s6-setuidgid app \
+    /usr/local/bin/gunicorn -c /app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application

config/root/etc/s6-overlay/s6-rc.d/gunicorn/type

@@ -0,0 +1 @@
+longrun

config/root/etc/s6-overlay/s6-rc.d/nginx/dependencies

@@ -0,0 +1 @@
+gunicorn

config/root/etc/s6-overlay/s6-rc.d/nginx/run

@@ -0,0 +1,5 @@
+#!/command/with-contenv bash
+
+cd /
+
+/usr/sbin/nginx

config/root/etc/s6-overlay/s6-rc.d/nginx/type

@@ -0,0 +1 @@
+longrun

config/root/etc/s6-overlay/s6-rc.d/redis/run

@@ -0,0 +1,4 @@
+#!/command/with-contenv bash
+
+exec s6-setuidgid redis \
+    /usr/bin/redis-server /etc/redis/redis.conf

config/root/etc/s6-overlay/s6-rc.d/redis/type

@@ -0,0 +1 @@
+longrun

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run

@@ -0,0 +1,27 @@
+#!/command/with-contenv bash
+
+# Change runtime user UID and GID
+PUID="${PUID:-911}"
+PUID="${PUID:-911}"
+groupmod -o -g "$PGID" app
+usermod -o -u "$PUID" app
+
+# Reset permissions
+chown -R app:app /run/app
+chmod -R 0700 /run/app
+chown -R app:app /config
+chmod -R 0755 /config
+chown -R app:app /downloads
+chmod -R 0755 /downloads
+chown -R root:app /app
+chmod -R 0750 /app
+chown -R app:app /app/common/static
+chmod -R 0750 /app/common/static
+chown -R app:app /app/static
+chmod -R 0750 /app/static
+find /app -type f ! -iname healthcheck.py -exec chmod 640 {} \;
+chmod 0755 /app/healthcheck.py
+
+# Run migrations
+exec s6-setuidgid app \
+    /usr/bin/python3 /app/manage.py migrate

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/timeout-up

@@ -0,0 +1 @@
+60000

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/type

@@ -0,0 +1 @@
+oneshot

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/up

@@ -0,0 +1,3 @@
+#!/command/execlineb -P
+
+/etc/s6-overlay/s6-rc.d/tubesync-init/run

config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/dependencies

@@ -0,0 +1 @@
+gunicorn

config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run

@@ -1,4 +1,4 @@
-#!/usr/bin/with-contenv bash
+#!/command/with-contenv bash
 
 exec s6-setuidgid app \
     /usr/bin/python3 /app/manage.py process_tasks

config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/type

@@ -0,0 +1 @@
+longrun

config/root/etc/services.d/gunicorn/run

@@ -1,9 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-UMASK_SET=${UMASK_SET:-022}
-umask "$UMASK_SET"
-
-cd /app || exit
-
-exec s6-setuidgid app \
-    /usr/local/bin/gunicorn -c /app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application

config/root/etc/services.d/nginx/run

@@ -1,5 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-cd /
-
-/usr/sbin/nginx

docs/create-missing-metadata.md

@@ -0,0 +1,37 @@
+# TubeSync
+
+## Advanced usage guide - creating missing metadata
+
+This is a new feature in v0.9 of TubeSync and later. It allows you to create or
+re-create missing metadata in your TubeSync download directories for missing `nfo`
+files and thumbnails.
+
+If you add a source with "write NFO files" or "copy thumbnails" disabled, download
+some media and then update the source to write NFO files or copy thumbnails then
+TubeSync will not automatically retroactively attempt to copy or create your missing
+metadata files. You can use a special one-off command to manually write missing
+metadata files to the correct locations.
+
+## Requirements
+
+You have added a source without metadata writing enabled, downloaded some media, then
+updated the source to enable metadata writing.
+
+## Steps
+
+### 1. Run the batch metadata sync command
+
+Execute the following Django command:
+
+`./manage.py sync-missing-metadata`
+
+When deploying TubeSync inside a container, you can execute this with:
+
+`docker exec -ti tubesync python3 /app/manage.py sync-missing-metadata`
+
+This command will log what its doing to the terminal when you run it.
+
+Internally, this command loops over all your sources which have been saved with
+"write NFO files" or "copy thumbnails" enabled. Then, loops over all media saved to
+that source and confirms that the appropriate thumbnail files have been copied over and
+the NFO file has been written if enabled.

docs/import-existing-media.md

@@ -0,0 +1,81 @@
+# TubeSync
+
+## Advanced usage guide - importing existing media
+
+This is a new feature in v0.9 of TubeSync and later. It allows you to mark existing
+downloaded media as "downloaded" in TubeSync. You can use this feature if, for example,
+you already have an extensive catalogue of downloaded media which you want to mark
+as downloaded into TubeSync so TubeSync doesn't re-download media you already have.
+
+## Requirements
+
+Your existing downloaded media MUST contain the unique ID. For YouTube videos, this is
+means the YouTube video ID MUST be in the filename.
+
+Supported extensions to be imported are .m4a, .ogg, .mkv, .mp3, .mp4 and .avi. Your
+media you want to import must end in one of these file extensions.
+
+## Caveats
+
+As TubeSync does not probe media and your existing media may be re-encoded or in
+different formats to what is available in the current media metadata there is no way
+for TubeSync to know what codecs, resolution, bitrate etc. your imported media is in.
+Any manually imported existing local media will display blank boxes for this
+information on the TubeSync interface as it's unavailable.
+
+## Steps
+
+### 1. Add your source to TubeSync
+
+Add your source to TubeSync, such as a YouTube channel. **Make sure you untick the
+"download media" checkbox.**
+
+This will allow TubeSync to index all the available media on your source, but won't
+start downloading any media.
+
+### 2. Wait
+
+Wait for all the media on your source to be indexed. This may take some time.
+
+### 3. Move your existing media into TubeSync
+
+You now need to move your existing media into TubeSync. You need to move the media
+files into the correct download directories created by TubeSync. For example, if you
+have downloaded videos for a YouTube channel "TestChannel", you would have added this
+as a source called TestChannel and in a directory called test-channel in Tubesync. It
+would have a download directory created on disk at:
+
+`/path/to/downloads/test-channel`
+
+You would move all of your pre-existing videos you downloaded outside of TubeSync for
+this channel into this directory.
+
+In short, your existing media needs to be moved into the correct TubeSync source
+directory to be detected.
+
+This is required so TubeSync can known which Source to link the media to.
+
+### 4. Run the batch import command
+
+Execute the following Django command:
+
+`./manage.py import-existing-media`
+
+When deploying TubeSync inside a container, you can execute this with:
+
+`docker exec -ti tubesync python3 /app/manage.py import-existing-media`
+
+This command will log what its doing to the terminal when you run it.
+
+Internally, `import-existing-media` looks for the unique media key (for YouTube, this
+is the YouTube video ID) in the filename and detects the source to link it to based
+on the directory the media file is inside.
+
+
+### 5. Re-enable downloading at the source
+
+Edit your source and re-enable / tick the "download media" option. This will allow
+TubeSync to download any missing media you did not manually import.
+
+Note that TubeSync will still get screenshots write `nfo` files etc. for files you
+manually import if enabled at the source level.

docs/other-database-backends.md

@@ -0,0 +1,80 @@
+# TubeSync
+
+## Advanced usage guide - using other database backends
+
+This is a new feature in v1.0 of TubeSync and later. It allows you to use a custom
+existing external database server instead of the default SQLite database. You may want
+to use this if you encounter performance issues with adding very large or a large
+number of channels and database write contention (as shown by errors in the log)
+become an issue.
+
+## Requirements
+
+TubeSync supports SQLite (the automatic default) as well as PostgreSQL, MySQL and
+MariaDB. For MariaDB just follow the MySQL instructions as the driver is the same.
+
+You should start with a blank install of TubeSync. Migrating to a new database will
+reset your database. If you are comfortable with Django you can export and re-import
+existing database data with:
+
+```bash
+$ docker exec -i tubesync python3 /app/manage.py dumpdata > some-file.json
+```
+
+Then change you database backend over, then use
+
+```bash
+$ cat some-file.json | docker exec -i tubesync python3 /app/manage.py loaddata --format=json -
+```
+
+As detailed in the Django documentation:
+
+https://docs.djangoproject.com/en/3.1/ref/django-admin/#dumpdata
+
+and:
+
+https://docs.djangoproject.com/en/3.1/ref/django-admin/#loaddata
+
+Further instructions are beyond the scope of TubeSync documenation and you should refer
+to Django documentation for more details.
+
+If you are not comfortable with the above, then skip the `dumpdata` steps, however
+remember you will start again with a completely new database.
+
+## Steps
+
+### 1. Create a database in your external database server
+
+You need to create a database and a user with permissions to access the database in
+your chosen external database server. Steps vary between PostgreSQL, MySQL and MariaDB
+so this is up to you to work out.
+
+### 2. Set the database connection string environment variable
+
+You need to provide the database connection details to TubeSync via an environment
+variable. The environment variable name is `DATABASE_CONNECTION` and the format is the
+standard URL-style string. Examples are:
+
+`postgresql://tubesync:password@localhost:5432/tubesync`
+
+and
+
+`mysql://tubesync:password@localhost:3306/tubesync`
+
+*Important note:* For MySQL databases make SURE you create the tubesync database with
+`utf8mb4` encoding, like:
+
+`CREATE DATABASE tubesync CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;`
+
+Without `utf8mb4` encoding things like emojis in video titles (or any extended UTF8
+characters) can cause issues.
+
+### 3. Start TubeSync and check the logs
+
+Once you start TubeSync with the new database connection you should see the folling log
+entry in the container or stdout logs:
+
+`2021-04-04 22:42:17,912 [tubesync/INFO] Using database connection: django.db.backends.postgresql://tubesync:[hidden]@localhost:5432/tubesync`
+
+If you see a line similar to the above and the web interface loads, congratulations,
+you are now using an external database server for your TubeSync data!

docs/reset-tasks.md

@@ -0,0 +1,33 @@
+# TubeSync
+
+## Advanced usage guide - reset tasks from the command line
+
+This is a new feature in v1.0 of TubeSync and later. It allows you to reset all
+scheduled tasks from the command line as well as the "reset tasks" button in the
+"tasks" tab of the dashboard.
+
+This is useful for TubeSync installations where you may have a lot of media and
+sources added and the "reset tasks" button may take too long to the extent where
+the page times out (with a 502 error or similar issue).
+
+## Requirements
+
+You have added some sources and media
+
+## Steps
+
+### 1. Run the reset tasks command
+
+Execute the following Django command:
+
+`./manage.py reset-tasks`
+
+When deploying TubeSync inside a container, you can execute this with:
+
+`docker exec -ti tubesync python3 /app/manage.py reset-tasks`
+
+This command will log what its doing to the terminal when you run it.
+
+When this is run, new tasks will be immediately created so all your sources will be
+indexed again straight away, any missing information such as thumbnails will be
+redownloaded, etc.

docs/using-cookies.md

@@ -0,0 +1,50 @@
+# TubeSync
+
+## Advanced usage guide - using exported cookies
+
+This is a new feature in v0.10 of TubeSync and later. It allows you to use the cookies
+file exported from your browser in "Netscape" format with TubeSync to authenticate
+to YouTube. This can bypass some throttling, age restrictions and other blocks at
+YouTube.
+
+**IMPORTANT NOTE**: Using cookies exported from your browser that is authenticated
+to YouTube identifes your Google account as using TubeSync. This may result in
+potential account impacts and is entirely at your own risk. Do not use this
+feature unless you really know what you're doing.
+
+## Requirements
+
+Have a browser that supports exporting your cookies and be logged into YouTube.
+
+## Steps
+
+### 1. Export your cookies
+
+You need to export cookies for youtube.com from your browser, you can either do
+this manually or there are plug-ins to automate this for you. This file must be
+in the "Netscape" cookie export format.
+
+Save your cookies as a `cookies.txt` file.
+
+### 2. Import into TubeSync
+
+Drop the `cookies.txt` file into your TubeSync `config` directory.
+
+If detected correctly, you will see something like this in the worker or container
+logs:
+
+```
+YYYY-MM-DD HH:MM:SS,mmm [tubesync/INFO] [youtube-dl] using cookies.txt from: /config/cookies.txt
+```
+
+If you see that line it's working correctly.
+
+If you see errors in your logs like this:
+
+```
+http.cookiejar.LoadError: '/config/cookies.txt' does not look like a Netscape format cookies file
+```
+
+Then your `cookies.txt` file was not generated or created correctly as it's not
+in the required "Netscape" format. You can fix this by exporting your `cookies.txt`
+in the correct "Netscape" format.

pip.conf

@@ -0,0 +1,2 @@
+[global]
+extra-index-url=https://www.piwheels.org/simple

tubesync/common/context_processors.py

@@ -1,10 +1,10 @@
 from django.conf import settings
-from .third_party_versions import youtube_dl_version, ffmpeg_version
+from .third_party_versions import yt_dlp_version, ffmpeg_version
 
 
 def app_details(request):
     return {
         'app_version': str(settings.VERSION),
-        'youtube_dl_version': youtube_dl_version,
+        'yt_dlp_version': yt_dlp_version,
         'ffmpeg_version': ffmpeg_version,
     }

tubesync/common/errors.py

@@ -20,3 +20,10 @@ class DownloadFailedException(Exception):
         exist.
     '''
     pass
+
+
+class DatabaseConnectionError(Exception):
+    '''
+        Raised when parsing or initially connecting to a database.
+    '''
+    pass

tubesync/common/middleware.py

@@ -1,4 +1,6 @@
+from django.conf import settings
 from django.forms import BaseForm
+from basicauth.middleware import BasicAuthMiddleware as BaseBasicAuthMiddleware
 
 
 class MaterializeDefaultFieldsMiddleware:
@@ -19,3 +21,12 @@ class MaterializeDefaultFieldsMiddleware:
                 for _, field in v.fields.items():
                     field.widget.attrs.update({'class':'browser-default'})
         return response
+
+
+class BasicAuthMiddleware(BaseBasicAuthMiddleware):
+
+    def process_request(self, request):
+        bypass_uris = getattr(settings, 'BASICAUTH_ALWAYS_ALLOW_URIS', [])
+        if request.path in bypass_uris:
+            return None
+        return super().process_request(request)

tubesync/common/static/styles/_fonts.scss

@@ -1,20 +1,20 @@
 @font-face {
     font-family: 'roboto-light';
-    src: url('/static/fonts/roboto/roboto-light.woff') format('woff');
+    src: url('../fonts/roboto/roboto-light.woff') format('woff');
     font-weight: normal;
     font-style: normal;
 }
 
 @font-face {
     font-family: 'roboto-regular';
-    src: url('/static/fonts/roboto/roboto-regular.woff') format('woff');
+    src: url('../fonts/roboto/roboto-regular.woff') format('woff');
     font-weight: normal;
     font-style: normal;
 }
 
 @font-face {
     font-family: 'roboto-bold';
-    src: url('/static/fonts/roboto/roboto-bold.woff') format('woff');
+    src: url('../fonts/roboto/roboto-bold.woff') format('woff');
     font-weight: bold;
     font-style: normal;
 }

tubesync/common/static/styles/_forms.scss

@@ -4,7 +4,7 @@
     }
     .help-text {
         color: $form-help-text-colour;
-        padding: 1rem 0 1rem 0;
+        padding-bottom: 1rem;
     }
     label {
         text-transform: uppercase;

tubesync/common/static/styles/_template.scss

@@ -5,6 +5,13 @@ html {
     color: $text-colour;
 }
 
+body {
+    display: flex;
+    min-height: 100vh;
+    flex-direction: column;
+    justify-content: space-between;
+}
+
 header {
 
     background-color: $header-background-colour;
@@ -174,8 +181,10 @@ main {
             display: inline-block;
             font-weight: bold;
             text-decoration: none;
-            padding: 5px 10px 5px 10px;
-            margin: 0 3px 0 3px;
+            padding: 5px 8px 4px 8px;
+            margin: 0 3px 6px 3px;
+            min-width: 40px;
+            min-height: 40px;
             background-color: $pagination-background-colour;
             color: $pagination-text-colour;
             border: 2px $pagination-border-colour solid;

tubesync/common/templates/base.html

@@ -16,6 +16,8 @@
 
   <body>
 
+    <div class="app">
+
       <header>
         <div class="container">
           <a href="{% url 'sync:dashboard' %}">
@@ -43,6 +45,8 @@
         </div>
       </main>
 
+    </div>
+
     <footer>
       <div class="container">
         <p>
@@ -53,7 +57,7 @@
         </p>
         <p>
           <a href="https://github.com/meeb/tubesync" class="nowrap" target="_blank"><i class="fab fa-github"></i> TubeSync</a> version <strong>{{ app_version }}</strong> with
-          <a href="https://yt-dl.org/" class="nowrap" target="_blank"><i class="fas fa-link"></i> youtube-dl</a> version <strong>{{ youtube_dl_version }}</strong> and 
+          <a href="https://github.com/yt-dlp/yt-dlp" class="nowrap" target="_blank"><i class="fas fa-link"></i> yt-dlp</a> version <strong>{{ yt_dlp_version }}</strong> and 
           <a href="https://ffmpeg.org/" class="nowrap" target="_blank"><i class="fas fa-link"></i> FFmpeg</a> version <strong>{{ ffmpeg_version }}</strong>. 
         </p>
       </div>

tubesync/common/templates/pagination.html

@@ -3,7 +3,7 @@
   <div class="col s12">
     <div class="pagination">
       {% for i in paginator.page_range %}
-        <a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}">{{ i }}</a>
+        <a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}{% if show_skipped %}&show_skipped=yes{% endif %}">{{ i }}</a>
       {% endfor %}
     </div>
   </div>

tubesync/common/tests.py

@@ -2,6 +2,8 @@ import os.path
 from django.conf import settings
 from django.test import TestCase, Client
 from .testutils import prevent_request_warnings
+from .utils import parse_database_connection_string, clean_filename
+from .errors import DatabaseConnectionError
 
 
 class ErrorPageTestCase(TestCase):
@@ -61,3 +63,75 @@ class CommonStaticTestCase(TestCase):
         favicon_real_path = os.path.join(os.sep.join(root_parts),
                                          os.sep.join(url_parts))
         self.assertTrue(os.path.exists(favicon_real_path))
+
+
+class UtilsTestCase(TestCase):
+
+    def test_parse_database_connection_string(self):
+        database_dict = parse_database_connection_string(
+            'postgresql://tubesync:password@localhost:5432/tubesync')
+        self.assertEqual(database_dict,
+            {
+                'DRIVER': 'postgresql',
+                'ENGINE': 'django.db.backends.postgresql',
+                'USER': 'tubesync',
+                'PASSWORD': 'password',
+                'HOST': 'localhost',
+                'PORT': 5432,
+                'NAME': 'tubesync',
+                'CONN_MAX_AGE': 300,
+                'OPTIONS': {},
+            }
+        )
+        database_dict = parse_database_connection_string(
+            'mysql://tubesync:password@localhost:3306/tubesync')
+        self.assertEqual(database_dict,
+            {
+                'DRIVER': 'mysql',
+                'ENGINE': 'django.db.backends.mysql',
+                'USER': 'tubesync',
+                'PASSWORD': 'password',
+                'HOST': 'localhost',
+                'PORT': 3306,
+                'NAME': 'tubesync',
+                'CONN_MAX_AGE': 300,
+                'OPTIONS': {'charset': 'utf8mb4'}
+            }
+        )
+        # Invalid driver
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'test://tubesync:password@localhost:5432/tubesync')
+        # No username
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'postgresql://password@localhost:5432/tubesync')
+        # No database name
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'postgresql://tubesync:password@5432')
+        # Invalid port
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'postgresql://tubesync:password@localhost:test/tubesync')
+        # Invalid port
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'postgresql://tubesync:password@localhost:65537/tubesync')
+        # Invalid username or password
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'postgresql://tubesync:password:test@localhost:5432/tubesync')
+        # Invalid database name
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'postgresql://tubesync:password@localhost:5432/tubesync/test')
+
+    def test_clean_filename(self):
+        self.assertEqual(clean_filename('a'), 'a')
+        self.assertEqual(clean_filename('a\t'), 'a')
+        self.assertEqual(clean_filename('a\n'), 'a')
+        self.assertEqual(clean_filename('a a'), 'a a')
+        self.assertEqual(clean_filename('a  a'), 'a  a')
+        self.assertEqual(clean_filename('a\t\t\ta'), 'a   a')
+        self.assertEqual(clean_filename('a\t\t\ta\t\t\t'), 'a   a')

tubesync/common/third_party_versions.py

@@ -1,7 +1,7 @@
-from youtube_dl import version as yt_version
+from yt_dlp import version as yt_dlp_version
 
 
-youtube_dl_version = str(yt_version.__version__)
+yt_dlp_version = str(yt_dlp_version.__version__)
 ffmpeg_version = '(shared install)'
 
 

tubesync/common/utils.py

@@ -1,4 +1,96 @@
-from urllib.parse import urlunsplit, urlencode
+import string
+from datetime import datetime
+from urllib.parse import urlunsplit, urlencode, urlparse
+from yt_dlp.utils import LazyList
+from .errors import DatabaseConnectionError
+
+
+def parse_database_connection_string(database_connection_string):
+    '''
+        Parses a connection string in a URL style format, such as:
+            postgresql://tubesync:password@localhost:5432/tubesync
+            mysql://someuser:somepassword@localhost:3306/tubesync
+        into a Django-compatible settings.DATABASES dict format. 
+    '''
+    valid_drivers = ('postgresql', 'mysql')
+    default_ports = {
+        'postgresql': 5432,
+        'mysql': 3306,
+    }
+    django_backends = {
+        'postgresql': 'django.db.backends.postgresql',
+        'mysql': 'django.db.backends.mysql',
+    }
+    backend_options = {
+        'postgresql': {},
+        'mysql': {
+            'charset': 'utf8mb4',
+        }
+    }
+    try:
+        parts = urlparse(str(database_connection_string))
+    except Exception as e:
+        raise DatabaseConnectionError(f'Failed to parse "{database_connection_string}" '
+                                      f'as a database connection string: {e}') from e
+    driver = parts.scheme
+    user_pass_host_port = parts.netloc
+    database = parts.path
+    if driver not in valid_drivers:
+        raise DatabaseConnectionError(f'Database connection string '
+                                      f'"{database_connection_string}" specified an '
+                                      f'invalid driver, must be one of {valid_drivers}')
+    django_driver = django_backends.get(driver)
+    host_parts = user_pass_host_port.split('@')
+    if len(host_parts) != 2:
+        raise DatabaseConnectionError(f'Database connection string netloc must be in '
+                                      f'the format of user:pass@host')
+    user_pass, host_port = host_parts
+    user_pass_parts = user_pass.split(':')
+    if len(user_pass_parts) != 2:
+        raise DatabaseConnectionError(f'Database connection string netloc must be in '
+                                      f'the format of user:pass@host')
+    username, password = user_pass_parts
+    host_port_parts = host_port.split(':')
+    if len(host_port_parts) == 1:
+        # No port number, assign a default port
+        hostname = host_port_parts[0]
+        port = default_ports.get(driver)
+    elif len(host_port_parts) == 2:
+        # Host name and port number
+        hostname, port = host_port_parts
+        try:
+            port = int(port)
+        except (ValueError, TypeError) as e:
+            raise DatabaseConnectionError(f'Database connection string contained an '
+                                          f'invalid port, ports must be integers: '
+                                          f'{e}') from e
+        if not 0 < port < 63336:
+            raise DatabaseConnectionError(f'Database connection string contained an '
+                                          f'invalid port, ports must be between 1 and '
+                                          f'65535, got {port}')
+    else:
+        # Malformed
+        raise DatabaseConnectionError(f'Database connection host must be a hostname or '
+                                      f'a hostname:port combination')
+    if database.startswith('/'):
+        database = database[1:]
+    if not database:
+        raise DatabaseConnectionError(f'Database connection string path must be a '
+                                      f'string in the format of /databasename')    
+    if '/' in database:
+        raise DatabaseConnectionError(f'Database connection string path can only '
+                                      f'contain a single string name, got: {database}')
+    return {
+        'DRIVER': driver,
+        'ENGINE': django_driver,
+        'NAME': database,
+        'USER': username,
+        'PASSWORD': password,
+        'HOST': hostname,
+        'PORT': port,
+        'CONN_MAX_AGE': 300,
+        'OPTIONS': backend_options.get(driver),
+    }
 
 
 def get_client_ip(request):
@@ -14,3 +106,26 @@ def append_uri_params(uri, params):
     uri = str(uri)
     qs = urlencode(params)
     return urlunsplit(('', '', uri, qs, ''))
+
+
+def clean_filename(filename):
+    if not isinstance(filename, str):
+        raise ValueError(f'filename must be a str, got {type(filename)}')
+    to_scrub = '<>\/:*?"|%'
+    for char in to_scrub:
+        filename = filename.replace(char, '')
+    clean_filename = ''
+    for c in filename:
+        if c in string.whitespace:
+            c = ' '
+        if ord(c) > 30:
+            clean_filename += c
+    return clean_filename.strip()
+
+
+def json_serial(obj):
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    if isinstance(obj, LazyList):
+        return list(obj)
+    raise TypeError(f'Type {type(obj)} is not json_serial()-able')

tubesync/sync/admin.py

@@ -7,7 +7,7 @@ class SourceAdmin(admin.ModelAdmin):
 
     ordering = ('-created',)
     list_display = ('uuid', 'name', 'source_type', 'last_crawl',
-                    'has_failed')
+                    'download_media', 'has_failed')
     readonly_fields = ('uuid', 'created')
     search_fields = ('uuid', 'key', 'name')
 

tubesync/sync/management/commands/delete-source.py

@@ -0,0 +1,51 @@
+import os
+import uuid
+from django.utils.translation import gettext_lazy as _
+from django.core.management.base import BaseCommand, CommandError
+from django.db.models import signals
+from common.logger import log
+from sync.models import Source, Media, MediaServer
+from sync.signals import media_post_delete
+from sync.tasks import rescan_media_server
+
+
+class Command(BaseCommand):
+
+    help = ('Deletes a source by UUID')
+
+    def add_arguments(self, parser):
+        parser.add_argument('--source', action='store', required=True, help='Source UUID')
+
+    def handle(self, *args, **options):
+        source_uuid_str = options.get('source', '')
+        try:
+            source_uuid = uuid.UUID(source_uuid_str)
+        except Exception as e:
+            raise CommandError(f'Failed to parse source UUID: {e}')
+        log.info(f'Deleting source with UUID: {source_uuid}')
+        # Fetch the source by UUID
+        try:
+            source = Source.objects.get(uuid=source_uuid)
+        except Source.DoesNotExist:
+            raise CommandError(f'Source does not exist with '
+                               f'UUID: {source_uuid}')
+        # Detach post-delete signal for Media so we don't spam media servers
+        signals.post_delete.disconnect(media_post_delete, sender=Media)
+        # Delete the source, triggering pre-delete signals for each media item
+        log.info(f'Found source with UUID "{source.uuid}" with name '
+                 f'"{source.name}" and deleting it, this may take some time!')
+        source.delete()
+        # Update any media servers
+        for mediaserver in MediaServer.objects.all():
+            log.info(f'Scheduling media server updates')
+            verbose_name = _('Request media server rescan for "{}"')
+            rescan_media_server(
+                str(mediaserver.pk),
+                priority=0,
+                verbose_name=verbose_name.format(mediaserver),
+                remove_existing_tasks=True
+            )
+        # Re-attach signals
+        signals.post_delete.connect(media_post_delete, sender=Media)
+        # All done
+        log.info('Done')

tubesync/sync/management/commands/import-existing-media.py

@@ -0,0 +1,55 @@
+import os
+from pathlib import Path
+from django.core.management.base import BaseCommand, CommandError
+from common.logger import log
+from sync.models import Source, Media
+
+
+class Command(BaseCommand):
+
+    help = ('Scans download media directories for media not yet downloaded and ',
+            'marks them as downloaded')
+    extra_extensions = ['mp3', 'mp4', 'avi']
+
+    def handle(self, *args, **options):
+        log.info('Building directory to Source map...')
+        dirmap = {}
+        for s in Source.objects.all():
+            dirmap[s.directory_path] = s
+        log.info(f'Scanning sources...')
+        file_extensions = list(Source.EXTENSIONS) + self.extra_extensions
+        for sourceroot, source in dirmap.items():
+            media = list(Media.objects.filter(source=source, downloaded=False,
+                                              skip=False))
+            if not media:
+                log.info(f'Source "{source}" has no missing media')
+                continue
+            log.info(f'Scanning Source "{source}" directory for media to '
+                     f'import: {sourceroot}, looking for {len(media)} '
+                     f'undownloaded and unskipped items')
+            on_disk = []
+            for (root, dirs, files) in os.walk(sourceroot):
+                rootpath = Path(root)
+                for filename in files:
+                    filepart, ext = os.path.splitext(filename)
+                    if ext.startswith('.'):
+                        ext = ext[1:]
+                    ext = ext.strip().lower()
+                    if ext not in file_extensions:
+                        continue
+                    on_disk.append(str(rootpath / filename))
+            filemap = {}
+            for item in media:
+                for filepath in on_disk:
+                    if item.key in filepath:
+                        # The unique item key is in the file name on disk, map it to
+                        # the undownloaded media item
+                        filemap[filepath] = item
+                        continue
+            for filepath, item in filemap.items():
+                log.info(f'Matched on-disk file: {filepath} '
+                         f'to media item: {item.source} / {item}')
+                item.media_file.name = filepath
+                item.downloaded = True
+                item.save()
+        log.info('Done')

tubesync/sync/management/commands/list-sources.py

@@ -0,0 +1,15 @@
+import os
+from django.core.management.base import BaseCommand, CommandError
+from common.logger import log
+from sync.models import Source, Media, MediaServer
+
+
+class Command(BaseCommand):
+
+    help = ('Lists sources')
+
+    def handle(self, *args, **options):
+        log.info('Listing sources...')
+        for source in Source.objects.all():
+            log.info(f' - {source.uuid}: {source.name}')
+        log.info('Done')

tubesync/sync/management/commands/reset-tasks.py

@@ -0,0 +1,33 @@
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import gettext_lazy as _
+from background_task.models import Task
+from sync.models import Source
+from sync.tasks import index_source_task
+
+
+from common.logger import log
+
+
+class Command(BaseCommand):
+
+    help = 'Resets all tasks'
+
+    def handle(self, *args, **options):
+        log.info('Resettings all tasks...')
+        # Delete all tasks
+        Task.objects.all().delete()
+        # Iter all tasks
+        for source in Source.objects.all():
+            # Recreate the initial indexing task
+            log.info(f'Resetting tasks for source: {source}')
+            verbose_name = _('Index media from source "{}"')
+            index_source_task(
+                str(source.pk),
+                repeat=source.index_schedule,
+                queue=str(source.pk),
+                priority=5,
+                verbose_name=verbose_name.format(source.name)
+            )
+            # This also chains down to call each Media objects .save() as well
+            source.save()
+        log.info('Done')

tubesync/sync/management/commands/sync-missing-metadata.py

@@ -0,0 +1,34 @@
+import os
+from shutil import copyfile
+from django.core.management.base import BaseCommand, CommandError
+from django.db.models import Q
+from common.logger import log
+from sync.models import Source, Media
+from sync.utils import write_text_file
+
+
+class Command(BaseCommand):
+
+    help = 'Syncs missing metadata (such as nfo files) if source settings are updated'
+
+    def handle(self, *args, **options):
+        log.info('Syncing missing metadata...')
+        sources = Source.objects.filter(Q(copy_thumbnails=True) | Q(write_nfo=True))
+        for source in sources.order_by('name'):
+            log.info(f'Finding media for source: {source}')
+            for item in Media.objects.filter(source=source, downloaded=True):
+                log.info(f'Checking media for missing metadata: {source} / {item}')
+                thumbpath = item.thumbpath
+                if not thumbpath.is_file():
+                    if item.thumb:
+                        log.info(f'Copying missing thumbnail from: {item.thumb.path} '
+                                 f'to: {thumbpath}')
+                        copyfile(item.thumb.path, thumbpath)
+                    else:
+                        log.error(f'Tried to copy missing thumbnail for {item} but '
+                                  f'the thumbnail has not been downloaded')
+                nfopath = item.nfopath
+                if not nfopath.is_file():
+                    log.info(f'Writing missing NFO file: {nfopath}')
+                    write_text_file(nfopath, item.nfoxml)
+        log.info('Done')

tubesync/sync/management/commands/youtube-dl-info.py

@@ -1,6 +1,7 @@
 import json
 from django.core.management.base import BaseCommand, CommandError
 from sync.youtube import get_media_info
+from common.utils import json_serial
 
 
 class Command(BaseCommand):
@@ -14,5 +15,6 @@ class Command(BaseCommand):
         url = options['url']
         self.stdout.write(f'Showing information for URL: {url}')
         info = get_media_info(url)
-        self.stdout.write(json.dumps(info, indent=4, sort_keys=True))
+        d = json.dumps(info, indent=4, sort_keys=True, default=json_serial)
+        self.stdout.write(d)
         self.stdout.write('Done')

tubesync/sync/matching.py

@@ -66,7 +66,7 @@ def get_best_audio_format(media):
     # No codecs matched
     if media.source.can_fallback:
         # Can fallback, find the next highest bitrate non-matching codec
-        return False, audio_formats[0]
+        return False, audio_formats[0]['id']
     else:
         # Can't fallback
         return False, False

tubesync/sync/mediaservers.py

@@ -124,7 +124,7 @@ class PlexMediaServer(MediaServer):
         # Seems we have a valid library sections page, get the library IDs
         remote_libraries = {}
         try:
-            for parent in parsed_response.getiterator('MediaContainer'):
+            for parent in parsed_response.iter('MediaContainer'):
                 for d in parent:
                     library_id = d.attrib['key']
                     library_name = d.attrib['title']

tubesync/sync/migrations/0009_auto_20210218_0442.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.1.6 on 2021-02-18 04:42
+
+import django.core.files.storage
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0008_source_download_cap'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='download_media',
+            field=models.BooleanField(default=True, help_text='Download media from this source, if not selected the source will only be indexed', verbose_name='download media'),
+        ),
+        migrations.AlterField(
+            model_name='media',
+            name='media_file',
+            field=models.FileField(blank=True, help_text='Media file', max_length=200, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='media_format',
+            field=models.CharField(default='{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'),
+        ),
+    ]

tubesync/sync/migrations/0010_auto_20210924_0554.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.2.7 on 2021-09-24 05:54
+
+import django.core.files.storage
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0009_auto_20210218_0442'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='media',
+            name='media_file',
+            field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='index_schedule',
+            field=models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours'), (259200, 'Every 3 days'), (604800, 'Every 7 days'), (0, 'Never')], db_index=True, default=86400, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='media_format',
+            field=models.CharField(default='{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'),
+        ),
+    ]

tubesync/sync/migrations/0011_auto_20220201_1654.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.11 on 2022-02-01 16:54
+
+import django.core.files.storage
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0010_auto_20210924_0554'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='write_json',
+            field=models.BooleanField(
+                default=False, help_text='Write a JSON file with the media info, these may be detected and used by some media servers', verbose_name='write json'),
+        ),
+    ]

tubesync/sync/migrations/0012_alter_media_downloaded_format.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.12 on 2022-04-06 06:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0011_auto_20220201_1654'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='media',
+            name='downloaded_format',
+            field=models.CharField(blank=True, help_text='Video format (resolution) of the downloaded media', max_length=30, null=True, verbose_name='downloaded format'),
+        ),
+    ]

tubesync/sync/migrations/0013_fix_elative_media_file.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.2.12 on 2022-04-06 06:19
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+def fix_media_file(apps, schema_editor):
+    Media = apps.get_model('sync', 'Media')
+    for media in Media.objects.filter(downloaded=True):
+        download_dir = str(settings.DOWNLOAD_ROOT)
+
+        if media.media_file.name.startswith(download_dir):
+            media.media_file.name = media.media_file.name[len(download_dir) + 1:]
+            media.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0012_alter_media_downloaded_format'),
+    ]
+
+    operations = [
+        migrations.RunPython(fix_media_file)
+    ]

tubesync/sync/migrations/0014_alter_media_media_file.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.15 on 2022-12-28 20:33
+
+import django.core.files.storage
+from django.conf import settings
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0013_fix_elative_media_file'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='media',
+            name='media_file',
+            field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=str(settings.DOWNLOAD_ROOT)), upload_to=sync.models.get_media_file_path, verbose_name='media file'),
+        ),
+    ]

tubesync/sync/models.py

@@ -12,6 +12,7 @@ from django.utils.text import slugify
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from common.errors import NoFormatException
+from common.utils import clean_filename
 from .youtube import (get_media_info as get_youtube_media_info,
                       download_media as download_youtube_media)
 from .utils import seconds_to_timestr, parse_media_format
@@ -20,7 +21,7 @@ from .matching import (get_best_combined_format, get_best_audio_format,
 from .mediaservers import PlexMediaServer
 
 
-media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT))
+media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/')
 
 
 class Source(models.Model):
@@ -100,6 +101,11 @@ class Source(models.Model):
         (FALLBACK_NEXT_BEST_HD, _('Get next best resolution but at least HD'))
     )
 
+    EXTENSION_M4A = 'm4a'
+    EXTENSION_OGG = 'ogg'
+    EXTENSION_MKV = 'mkv'
+    EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
+
     # Fontawesome icons used for the source on the front end
     ICONS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
@@ -112,6 +118,12 @@ class Source(models.Model):
         SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}',
         SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
     }
+    # Format used to create indexable URLs
+    INDEX_URLS = {
+        SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos',
+        SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos',
+        SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
+    }
     # Callback functions to get a list of media from the source
     INDEXERS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
@@ -146,6 +158,9 @@ class Source(models.Model):
         EVERY_6_HOURS = 21600, _('Every 6 hours')
         EVERY_12_HOURS = 43200, _('Every 12 hours')
         EVERY_24_HOURS = 86400, _('Every 24 hours')
+        EVERY_3_DAYS = 259200, _('Every 3 days')
+        EVERY_7_DAYS = 604800, _('Every 7 days')
+        NEVER = 0, _('Never')
 
     uuid = models.UUIDField(
         _('uuid'),
@@ -200,15 +215,20 @@ class Source(models.Model):
         _('media format'),
         max_length=200,
         default=settings.MEDIA_FORMATSTR_DEFAULT,
-        help_text=_('File format to use for saving files')
+        help_text=_('File format to use for saving files, detailed options at bottom of page.')
     )
     index_schedule = models.IntegerField(
         _('index schedule'),
         choices=IndexSchedule.choices,
         db_index=True,
-        default=IndexSchedule.EVERY_6_HOURS,
+        default=IndexSchedule.EVERY_24_HOURS,
         help_text=_('Schedule of how often to index the source for new media')
     )
+    download_media = models.BooleanField(
+        _('download media'),
+        default=True,
+        help_text=_('Download media from this source, if not selected the source will only be indexed')
+    )
     download_cap = models.IntegerField(
         _('download cap'),
         choices=CapChoices.choices,
@@ -278,6 +298,11 @@ class Source(models.Model):
         default=False,
         help_text=_('Write an NFO file in XML with the media info, these may be detected and used by some media servers')
     )
+    write_json = models.BooleanField(
+        _('write json'),
+        default=False,
+        help_text=_('Write a JSON file with the media info, these may be detected and used by some media servers')
+    )
     has_failed = models.BooleanField(
         _('has failed'),
         default=False,
@@ -327,23 +352,32 @@ class Source(models.Model):
         '''
         if self.is_audio:
             if self.source_acodec == self.SOURCE_ACODEC_MP4A:
-                return 'm4a'
+                return self.EXTENSION_M4A
             elif self.source_acodec == self.SOURCE_ACODEC_OPUS:
-                return 'ogg'
+                return self.EXTENSION_OGG
             else:
                 raise ValueError('Unable to choose audio extension, uknown acodec')
         else:
-            return 'mkv'
+            return self.EXTENSION_MKV
 
     @classmethod
     def create_url(obj, source_type, key):
         url = obj.URLS.get(source_type)
         return url.format(key=key)
 
+    @classmethod
+    def create_index_url(obj, source_type, key):
+        url = obj.INDEX_URLS.get(source_type)
+        return url.format(key=key)
+
     @property
     def url(self):
         return Source.create_url(self.source_type, self.key)
 
+    @property
+    def index_url(self):
+        return Source.create_index_url(self.source_type, self.key)
+
     @property
     def format_summary(self):
         if self.source_resolution == Source.SOURCE_RESOLUTION_AUDIO:
@@ -358,10 +392,14 @@ class Source(models.Model):
     @property
     def directory_path(self):
         download_dir = Path(media_file_storage.location)
+        return download_dir / self.type_directory_path
+
+    @property
+    def type_directory_path(self):
         if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO:
-            return download_dir / settings.DOWNLOAD_AUDIO_DIR / self.directory
+            return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory
         else:
-            return download_dir / settings.DOWNLOAD_VIDEO_DIR / self.directory
+            return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory
 
     def make_directory(self):
         return os.makedirs(self.directory_path, exist_ok=True)
@@ -399,19 +437,19 @@ class Source(models.Model):
             fmt.append('60fps')
         if self.prefer_hdr:
             fmt.append('hdr')
+        now = timezone.now()
         return {
-            'yyyymmdd': timezone.now().strftime('%Y%m%d'),
-            'yyyy_mm_dd': timezone.now().strftime('%Y-%m-%d'),
-            'yyyy': timezone.now().strftime('%Y'),
-            'mm': timezone.now().strftime('%m'),
-            'dd': timezone.now().strftime('%d'),
+            'yyyymmdd': now.strftime('%Y%m%d'),
+            'yyyy_mm_dd': now.strftime('%Y-%m-%d'),
+            'yyyy': now.strftime('%Y'),
+            'mm': now.strftime('%m'),
+            'dd': now.strftime('%d'),
             'source': self.slugname,
             'source_full': self.name,
             'title': 'some-media-title-name',
             'title_full': 'Some Media Title Name',
             'key': 'SoMeUnIqUiD',
             'format': '-'.join(fmt),
-            'playlist_index': 1,
             'playlist_title': 'Some Playlist Title',
             'ext': self.extension,
             'resolution': self.source_resolution if self.source_resolution else '',
@@ -436,25 +474,14 @@ class Source(models.Model):
         indexer = self.INDEXERS.get(self.source_type, None)
         if not callable(indexer):
             raise Exception(f'Source type f"{self.source_type}" has no indexer')
-        response = indexer(self.url)
+        response = indexer(self.index_url)
+        if not isinstance(response, dict):
+            return []
+        entries = response.get('entries', [])
 
-        # Account for nested playlists, such as a channel of playlists of playlists
-        def _recurse_playlists(playlist):
-            videos = []
-            if not playlist:
-                return videos
-            entries = playlist.get('entries', [])
-            for entry in entries:
-                if not entry:
-                    continue
-                subentries = entry.get('entries', [])
-                if subentries:
-                    videos = videos + _recurse_playlists(entry)
-                else:
-                    videos.append(entry)
-            return videos
-
-        return _recurse_playlists(response)
+        if settings.MAX_ENTRIES_PROCESSING:
+            entries = entries[:settings.MAX_ENTRIES_PROCESSING]
+        return entries
 
 
 def get_media_thumb_path(instance, filename):
@@ -480,6 +507,12 @@ class Media(models.Model):
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/watch?v={key}',
         Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}',
     }
+    # Callback functions to get a list of media from the source
+    INDEXERS = {
+        Source.SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
+        Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
+        Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
+    }
     # Maps standardised names to names used in source metdata
     METADATA_FIELDS = {
         'upload_date': {
@@ -542,11 +575,6 @@ class Media(models.Model):
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count',
         },
-        'playlist_index': {
-            Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_index',
-            Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_index',
-            Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'playlist_index',
-        },
         'playlist_title': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title',
@@ -557,14 +585,18 @@ class Media(models.Model):
     STATE_SCHEDULED = 'scheduled'
     STATE_DOWNLOADING = 'downloading'
     STATE_DOWNLOADED = 'downloaded'
+    STATE_SKIPPED = 'skipped'
+    STATE_DISABLED_AT_SOURCE = 'source-disabled'
     STATE_ERROR = 'error'
     STATES = (STATE_UNKNOWN, STATE_SCHEDULED, STATE_DOWNLOADING, STATE_DOWNLOADED,
-              STATE_ERROR)
+              STATE_SKIPPED, STATE_DISABLED_AT_SOURCE, STATE_ERROR)
     STATE_ICONS = {
         STATE_UNKNOWN: '<i class="far fa-question-circle" title="Unknown download state"></i>',
         STATE_SCHEDULED: '<i class="far fa-clock" title="Scheduled to download"></i>',
         STATE_DOWNLOADING: '<i class="fas fa-download" title="Downloading now"></i>',
         STATE_DOWNLOADED: '<i class="far fa-check-circle" title="Downloaded"></i>',
+        STATE_SKIPPED: '<i class="fas fa-exclamation-circle" title="Skipped"></i>',
+        STATE_DISABLED_AT_SOURCE: '<i class="fas fa-stop-circle" title="Media downloading disabled at source"></i>',
         STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>',
     }
 
@@ -637,7 +669,7 @@ class Media(models.Model):
     media_file = models.FileField(
         _('media file'),
         upload_to=get_media_file_path,
-        max_length=200,
+        max_length=255,
         blank=True,
         null=True,
         storage=media_file_storage,
@@ -667,7 +699,7 @@ class Media(models.Model):
         max_length=30,
         blank=True,
         null=True,
-        help_text=_('Audio codec of the downloaded media')
+        help_text=_('Video format (resolution) of the downloaded media')
     )
     downloaded_height = models.PositiveIntegerField(
         _('downloaded height'),
@@ -794,6 +826,23 @@ class Media(models.Model):
         hdr = ''
         # If the download has completed use existing values
         if self.downloaded:
+            # Check if there's any stored meta data at all
+            if (not self.downloaded_video_codec and \
+                not self.downloaded_audio_codec):
+                # Marked as downloaded but no metadata, imported?
+                return {
+                    'resolution': resolution,
+                    'height': height,
+                    'width': width,
+                    'vcodec': vcodec,
+                    'acodec': acodec,
+                    'fps': fps,
+                    'hdr': hdr,
+                    'format': tuple(fmt),
+                }
+            if self.downloaded_format:
+                resolution = self.downloaded_format.lower()
+            elif self.downloaded_height:
                 resolution = f'{self.downloaded_height}p'
             if self.downloaded_format != 'audio':
                 vcodec = self.downloaded_video_codec.lower()
@@ -821,7 +870,7 @@ class Media(models.Model):
         # Otherwise, calculate from matched format codes
         vformat = None
         aformat = None
-        if '+' in format_str:
+        if format_str and '+' in format_str:
             # Seperate audio and video streams
             vformat_code, aformat_code = format_str.split('+')
             vformat = self.get_format_by_code(vformat_code)
@@ -830,7 +879,7 @@ class Media(models.Model):
             # Combined stream or audio only
             cformat = self.get_format_by_code(format_str)
             aformat = cformat
-            if cformat['vcodec']:
+            if cformat and cformat['vcodec']:
                 # Combined
                 vformat = cformat
         if vformat:
@@ -838,6 +887,7 @@ class Media(models.Model):
             fmt.append(resolution)
             vcodec = vformat['vcodec'].lower()
             fmt.append(vcodec)
+        if aformat:
             acodec = aformat['acodec'].lower()
             fmt.append(acodec)
         if vformat:
@@ -887,10 +937,9 @@ class Media(models.Model):
             'source': self.source.slugname,
             'source_full': self.source.name,
             'title': self.slugtitle,
-            'title_full': self.title,
+            'title_full': clean_filename(self.title),
             'key': self.key,
             'format': '-'.join(display_format['format']),
-            'playlist_index': self.playlist_index,
             'playlist_title': self.playlist_title,
             'ext': self.source.extension,
             'resolution': display_format['resolution'],
@@ -902,10 +951,17 @@ class Media(models.Model):
             'hdr': display_format['hdr'],
         }
 
+    @property
+    def has_metadata(self):
+        return self.metadata is not None
+
     @property
     def loaded_metadata(self):
         try:
-            return json.loads(self.metadata)
+            data = json.loads(self.metadata)
+            if not isinstance(data, dict):
+                return {}
+            return data
         except Exception as e:
             return {}
 
@@ -942,7 +998,10 @@ class Media(models.Model):
     @property
     def upload_date(self):
         field = self.get_metadata_field('upload_date')
+        try:
             upload_date_str = self.loaded_metadata.get(field, '').strip()
+        except (AttributeError, ValueError) as e:
+            return None
         try:
             return datetime.strptime(upload_date_str, '%Y%m%d')
         except (AttributeError, ValueError) as e:
@@ -951,7 +1010,12 @@ class Media(models.Model):
     @property
     def duration(self):
         field = self.get_metadata_field('duration')
-        return int(self.loaded_metadata.get(field, 0))
+        duration = self.loaded_metadata.get(field, 0)
+        try:
+            duration = int(duration)
+        except ValueError:
+            duration = 0
+        return duration
 
     @property
     def duration_formatted(self):
@@ -974,8 +1038,12 @@ class Media(models.Model):
     def votes(self):
         field = self.get_metadata_field('upvotes')
         upvotes = self.loaded_metadata.get(field, 0)
+        if not isinstance(upvotes, int):
+            upvotes = 0
         field = self.get_metadata_field('downvotes')
         downvotes = self.loaded_metadata.get(field, 0)
+        if not isinstance(downvotes, int):
+            downvotes = 0
         return upvotes + downvotes
 
     @property
@@ -993,11 +1061,6 @@ class Media(models.Model):
         field = self.get_metadata_field('formats')
         return self.loaded_metadata.get(field, [])
 
-    @property
-    def playlist_index(self):
-        field = self.get_metadata_field('playlist_index')
-        return self.loaded_metadata.get(field, 0)
-
     @property
     def playlist_title(self):
         field = self.get_metadata_field('playlist_title')
@@ -1005,13 +1068,16 @@ class Media(models.Model):
 
     @property
     def filename(self):
-        # Otherwise, create a suitable filename from the source media_format
+        # Create a suitable filename from the source media_format
         media_format = str(self.source.media_format)
         media_details = self.format_dict
         return media_format.format(**media_details)
 
     @property
     def thumbname(self):
+        if self.downloaded and self.media_file:
+            filename = os.path.basename(self.media_file.path)
+        else:
             filename = self.filename
         prefix, ext = os.path.splitext(filename)
         return f'{prefix}.jpg'
@@ -1022,6 +1088,9 @@ class Media(models.Model):
 
     @property
     def nfoname(self):
+        if self.downloaded and self.media_file:
+            filename = os.path.basename(self.media_file.path)
+        else:
             filename = self.filename
         prefix, ext = os.path.splitext(filename)
         return f'{prefix}.nfo'
@@ -1030,6 +1099,19 @@ class Media(models.Model):
     def nfopath(self):
         return self.source.directory_path / self.nfoname
 
+    @property
+    def jsonname(self):
+        if self.downloaded and self.media_file:
+            filename = os.path.basename(self.media_file.path)
+        else:
+            filename = self.filename
+        prefix, ext = os.path.splitext(filename)
+        return f'{prefix}.info.json'
+    
+    @property
+    def jsonpath(self):
+        return self.source.directory_path / self.jsonname
+
     @property
     def directory_path(self):
         # Otherwise, create a suitable filename from the source media_format
@@ -1161,6 +1243,10 @@ class Media(models.Model):
                 return self.STATE_ERROR
             else:
                 return self.STATE_SCHEDULED
+        if self.skip:
+            return self.STATE_SKIPPED
+        if not self.source.download_media:
+            return self.STATE_DISABLED_AT_SOURCE
         return self.STATE_UNKNOWN
 
     def get_download_state_icon(self, task=None):
@@ -1174,10 +1260,20 @@ class Media(models.Model):
                                     f'no valid format available')
         # Download the media with youtube-dl
         download_youtube_media(self.url, format_str, self.source.extension,
-                               str(self.filepath))
+                               str(self.filepath), self.source.write_json)
         # Return the download paramaters
         return format_str, self.source.extension
 
+    def index_metadata(self):
+        '''
+            Index the media metadata returning a dict of info.
+        '''
+        indexer = self.INDEXERS.get(self.source.source_type, None)
+        if not callable(indexer):
+            raise Exception(f'Media with source type f"{self.source.source_type}" '
+                            f'has no indexer')
+        return indexer(self.url)
+
 
 class MediaServer(models.Model):
     '''

tubesync/sync/signals.py

@@ -8,8 +8,9 @@ from background_task.models import Task
 from common.logger import log
 from .models import Source, Media, MediaServer
 from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task,
-                    download_media_thumbnail, map_task_to_instance,
-                    check_source_directory_exists, download_media, rescan_media_server)
+                    download_media_thumbnail, download_media_metadata,
+                    map_task_to_instance, check_source_directory_exists,
+                    download_media, rescan_media_server)
 from .utils import delete_file
 
 
@@ -46,6 +47,7 @@ def source_post_save(sender, instance, created, **kwargs):
             priority=0,
             verbose_name=verbose_name.format(instance.name)
         )
+        if instance.index_schedule > 0:
             delete_task_by_source('sync.tasks.index_source_task', instance.pk)
             log.info(f'Scheduling media indexing for source: {instance.name}')
             verbose_name = _('Index media from source "{}"')
@@ -91,18 +93,69 @@ def task_task_failed(sender, task_id, completed_task, **kwargs):
 
 @receiver(post_save, sender=Media)
 def media_post_save(sender, instance, created, **kwargs):
-    # Triggered after media is saved, Recalculate the "can_download" flag, this may
+    # Triggered after media is saved
+    cap_changed = False
+    can_download_changed = False
+    # Reset the skip flag if the download cap has changed if the media has not
+    # already been downloaded
+    if not instance.downloaded:
+        max_cap_age = instance.source.download_cap_date
+        published = instance.published
+        if not published:
+            if not instance.skip:
+                log.warn(f'Media: {instance.source} / {instance} has no published date '
+                         f'set, marking to be skipped')
+                instance.skip = True
+                cap_changed = True
+            else:
+                log.debug(f'Media: {instance.source} / {instance} has no published date '
+                          f'set but is already marked to be skipped')
+        else:
+            if max_cap_age:
+                if published > max_cap_age and instance.skip:
+                    # Media was published after the cap date but is set to be skipped
+                    log.info(f'Media: {instance.source} / {instance} has a valid '
+                            f'publishing date, marking to be unskipped')
+                    instance.skip = False
+                    cap_changed = True
+                elif published <= max_cap_age and not instance.skip:
+                    log.info(f'Media: {instance.source} / {instance} is too old for '
+                            f'the download cap date, marking to be skipped')
+                    instance.skip = True
+                    cap_changed = True
+            else:
+                if instance.skip:
+                    # Media marked to be skipped but source download cap removed
+                    log.info(f'Media: {instance.source} / {instance} has a valid '
+                            f'publishing date, marking to be unskipped')
+                    instance.skip = False
+                    cap_changed = True
+    # Recalculate the "can_download" flag, this may
     # need to change if the source specifications have been changed
-    post_save.disconnect(media_post_save, sender=Media)
+    if instance.metadata:
         if instance.get_format_str():
             if not instance.can_download:
                 instance.can_download = True
-            instance.save()
+                can_download_changed = True
         else:
             if instance.can_download:
                 instance.can_download = False
+                can_download_changed = True
+    # Save the instance if any changes were required
+    if cap_changed or can_download_changed:
+        post_save.disconnect(media_post_save, sender=Media)
         instance.save()
         post_save.connect(media_post_save, sender=Media)
+    # If the media is missing metadata schedule it to be downloaded
+    if not instance.metadata:
+        log.info(f'Scheduling task to download metadata for: {instance.url}')
+        verbose_name = _('Downloading metadata for "{}"')
+        download_media_metadata(
+            str(instance.pk),
+            priority=10,
+            verbose_name=verbose_name.format(instance.pk),
+            remove_existing_tasks=True
+        )
     # If the media is missing a thumbnail schedule it to be downloaded
     if not instance.thumb_file_exists:
         instance.thumb = None
@@ -124,7 +177,8 @@ def media_post_save(sender, instance, created, **kwargs):
     if not instance.media_file_exists:
         instance.downloaded = False
         instance.media_file = None
-    if not instance.downloaded and instance.can_download and not instance.skip:
+    if (not instance.downloaded and instance.can_download and not instance.skip
+        and instance.source.download_media):
         delete_task_by_media('sync.tasks.download_media', (str(instance.pk),))
         verbose_name = _('Downloading media for "{}"')
         download_media(

tubesync/sync/tasks.py

@@ -10,7 +10,7 @@ import math
 import uuid
 from io import BytesIO
 from hashlib import sha1
-from datetime import timedelta
+from datetime import timedelta, datetime
 from shutil import copyfile
 from PIL import Image
 from django.conf import settings
@@ -22,6 +22,7 @@ from background_task import background
 from background_task.models import Task, CompletedTask
 from common.logger import log
 from common.errors import NoMediaException, DownloadFailedException
+from common.utils import json_serial
 from .models import Source, Media, MediaServer
 from .utils import (get_remote_image, resize_image_to_height, delete_file,
                     write_text_file)
@@ -175,34 +176,10 @@ def index_source_task(source_id):
             # Video has no unique key (ID), it can't be indexed
             continue
         try:
-            media = Media.objects.get(key=key)
+            media = Media.objects.get(key=key, source=source)
         except Media.DoesNotExist:
             media = Media(key=key)
         media.source = source
-        media.metadata = json.dumps(video)
-        upload_date = media.upload_date
-        # Media must have a valid upload date
-        if upload_date:
-            media.published = timezone.make_aware(upload_date)
-        else:
-            log.error(f'Media has no upload date, skipping: {source} / {media}')
-            continue
-        # If the source has a download cap date check the upload date is allowed
-        max_cap_age = source.download_cap_date
-        if max_cap_age:
-            if media.published < max_cap_age:
-                # Media was published after the cap date, skip it
-                log.warn(f'Media: {source} / {media} is older than cap age '
-                         f'{max_cap_age}, skipping')
-                continue
-        # If the source has a cut-off check the upload date is within the allowed delta
-        if source.delete_old_media and source.days_to_keep > 0:
-            delta = timezone.now() - timedelta(days=source.days_to_keep)
-            if media.published < delta:
-                # Media was published after the cutoff date, skip it
-                log.warn(f'Media: {source} / {media} is older than '
-                         f'{source.days_to_keep} days, skipping')
-                continue
         try:
             media.save()
             log.info(f'Indexed media: {source} / {media}')
@@ -234,6 +211,61 @@ def check_source_directory_exists(source_id):
         source.make_directory()
 
 
+@background(schedule=0)
+def download_media_metadata(media_id):
+    '''
+        Downloads the metadata for a media item.
+    '''
+    try:
+        media = Media.objects.get(pk=media_id)
+    except Media.DoesNotExist:
+        # Task triggered but the media no longer exists, do nothing
+        log.error(f'Task download_media_metadata(pk={media_id}) called but no '
+                  f'media exists with ID: {media_id}')
+        return
+    source = media.source
+    metadata = media.index_metadata()
+    media.metadata = json.dumps(metadata, default=json_serial)
+    upload_date = media.upload_date
+    # Media must have a valid upload date
+    if upload_date:
+        media.published = timezone.make_aware(upload_date)
+    else:
+        log.error(f'Media has no upload date, skipping: {source} / {media}')
+        media.skip = True
+    # If the source has a download cap date check the upload date is allowed
+    max_cap_age = source.download_cap_date
+    if media.published and max_cap_age:
+        if media.published < max_cap_age:
+            # Media was published after the cap date, skip it
+            log.warn(f'Media: {source} / {media} is older than cap age '
+                     f'{max_cap_age}, skipping')
+            media.skip = True
+    # If the source has a cut-off check the upload date is within the allowed delta
+    if source.delete_old_media and source.days_to_keep > 0:
+        if not isinstance(media.published, datetime):
+            # Media has no known published date or incomplete metadata
+            log.warn(f'Media: {source} / {media} has no published date, skipping')
+            media.skip = True
+        else:
+            delta = timezone.now() - timedelta(days=source.days_to_keep)
+            if media.published < delta:
+                # Media was published after the cutoff date, skip it
+                log.warn(f'Media: {source} / {media} is older than '
+                         f'{source.days_to_keep} days, skipping')
+                media.skip = True
+    # Check we can download the media item
+    if not media.skip:
+        if media.get_format_str():
+            media.can_download = True
+        else:
+            media.can_download = False
+    # Save the media
+    media.save()
+    log.info(f'Saved {len(media.metadata)} bytes of metadata for: '
+             f'{source} / {media_id}')
+
+
 @background(schedule=0)
 def download_media_thumbnail(media_id, url):
     '''
@@ -279,8 +311,27 @@ def download_media(media_id):
         return
     if media.skip:
         # Media was toggled to be skipped after the task was scheduled
-        log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but it '
-                 f'is now marked to be skipped, not downloading')
+        log.warn(f'Download task triggered for  media: {media} (UUID: {media.pk}) but '
+                 f'it is now marked to be skipped, not downloading')
+        return
+    if media.downloaded and media.media_file:
+        # Media has been marked as downloaded before the download_media task was fired,
+        # skip it
+        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
+                 f'it has already been marked as downloaded, not downloading again')
+        return
+    if not media.source.download_media:
+        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
+                 f'the source {media.source} has since been marked to not download, '
+                 f'not downloading')
+        return
+    max_cap_age = media.source.download_cap_date
+    published = media.published
+    if max_cap_age and published:
+        if published <= max_cap_age:
+            log.warn(f'Download task triggered media: {media} (UUID: {media.pk}) but '
+                     f'the source has a download cap and the media is now too old, '
+                     f'not downloading')
             return
     filepath = media.filepath
     log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"')
@@ -290,7 +341,7 @@ def download_media(media_id):
         log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: '
                  f'"{filepath}"')
         # Link the media file to the object and update info about the download
-        media.media_file.name = str(filepath)
+        media.media_file.name = str(media.source.type_directory_path / media.filename)
         media.downloaded = True
         media.download_date = timezone.now()
         media.downloaded_filesize = os.path.getsize(filepath)
@@ -315,7 +366,7 @@ def download_media(media_id):
             media.downloaded_audio_codec = cformat['acodec']
             if cformat['vcodec']:
                 # Combined
-                media.downloaded_format = vformat['format']
+                media.downloaded_format = cformat['format']
                 media.downloaded_height = cformat['height']
                 media.downloaded_width = cformat['width']
                 media.downloaded_video_codec = cformat['vcodec']

tubesync/sync/templates/sync/_mediaformatvars.html

@@ -25,12 +25,12 @@
     </tr>
     <tr>
       <td>{mm}</td>
-      <td>Media publish year in MM</td>
+      <td>Media publish month in MM</td>
       <td>01</td>
     </tr>
     <tr>
       <td>{dd}</td>
-      <td>Media publish year in DD</td>
+      <td>Media publish day in DD</td>
       <td>31</td>
     </tr>
     <tr>
@@ -63,11 +63,6 @@
       <td>Media format string</td>
       <td>720p-avc1-mp4a</td>
     </tr>
-    <tr>
-      <td>{playlist_index}</td>
-      <td>Playlist index of media, if it's in a playlist</td>
-      <td>12</td>
-    </tr>
     <tr>
       <td>{playlist_title}</td>
       <td>Playlist title of media, if it's in a playlist</td>

tubesync/sync/templates/sync/dashboard.html

@@ -101,7 +101,7 @@
 </div>
 <div class="row">
   <div class="col s12">
-    <h2 class="truncate">Runtime infomation</h2>
+    <h2 class="truncate">Runtime information</h2>
   </div>
 </div>
 <div class="row">
@@ -123,6 +123,10 @@
         <td class="hide-on-small-only">Downloads directory</td>
         <td><span class="hide-on-med-and-up">Downloads directory<br></span><strong>{{ downloads_dir }}</strong></td>
       </tr>
+      <tr title="Database connection used by TubeSync">
+        <td class="hide-on-small-only">Database</td>
+        <td><span class="hide-on-med-and-up">Database<br></span><strong>{{ database_connection }}</strong></td>
+      </tr>
     </table>
   </div>
 </div>

tubesync/sync/templates/sync/media-item.html

@@ -9,6 +9,12 @@
     {% if media.title %}<h2 class="truncate"><strong>{{ media.title }}</strong></h2>{% endif %}
     <p class="truncate"><strong><a href="{{ media.url }}" target="_blank"><i class="fas fa-link"></i> {{ media.url }}</a></strong></p>
     <p class="truncate">Downloading to: <strong>{{ media.source.directory_path }}</strong></p>
+    {% if download_state == 'downloaded' %}
+    <video controls style="width: 100%">
+      <source src="{% url 'sync:media-content' pk=media.pk %}">
+    </video>
+    <p class="truncate"><a href="{% url 'sync:media-content' pk=media.pk %}" download="{{ media.filename }}">Download</a></p>
+    {% endif %}
   </div>
 </div>
 {% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
@@ -64,8 +70,14 @@
         <td class="hide-on-small-only">Fallback</td>
         <td><span class="hide-on-med-and-up">Fallback<br></span><strong>{{ media.source.get_fallback_display }}</strong></td>
       </tr>
+      {% if not media.source.download_media %}
+      <tr title="Is media marked to be downloaded at the source?">
+        <td class="hide-on-small-only">Source download?</td>
+        <td><span class="hide-on-med-and-up">Source download?<br></span><strong>{% if media.source.download_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
+      {% endif %}
       {% if media.skip %}
-      <tr title="Has the media been downloaded?">
+      <tr title="Is the media marked to be skipped?">
         <td class="hide-on-small-only">Skipping?</td>
         <td><span class="hide-on-med-and-up">Skipping?<br></span><strong>{% if media.skip %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
@@ -109,7 +121,7 @@
       {% else %}
       <tr title="Can the media be downloaded?">
         <td class="hide-on-small-only">Can download?</td>
-        <td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if youtube_dl_format %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+        <td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if media.can_download %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
       {% endif %}
       <tr title="The available media formats">

tubesync/sync/templates/sync/media.html

@@ -4,9 +4,23 @@
 
 {% block content %}
 <div class="row">
-  <div class="col s12">
+  <div class="col s12 m6">
     <h1 class="truncate">Media</h1>
   </div>
+  <div class="col s12 m3">
+    {% if show_skipped %}
+    <a href="{% url 'sync:media' %}{% if source %}?filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye-slash"></i> Hide skipped media</a>
+    {% else %}
+    <a href="{% url 'sync:media' %}?show_skipped=yes{% if source %}&filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye"></i> Show skipped media</a>
+    {% endif %}
+  </div>
+  <div class="col s12 m3">
+    {% if only_skipped %}
+    <a href="{% url 'sync:media' %}{% if source %}?filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye-slash"></i> Only skipped media</a>
+    {% else %}
+    <a href="{% url 'sync:media' %}?only_skipped=yes{% if source %}&filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye"></i> Only skipped media</a>
+    {% endif %}
+  </div>
 </div>
 {% include 'infobox.html' with message=message %}
 <div class="row no-margin-bottom">
@@ -24,8 +38,12 @@
             {% else %}
               {% if m.skip %}
               <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped</span>
+              {% elif not m.source.download_media %}
+              <span class="error-text"><i class="fas fa-times" title="Not downloading media for this source"></i> Disabled at source</span>
+              {% elif not m.has_metadata %}
+              <i class="far fa-clock" title="Waiting for metadata"></i> Fetching metadata
               {% elif m.can_download %}
-              <i class="far fa-clock" title="Waiting to download or downloading"></i> {{ m.published|date:'Y-m-d' }}
+              <i class="far fa-clock" title="Waiting to download or downloading"></i> Downloading
               {% else %}
               <span class="error-text"><i class="fas fa-exclamation-triangle" title="No matching formats to download"></i> No matching formats</span>
               {% endif %}
@@ -44,5 +62,5 @@
   </div>
   {% endfor %}
 </div>
-{% include 'pagination.html' with pagination=sources.paginator filter=source.pk %}
+{% include 'pagination.html' with pagination=sources.paginator filter=source.pk show_skipped=show_skipped %}
 {% endblock %}

tubesync/sync/templates/sync/source.html

@@ -61,6 +61,10 @@
         <td class="hide-on-small-only">Index schedule</td>
         <td><span class="hide-on-med-and-up">Index schedule<br></span><strong>{{ source.get_index_schedule_display }}</strong></td>
       </tr>
+      <tr title="Download media from this source">
+        <td class="hide-on-small-only">Download media?</td>
+        <td><span class="hide-on-med-and-up">Download media?<br></span><strong>{% if source.download_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
       <tr title="When then source was created locally in TubeSync">
         <td class="hide-on-small-only">Created</td>
         <td><span class="hide-on-med-and-up">Created<br></span><strong>{{ source.created|date:'Y-m-d H:i:s' }}</strong></td>
@@ -107,6 +111,10 @@
         <td class="hide-on-small-only">Write NFO?</td>
         <td><span class="hide-on-med-and-up">Write NFO?<br></span><strong>{% if source.write_nfo %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
+      <tr title="Should a JSON file be written with the media?">
+        <td class="hide-on-small-only">Write JSON?</td>
+        <td><span class="hide-on-med-and-up">Write JSON?<br></span><strong>{% if source.write_json %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
       {% if source.delete_old_media and source.days_to_keep > 0 %}
       <tr title="Days after which your media from this source will be locally deleted">
         <td class="hide-on-small-only">Delete old media</td>

tubesync/sync/templates/sync/sources.html

@@ -30,7 +30,7 @@
         {% if source.has_failed %}
         <span class="error-text"><i class="fas fa-exclamation-triangle"></i> <strong>Source has permanent failures</strong></span>
         {% else %}
-        <strong>{{ source.media_count }}</strong> media items{% if source.delete_old_media and source.days_to_keep > 0 %}, keep {{ source.days_to_keep }} days of media{% endif %}
+        <strong>{{ source.media_count }}</strong> media items, <strong>{{ source.downloaded_count }}</strong> downloaded{% if source.delete_old_media and source.days_to_keep > 0 %}, keeping {{ source.days_to_keep }} days of media{% endif %}
         {% endif %}
       </a>
     {% empty %}

tubesync/sync/testdata/metadata.json

@@ -9,7 +9,6 @@
     "average_rating": 1.2345,
     "dislike_count": 123,
     "like_count": 456,
-    "playlist_index": 789,
     "playlist_title": "test playlist",
     "uploader": "test uploader",
     "categories":[

tubesync/sync/tests.py

@@ -36,6 +36,9 @@ class FrontEndTestCase(TestCase):
         test_sources = {
             'youtube-channel': {
                 'valid': (
+                    'https://m.youtube.com/testchannel',
+                    'https://m.youtube.com/c/testchannel',
+                    'https://m.youtube.com/c/testchannel/videos',
                     'https://www.youtube.com/testchannel',
                     'https://www.youtube.com/c/testchannel',
                     'https://www.youtube.com/c/testchannel/videos',
@@ -47,6 +50,7 @@ class FrontEndTestCase(TestCase):
                 'invalid_domain': (
                     'https://www.test.com/c/testchannel',
                     'https://www.example.com/c/testchannel',
+                    'https://n.youtube.com/c/testchannel',
                 ),
                 'invalid_path': (
                     'https://www.youtube.com/test/invalid',
@@ -62,6 +66,8 @@ class FrontEndTestCase(TestCase):
             },
             'youtube-channel-id': {
                 'valid': (
+                    'https://m.youtube.com/channel/channelid',
+                    'https://m.youtube.com/channel/channelid/videos',
                     'https://www.youtube.com/channel/channelid',
                     'https://www.youtube.com/channel/channelid/videos',
                 ),
@@ -72,6 +78,7 @@ class FrontEndTestCase(TestCase):
                 'invalid_domain': (
                     'https://www.test.com/channel/channelid',
                     'https://www.example.com/channel/channelid',
+                    'https://n.youtube.com/channel/channelid',
                 ),
                 'invalid_path': (
                     'https://www.youtube.com/test/invalid',
@@ -83,6 +90,8 @@ class FrontEndTestCase(TestCase):
             },
             'youtube-playlist': {
                 'valid': (
+                    'https://m.youtube.com/playlist?list=testplaylist',
+                    'https://m.youtube.com/watch?v=testvideo&list=testplaylist',
                     'https://www.youtube.com/playlist?list=testplaylist',
                     'https://www.youtube.com/watch?v=testvideo&list=testplaylist',
                 ),
@@ -93,6 +102,7 @@ class FrontEndTestCase(TestCase):
                 'invalid_domain': (
                     'https://www.test.com/playlist?list=testplaylist',
                     'https://www.example.com/playlist?list=testplaylist',
+                    'https://n.youtube.com/playlist?list=testplaylist',
                 ),
                 'invalid_path': (
                     'https://www.youtube.com/notplaylist?list=testplaylist',
@@ -344,21 +354,25 @@ class FrontEndTestCase(TestCase):
                 }]
             } 
         '''
+        past_date = timezone.make_aware(datetime(year=2000, month=1, day=1))
         test_media1 = Media.objects.create(
             key='mediakey1',
             source=test_source,
+            published=past_date,
             metadata=test_minimal_metadata
         )
         test_media1_pk = str(test_media1.pk)
         test_media2 = Media.objects.create(
             key='mediakey2',
             source=test_source,
+            published=past_date,
             metadata=test_minimal_metadata
         )
         test_media2_pk = str(test_media2.pk)
         test_media3 = Media.objects.create(
             key='mediakey3',
             source=test_source,
+            published=past_date,
             metadata=test_minimal_metadata
         )
         test_media3_pk = str(test_media3.pk)
@@ -487,7 +501,7 @@ class FilepathTestCase(TestCase):
             metadata=metadata,
         )
 
-    def test_source_dirname(self):
+    def test_source_media_format(self):
         # Check media format validation is working
         # Empty
         self.source.media_format = ''
@@ -535,9 +549,6 @@ class FilepathTestCase(TestCase):
         self.source.media_format = 'test-{format}'
         self.assertEqual(self.source.get_example_media_format(),
                          'test-1080p-vp9-opus')
-        self.source.media_format = 'test-{playlist_index}'
-        self.assertEqual(self.source.get_example_media_format(),
-                         'test-1')
         self.source.media_format = 'test-{playlist_title}'
         self.assertEqual(self.source.get_example_media_format(),
                          'test-Some Playlist Title')
@@ -1161,14 +1172,14 @@ class FormatMatchingTestCase(TestCase):
             ('2160p', 'VP9', False, True): (True, '337'),              # Exact match, hdr
             ('2160p', 'VP9', True, False): (True, '315'),              # Exact match, 60fps
             ('2160p', 'VP9', True, True): (True, '337'),               # Exact match, 60fps+hdr
-            ('4320P', 'AVC1', False, False): (False, False),
-            ('4320P', 'AVC1', False, True): (False, False),
-            ('4320P', 'AVC1', True, False): (False, False),
-            ('4320P', 'AVC1', True, True): (False, False),
-            ('4320P', 'VP9', False, False): (False, False),
-            ('4320P', 'VP9', False, True): (False, False),
-            ('4320P', 'VP9', True, False): (True, '272'),              # Exact match, 60fps
-            ('4320P', 'VP9', True, True): (False, False),
+            ('4320p', 'AVC1', False, False): (False, False),
+            ('4320p', 'AVC1', False, True): (False, False),
+            ('4320p', 'AVC1', True, False): (False, False),
+            ('4320p', 'AVC1', True, True): (False, False),
+            ('4320p', 'VP9', False, False): (False, False),
+            ('4320p', 'VP9', False, True): (False, False),
+            ('4320p', 'VP9', True, False): (True, '272'),              # Exact match, 60fps
+            ('4320p', 'VP9', True, True): (False, False),
         }
         for params, expected in expected_matches.items():
             resolution, vcodec, prefer_60fps, prefer_hdr = params
@@ -1367,14 +1378,14 @@ class FormatMatchingTestCase(TestCase):
             ('2160p', 'VP9', False, True): (True, '337'),              # Exact match, hdr
             ('2160p', 'VP9', True, False): (True, '315'),              # Exact match, 60fps
             ('2160p', 'VP9', True, True): (True, '337'),               # Exact match, 60fps+hdr
-            ('4320P', 'AVC1', False, False): (False, '272'),           # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
-            ('4320P', 'AVC1', False, True): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
-            ('4320P', 'AVC1', True, False): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
-            ('4320P', 'AVC1', True, True): (False, '272'),             # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
-            ('4320P', 'VP9', False, False): (False, '272'),            # Fallback, 60fps (no other 8k streams)
-            ('4320P', 'VP9', False, True): (False, '272'),             # Fallback, 60fps (no other 8k streams)
-            ('4320P', 'VP9', True, False): (True, '272'),              # Exact match, 60fps
-            ('4320P', 'VP9', True, True): (False, '272'),              # Fallback, 60fps (no other 8k streams)
+            ('4320p', 'AVC1', False, False): (False, '272'),           # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'AVC1', False, True): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'AVC1', True, False): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'AVC1', True, True): (False, '272'),             # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'VP9', False, False): (False, '272'),            # Fallback, 60fps (no other 8k streams)
+            ('4320p', 'VP9', False, True): (False, '272'),             # Fallback, 60fps (no other 8k streams)
+            ('4320p', 'VP9', True, False): (True, '272'),              # Exact match, 60fps
+            ('4320p', 'VP9', True, True): (False, '272'),              # Fallback, 60fps (no other 8k streams)
         }
         for params, expected in expected_matches.items():
             resolution, vcodec, prefer_60fps, prefer_hdr = params

tubesync/sync/urls.py

@@ -2,7 +2,7 @@ from django.urls import path
 from .views import (DashboardView, SourcesView, ValidateSourceView, AddSourceView,
                     SourceView, UpdateSourceView, DeleteSourceView, MediaView,
                     MediaThumbView, MediaItemView, MediaRedownloadView, MediaSkipView,
-                    MediaEnableView, TasksView, CompletedTasksView, ResetTasks,
+                    MediaEnableView, MediaContent, TasksView, CompletedTasksView, ResetTasks,
                     MediaServersView, AddMediaServerView, MediaServerView,
                     DeleteMediaServerView, UpdateMediaServerView)
 
@@ -70,6 +70,10 @@ urlpatterns = [
          MediaEnableView.as_view(),
          name='enable-media'),
 
+    path('media-content/<uuid:pk>',
+         MediaContent.as_view(),
+         name='media-content'),
+
     # Task URLs
 
     path('tasks',

tubesync/sync/utils.py

@@ -14,9 +14,9 @@ def validate_url(url, validator):
         Validate a URL against a dict of validation requirements. Returns an extracted
         part of the URL if the URL is valid, if invalid raises a ValidationError.
     '''
-    valid_scheme, valid_netloc, valid_path, invalid_paths, valid_query, \
+    valid_scheme, valid_netlocs, valid_path, invalid_paths, valid_query, \
         extract_parts = (
-            validator['scheme'], validator['domain'], validator['path_regex'],
+            validator['scheme'], validator['domains'], validator['path_regex'],
             validator['path_must_not_match'], validator['qs_args'],
             validator['extract_key']
     )
@@ -25,8 +25,8 @@ def validate_url(url, validator):
     if url_scheme != valid_scheme:
         raise ValidationError(f'invalid scheme "{url_scheme}" must be "{valid_scheme}"')
     url_netloc = str(url_parts.netloc).strip().lower()
-    if url_netloc != valid_netloc:
-        raise ValidationError(f'invalid domain "{url_netloc}" must be "{valid_netloc}"')
+    if url_netloc not in valid_netlocs:
+        raise ValidationError(f'invalid domain "{url_netloc}" must be one of "{valid_netlocs}"')
     url_path = str(url_parts.path).strip()
     matches = re.findall(valid_path, url_path)
     if not matches:

tubesync/sync/views.py

@@ -10,7 +10,7 @@ from django.views.generic.detail import SingleObjectMixin
 from django.http import HttpResponse
 from django.urls import reverse_lazy
 from django.db import IntegrityError
-from django.db.models import Q, Count, Sum
+from django.db.models import Q, Count, Sum, When, Case
 from django.forms import ValidationError
 from django.utils.text import slugify
 from django.utils import timezone
@@ -78,6 +78,7 @@ class DashboardView(TemplateView):
         # Config and download locations
         data['config_dir'] = str(settings.CONFIG_BASE_DIR)
         data['downloads_dir'] = str(settings.DOWNLOAD_ROOT)
+        data['database_connection'] = settings.DATABASE_CONNECTION_STR
         return data
 
 
@@ -104,7 +105,10 @@ class SourcesView(ListView):
 
     def get_queryset(self):
         all_sources = Source.objects.all().order_by('name')
-        return all_sources.annotate(media_count=Count('media_source'))
+        return all_sources.annotate(
+            media_count=Count('media_source'),
+            downloaded_count=Count(Case(When(media_source__downloaded=True, then=1)))
+        )
 
     def get_context_data(self, *args, **kwargs):
         data = super().get_context_data(*args, **kwargs)
@@ -167,7 +171,7 @@ class ValidateSourceView(FormView):
     validation_urls = {
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL: {
             'scheme': 'https',
-            'domain': 'www.youtube.com',
+            'domains': ('m.youtube.com', 'www.youtube.com'),
             'path_regex': '^\/(c\/)?([^\/]+)(\/videos)?$',
             'path_must_not_match': ('/playlist', '/c/playlist'),
             'qs_args': [],
@@ -176,7 +180,7 @@ class ValidateSourceView(FormView):
         },
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: {
             'scheme': 'https',
-            'domain': 'www.youtube.com',
+            'domains': ('m.youtube.com', 'www.youtube.com'),
             'path_regex': '^\/channel\/([^\/]+)(\/videos)?$',
             'path_must_not_match': ('/playlist', '/c/playlist'),
             'qs_args': [],
@@ -185,7 +189,7 @@ class ValidateSourceView(FormView):
         },
         Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: {
             'scheme': 'https',
-            'domain': 'www.youtube.com',
+            'domains': ('m.youtube.com', 'www.youtube.com'),
             'path_regex': '^\/(playlist|watch)$',
             'path_must_not_match': (),
             'qs_args': ('list',),
@@ -265,23 +269,40 @@ class ValidateSourceView(FormView):
         return append_uri_params(url, fields)
 
 
-class AddSourceView(CreateView):
+class EditSourceMixin:
+    model = Source
+    fields = ('source_type', 'key', 'name', 'directory', 'media_format',
+              'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
+              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
+              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 'write_json')
+    errors = {
+        'invalid_media_format': _('Invalid media format, the media format contains '
+                                  'errors or is empty. Check the table at the end of '
+                                  'this page for valid media name variables'),
+    }
+
+    def form_valid(self, form):
+        # Perform extra validation to make sure the media_format is valid
+        obj = form.save(commit=False)
+        source_type = form.cleaned_data['media_format']
+        example_media_file = obj.get_example_media_format()
+        if example_media_file == '':
+            form.add_error(
+                'media_format',
+                ValidationError(self.errors['invalid_media_format'])
+            )
+        if form.errors:
+            return super().form_invalid(form)
+        return super().form_valid(form)
+
+
+class AddSourceView(EditSourceMixin, CreateView):
     '''
         Adds a new source, optionally takes some initial data querystring values to
         prepopulate some of the more unclear values.
     '''
 
     template_name = 'sync/source-add.html'
-    model = Source
-    fields = ('source_type', 'key', 'name', 'directory', 'media_format',
-              'index_schedule', 'download_cap', 'delete_old_media', 'days_to_keep',
-              'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
-              'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
-    errors = {
-        'invalid_media_format': _('Invalid media format, the media format contains '
-                                  'errors or is empty. Check the table at the end of '
-                                  'this page for valid media name variables'),
-    }
 
     def __init__(self, *args, **kwargs):
         self.prepopulated_data = {}
@@ -308,20 +329,6 @@ class AddSourceView(CreateView):
             initial[k] = v
         return initial
 
-    def form_valid(self, form):
-        # Perform extra validation to make sure the media_format is valid
-        obj = form.save(commit=False)
-        source_type = form.cleaned_data['media_format']
-        example_media_file = obj.get_example_media_format()
-        if example_media_file == '':
-            form.add_error(
-                'media_format',
-                ValidationError(self.errors['invalid_media_format'])
-            )
-        if form.errors:
-            return super().form_invalid(form)
-        return super().form_valid(form)
-
     def get_success_url(self):
         url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
         return append_uri_params(url, {'message': 'source-created'})
@@ -360,33 +367,9 @@ class SourceView(DetailView):
         return data
 
 
-class UpdateSourceView(UpdateView):
+class UpdateSourceView(EditSourceMixin, UpdateView):
 
     template_name = 'sync/source-update.html'
-    model = Source
-    fields = ('source_type', 'key', 'name', 'directory', 'media_format',
-              'index_schedule', 'download_cap', 'delete_old_media', 'days_to_keep',
-              'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
-              'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
-    errors = {
-        'invalid_media_format': _('Invalid media format, the media format contains '
-                                  'errors or is empty. Check the table at the end of '
-                                  'this page for valid media name variables'),
-    }
-
-    def form_valid(self, form):
-        # Perform extra validation to make sure the media_format is valid
-        obj = form.save(commit=False)
-        source_type = form.cleaned_data['media_format']
-        example_media_file = obj.get_example_media_format()
-        if example_media_file == '':
-            form.add_error(
-                'media_format',
-                ValidationError(self.errors['invalid_media_format'])
-            )
-        if form.errors:
-            return super().form_invalid(form)
-        return super().form_valid(form)
 
     def get_success_url(self):
         url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
@@ -412,11 +395,13 @@ class DeleteSourceView(DeleteView, FormMixin):
             for media in Media.objects.filter(source=source):
                 if media.media_file:
                     # Delete the media file
-                    delete_file(media.media_file.name)
+                    delete_file(media.media_file.path)
                     # Delete thumbnail copy if it exists
                     delete_file(media.thumbpath)
                     # Delete NFO file if it exists
                     delete_file(media.nfopath)
+                    # Delete JSON file if it exists
+                    delete_file(media.jsonpath)
         return super().post(request, *args, **kwargs)
 
     def get_success_url(self):
@@ -438,6 +423,8 @@ class MediaView(ListView):
 
     def __init__(self, *args, **kwargs):
         self.filter_source = None
+        self.show_skipped = False
+        self.only_skipped = False
         super().__init__(*args, **kwargs)
 
     def dispatch(self, request, *args, **kwargs):
@@ -447,13 +434,30 @@ class MediaView(ListView):
                 self.filter_source = Source.objects.get(pk=filter_by)
             except Source.DoesNotExist:
                 self.filter_source = None
+        show_skipped = request.GET.get('show_skipped', '').strip()
+        if show_skipped == 'yes':
+            self.show_skipped = True
+        if not self.show_skipped:
+            only_skipped = request.GET.get('only_skipped', '').strip()
+            if only_skipped == 'yes':
+                self.only_skipped = True
         return super().dispatch(request, *args, **kwargs)
 
     def get_queryset(self):
         if self.filter_source:
+            if self.show_skipped:
                 q = Media.objects.filter(source=self.filter_source)
+            elif self.only_skipped:
+                q = Media.objects.filter(source=self.filter_source, skip=True)
             else:
+                q = Media.objects.filter(source=self.filter_source, skip=False)
+        else:
+            if self.show_skipped:
                 q = Media.objects.all()
+            elif self.only_skipped:
+                q = Media.objects.filter(skip=True)
+            else:
+                q = Media.objects.filter(skip=False)
         return q.order_by('-published', '-created')
 
     def get_context_data(self, *args, **kwargs):
@@ -464,6 +468,8 @@ class MediaView(ListView):
             message = str(self.messages.get('filter', ''))
             data['message'] = message.format(name=self.filter_source.name)
             data['source'] = self.filter_source
+        data['show_skipped'] = self.show_skipped
+        data['only_skipped'] = self.only_skipped
         return data
 
 
@@ -613,6 +619,7 @@ class MediaSkipView(FormView, SingleObjectMixin):
             # If the media has an associated NFO file with it, also delete it
             delete_file(self.object.nfopath)
         # Reset all download data
+        self.object.metadata = None
         self.object.downloaded = False
         self.object.downloaded_audio_codec = None
         self.object.downloaded_video_codec = None
@@ -658,6 +665,25 @@ class MediaEnableView(FormView, SingleObjectMixin):
         return append_uri_params(url, {'message': 'enabled'})
 
 
+class MediaContent(DetailView):
+    '''
+        Redirect to nginx to download the file
+    '''
+    model = Media
+
+    def __init__(self, *args, **kwargs):
+        self.object = None
+        super().__init__(*args, **kwargs)
+
+    def dispatch(self, request, *args, **kwargs):
+        self.object = self.get_object()
+
+        headers = {
+            'X-Accel-Redirect': self.object.media_file.url,
+        }
+        return HttpResponse(headers=headers)
+
+
 class TasksView(ListView):
     '''
         A list of tasks queued to be completed. This is, for example, scraping for new
@@ -1011,7 +1037,7 @@ class UpdateMediaServerView(FormView, SingleObjectMixin):
 
     def get_context_data(self, *args, **kwargs):
         data = super().get_context_data(*args, **kwargs)
-        data['server_help'] = self.object.help_html
+        data['server_help'] = self.object.get_help_html
         return data
 
     def get_success_url(self):

tubesync/sync/youtube.py

@@ -8,7 +8,7 @@ import os
 from django.conf import settings
 from copy import copy
 from common.logger import log
-import youtube_dl
+import yt_dlp
 
 
 _youtubedl_cachedir = getattr(settings, 'YOUTUBE_DL_CACHEDIR', None)
@@ -19,36 +19,52 @@ if _youtubedl_cachedir:
 
 
 
-class YouTubeError(youtube_dl.utils.DownloadError):
+class YouTubeError(yt_dlp.utils.DownloadError):
     '''
         Generic wrapped error for all errors that could be raised by youtube-dl.
     '''
     pass
 
 
+def get_yt_opts():
+    opts = copy(_defaults)
+    cookie_file = settings.COOKIES_FILE
+    if cookie_file.is_file():
+        cookie_file_path = str(cookie_file.resolve())
+        log.info(f'[youtube-dl] using cookies.txt from: {cookie_file_path}')
+        opts.update({'cookiefile': cookie_file_path})
+    return opts
+
+
 def get_media_info(url):
     '''
         Extracts information from a YouTube URL and returns it as a dict. For a channel
         or playlist this returns a dict of all the videos on the channel or playlist
         as well as associated metadata.
     '''
-    opts = copy(_defaults)
+    opts = get_yt_opts()
     opts.update({
         'skip_download': True,
         'forcejson': True,
         'simulate': True,
-        'logger': log
+        'logger': log,
+        'extract_flat': True,
     })
     response = {}
-    with youtube_dl.YoutubeDL(opts) as y:
+    with yt_dlp.YoutubeDL(opts) as y:
         try:
             response = y.extract_info(url, download=False)
-        except youtube_dl.utils.DownloadError as e:
+        except yt_dlp.utils.DownloadError as e:
             raise YouTubeError(f'Failed to extract_info for "{url}": {e}') from e
+    if not response:
+        raise YouTubeError(f'Failed to extract_info for "{url}": No metadata was '
+                           f'returned by youtube-dl, check for error messages in the '
+                           f'logs above. This task will be retried later with an '
+                           f'exponential backoff.')
     return response
 
 
-def download_media(url, media_format, extension, output_file):
+def download_media(url, media_format, extension, output_file, info_json):
     '''
         Downloads a YouTube URL to a file on disk.
     '''
@@ -85,17 +101,18 @@ def download_media(url, media_format, extension, output_file):
             log.warn(f'[youtube-dl] unknown event: {str(event)}')
     hook.download_progress = 0
 
-    opts = copy(_defaults)
+    opts = get_yt_opts()
     opts.update({
         'format': media_format,
         'merge_output_format': extension,
         'outtmpl': output_file,
         'quiet': True,
         'progress_hooks': [hook],
+        'writeinfojson': info_json
     })
-    with youtube_dl.YoutubeDL(opts) as y:
+    with yt_dlp.YoutubeDL(opts) as y:
         try:
             return y.download([url])
-        except youtube_dl.utils.DownloadError as e:
+        except yt_dlp.utils.DownloadError as e:
             raise YouTubeError(f'Failed to download for "{url}": {e}') from e
     return False

tubesync/tubesync/celery.py

@@ -0,0 +1,19 @@
+import os
+from celery import Celery
+
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tubesync.settings')
+REDIS_CONNECTION = os.getenv('REDIS_CONNECTION', 'redis://localhost:6379/0')
+
+
+app = Celery('tubesync')
+app.config_from_object('django.conf:settings', namespace='CELERY')
+app.autodiscover_tasks()
+app.conf.broker_url = REDIS_CONNECTION
+app.conf.beat_schedule = {
+    '10-second-beat': {
+        'task': 'sync.tasks.housekeeping_task',
+        'schedule': 60.0,
+        'args': ()
+    },
+}

tubesync/tubesync/dbutils.py

@@ -0,0 +1,27 @@
+import importlib
+from django.conf import settings
+from django.db.backends.utils import CursorWrapper
+
+
+def patch_ensure_connection():
+    for name, config in settings.DATABASES.items():
+
+        # Don't patch for PostgreSQL, it doesn't need it and can cause issues
+        if config['ENGINE'] == 'django.db.backends.postgresql':
+            continue
+
+        module = importlib.import_module(config['ENGINE'] + '.base')
+
+        def ensure_connection(self):
+            if self.connection is not None:
+                try:
+                    with CursorWrapper(self.create_cursor(), self) as cursor:
+                        cursor.execute('SELECT 1;')
+                    return
+                except Exception:
+                    pass
+
+            with self.wrap_database_errors:
+                self.connect()
+
+        module.DatabaseWrapper.ensure_connection = ensure_connection

tubesync/tubesync/local_settings.py.container

@@ -1,34 +1,79 @@
 import os
 from pathlib import Path
+from urllib.parse import urljoin
+from common.logger import log
+from common.utils import parse_database_connection_string
 
 
 BASE_DIR = Path(__file__).resolve().parent.parent
 ROOT_DIR = Path('/')
 CONFIG_BASE_DIR = ROOT_DIR / 'config'
 DOWNLOADS_BASE_DIR = ROOT_DIR / 'downloads'
+DJANGO_URL_PREFIX = os.getenv('DJANGO_URL_PREFIX', None)
+STATIC_URL = str(os.getenv('DJANGO_STATIC_URL', '/static/'))
+if DJANGO_URL_PREFIX and STATIC_URL:
+    STATIC_URL = urljoin(DJANGO_URL_PREFIX, STATIC_URL[1:])
 
 
 # This is not ever meant to be a public web interface so this isn't too critical
 SECRET_KEY = str(os.getenv('DJANGO_SECRET_KEY', 'tubesync-django-secret'))
 
 
-ALLOWED_HOSTS_STR = str(os.getenv('TUBESYNC_HOSTS', '127.0.0.1,localhost'))
+ALLOWED_HOSTS_STR = str(os.getenv('TUBESYNC_HOSTS', '*'))
 ALLOWED_HOSTS = ALLOWED_HOSTS_STR.split(',')
 DEBUG = True if os.getenv('TUBESYNC_DEBUG', False) else False
-FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', None)
+FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX)
 
 
 TIME_ZONE = os.getenv('TZ', 'UTC')
 
 
-DATABASES = {
+database_dict = {}
+database_connection_env = os.getenv('DATABASE_CONNECTION', '')
+if database_connection_env:
+    database_dict = parse_database_connection_string(database_connection_env)
+
+
+if database_dict:
+    log.info(f'Using database connection: {database_dict["ENGINE"]}://'
+             f'{database_dict["USER"]}:[hidden]@{database_dict["HOST"]}:'
+             f'{database_dict["PORT"]}/{database_dict["NAME"]}')
+    DATABASES = {
+        'default': database_dict,
+    }
+    DATABASE_CONNECTION_STR = (f'{database_dict["DRIVER"]} at "{database_dict["HOST"]}:'
+                               f'{database_dict["PORT"]}" database '
+                               f'"{database_dict["NAME"]}"')
+else:
+    DATABASES = {
         'default': {
             'ENGINE': 'django.db.backends.sqlite3',
             'NAME': CONFIG_BASE_DIR / 'db.sqlite3',
         }
-}
+    }
+    DATABASE_CONNECTION_STR = f'sqlite at "{DATABASES["default"]["NAME"]}"'
+
+
+DEFAULT_THREADS = 1
+MAX_BACKGROUND_TASK_ASYNC_THREADS = 8
+BACKGROUND_TASK_ASYNC_THREADS = int(os.getenv('TUBESYNC_WORKERS', DEFAULT_THREADS))
+if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS:
+    BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS
 
 
 MEDIA_ROOT = CONFIG_BASE_DIR / 'media'
 DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR
 YOUTUBE_DL_CACHEDIR = CONFIG_BASE_DIR / 'cache'
+COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt'
+
+
+BASICAUTH_USERNAME = os.getenv('HTTP_USER', '').strip()
+BASICAUTH_PASSWORD = os.getenv('HTTP_PASS', '').strip()
+if BASICAUTH_USERNAME and BASICAUTH_PASSWORD:
+    BASICAUTH_DISABLE = False
+    BASICAUTH_USERS = {
+        BASICAUTH_USERNAME: BASICAUTH_PASSWORD,
+    }
+else:
+    BASICAUTH_DISABLE = True
+    BASICAUTH_USERS = {}

tubesync/tubesync/local_settings.py.example

@@ -16,6 +16,7 @@ DATABASES = {
         'NAME': CONFIG_BASE_DIR / 'db.sqlite3',
     }
 }
+DATABASE_CONNECTION_STR = f'sqlite at "{DATABASES["default"]["NAME"]}"'
 
 
 DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR / 'downloads'

tubesync/tubesync/settings.py

@@ -6,7 +6,7 @@ CONFIG_BASE_DIR = BASE_DIR
 DOWNLOADS_BASE_DIR = BASE_DIR
 
 
-VERSION = 0.6
+VERSION = '0.12.0'
 SECRET_KEY = ''
 DEBUG = False
 ALLOWED_HOSTS = []
@@ -37,6 +37,7 @@ MIDDLEWARE = [
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'whitenoise.middleware.WhiteNoiseMiddleware',
     'common.middleware.MaterializeDefaultFieldsMiddleware',
+    'common.middleware.BasicAuthMiddleware',
 ]
 
 
@@ -75,6 +76,9 @@ WSGI_APPLICATION = 'tubesync.wsgi.application'
 DATABASES = {}
 
 
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
+
 AUTH_PASSWORD_VALIDATORS = [
     {
         'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
@@ -114,20 +118,30 @@ Disallow: /
 '''.strip()
 
 
+X_FRAME_OPTIONS = 'SAMEORIGIN'
+
+
+BASICAUTH_DISABLE = True
+BASICAUTH_REALM = 'Authenticate to TubeSync'
+BASICAUTH_ALWAYS_ALLOW_URIS = ('/healthcheck',)
+BASICAUTH_USERS = {}
+
+
 HEALTHCHECK_FIREWALL = True
 HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',)
 
 
-MAX_ATTEMPTS = 10                           # Number of times tasks will be retried
+MAX_ATTEMPTS = 15                           # Number of times tasks will be retried
 MAX_RUN_TIME = 1800                         # Maximum amount of time in seconds a task can run
-BACKGROUND_TASK_RUN_ASYNC = False           # Run tasks async in the background
+BACKGROUND_TASK_RUN_ASYNC = True            # Run tasks async in the background
 BACKGROUND_TASK_ASYNC_THREADS = 1           # Number of async tasks to run at once
+MAX_BACKGROUND_TASK_ASYNC_THREADS = 8       # For sanity reasons
 BACKGROUND_TASK_PRIORITY_ORDERING = 'ASC'   # Use 'niceness' task priority ordering
 COMPLETED_TASKS_DAYS_TO_KEEP = 7            # Number of days to keep completed tasks
-
+MAX_ENTRIES_PROCESSING = 0                  # Number of videos to process on source refresh (0 for no limit)
 
 SOURCES_PER_PAGE = 100
-MEDIA_PER_PAGE = 72
+MEDIA_PER_PAGE = 144
 TASKS_PER_PAGE = 100
 
 
@@ -135,7 +149,7 @@ MEDIA_THUMBNAIL_WIDTH = 430                 # Width in pixels to resize thumbnai
 MEDIA_THUMBNAIL_HEIGHT = 240                # Height in pixels to resize thumbnails to
 
 
-VIDEO_HEIGHT_CUTOFF = 360       # Smallest resolution in pixels permitted to download
+VIDEO_HEIGHT_CUTOFF = 240       # Smallest resolution in pixels permitted to download
 VIDEO_HEIGHT_IS_HD = 500        # Height in pixels to count as 'HD'
 
 
@@ -145,10 +159,12 @@ YOUTUBE_DEFAULTS = {
     'age_limit': 99,        # 'Age in years' to spoof
     'ignoreerrors': True,   # Skip on errors (such as unavailable videos in playlists)
     'cachedir': False,      # Disable on-disk caching
+    'addmetadata': True,    # Embed metadata during postprocessing where available
 }
+COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt'
 
 
-MEDIA_FORMATSTR_DEFAULT = '{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}'
+MEDIA_FORMATSTR_DEFAULT = '{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}'
 
 
 try:
@@ -157,3 +173,7 @@ except ImportError as e:
     import sys
     sys.stderr.write(f'Unable to import local_settings: {e}\n')
     sys.exit(1)
+
+
+from .dbutils import patch_ensure_connection
+patch_ensure_connection()

tubesync/tubesync/wsgi.py

@@ -1,6 +1,25 @@
 import os
+from urllib.parse import urljoin
 from django.core.wsgi import get_wsgi_application
 
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tubesync.settings')
-application = get_wsgi_application()
+DJANGO_URL_PREFIX = os.getenv('DJANGO_URL_PREFIX', None)
+_application = get_wsgi_application()
+
+
+def application(environ, start_response):
+    script_name = None
+    if DJANGO_URL_PREFIX:
+        if DJANGO_URL_PREFIX.endswith('/'):
+            script_name = DJANGO_URL_PREFIX
+        else:
+            raise Exception(f'DJANGO_URL_PREFIX must end with a /, '
+                            f'got: {DJANGO_URL_PREFIX}')
+    if script_name:
+        static_url = urljoin(script_name, 'static/')
+        environ['SCRIPT_NAME'] = script_name
+        path_info = environ['PATH_INFO']
+        if path_info.startswith(script_name) and not path_info.startswith(static_url):
+            environ['PATH_INFO'] = path_info[len(script_name) - 1:]
+    return _application(environ, start_response)