bump to 0.13.0

Subtitles

.github/workflows/ci.yaml

@@ -7,9 +7,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    branches:
-      - main
 
 jobs:
   test:
@@ -27,7 +24,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install pipenv
-          pipenv install --system
+          pipenv install --system --skip-lock
       - name: Set up Django environment
         run: cp tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py
       - name: Run Django tests
@@ -35,19 +32,24 @@ jobs:
   containerise:
     runs-on: ubuntu-latest
     steps:
-    - name: Set up QEMU
+      - name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v1
-    - name: Set up Docker Buildx
+      - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v1
-    - name: Log into GitHub Container Registry
+      - name: Log into GitHub Container Registry
-      run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
+        run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
-    - name: Build and push
+      - name: Lowercase github username for ghcr
-      uses: docker/build-push-action@v2
+        id: string
-      with:
+        uses: ASzc/change-string-case-action@v1
-        platforms: linux/amd64
+        with:
-        push: true
+          string: ${{ github.actor }}
-        tags: ghcr.io/meeb/${{ env.IMAGE_NAME }}:latest
+      - name: Build and push
-        cache-from: type=registry,ref=ghcr.io/meeb/${{ env.IMAGE_NAME }}:latest
+        uses: docker/build-push-action@v2
-        cache-to: type=inline
+        with:
-        build-args: |
+          platforms: linux/amd64,linux/arm64
-          IMAGE_NAME=${{ env.IMAGE_NAME }}
+          push: true
+          tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest
+          cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest
+          cache-to: type=inline
+          build-args: |
+            IMAGE_NAME=${{ env.IMAGE_NAME }}

.github/workflows/release.yaml

@@ -11,23 +11,28 @@ jobs:
   containerise:
     runs-on: ubuntu-latest
     steps:
-    - name: Set up QEMU
+      - name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v1
-    - name: Get tag
+      - name: Get tag
-      id: tag
+        id: tag
-      uses: dawidd6/action-get-tag@v1
+        uses: dawidd6/action-get-tag@v1
-    - uses: docker/build-push-action@v2
+      - uses: docker/build-push-action@v2
-    - name: Set up Docker Buildx
+      - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v1
-    - name: Log into GitHub Container Registry
+      - name: Log into GitHub Container Registry
-      run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
+        run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
-    - name: Build and push
+      - name: Lowercase github username for ghcr
-      uses: docker/build-push-action@v2
+        id: string
-      with:
+        uses: ASzc/change-string-case-action@v1
-        platforms: linux/amd64
+        with:
-        push: true
+          string: ${{ github.actor }}
-        tags: ghcr.io/meeb/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
+      - name: Build and push
-        cache-from: type=registry,ref=ghcr.io/meeb/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
+        uses: docker/build-push-action@v2
-        cache-to: type=inline
+        with:
-        build-args: |
+          platforms: linux/amd64,linux/arm64
-          IMAGE_NAME=${{ env.IMAGE_NAME }}
+          push: true
+          tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
+          cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
+          cache-to: type=inline
+          build-args: |
+            IMAGE_NAME=${{ env.IMAGE_NAME }}

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -130,3 +131,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+Pipfile.lock
+.vscode/launch.json

Dockerfile

@@ -1,56 +1,77 @@
-FROM debian:buster-slim
+FROM debian:bookworm-slim
 
-ARG ARCH="amd64"
+ARG TARGETPLATFORM
-ARG S6_VERSION="2.2.0.3"
+ARG S6_VERSION="3.1.5.0"
-ARG FFMPEG_VERSION="4.3.2"
+ARG FFMPEG_DATE="autobuild-2023-09-24-14-11"
+ARG FFMPEG_VERSION="112171-g13a3e2a9b4"
 
 ENV DEBIAN_FRONTEND="noninteractive" \
-    HOME="/root" \
+  HOME="/root" \
-    LANGUAGE="en_US.UTF-8" \
+  LANGUAGE="en_US.UTF-8" \
-    LANG="en_US.UTF-8" \
+  LANG="en_US.UTF-8" \
-    LC_ALL="en_US.UTF-8" \
+  LC_ALL="en_US.UTF-8" \
-    TERM="xterm" \
+  TERM="xterm" \
-    S6_EXPECTED_SHA256="a7076cf205b331e9f8479bbb09d9df77dbb5cd8f7d12e9b74920902e0c16dd98" \
+  S6_CMD_WAIT_FOR_SERVICES_MAXTIME="0"
-    S6_DOWNLOAD="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${ARCH}.tar.gz" \
-    FFMPEG_EXPECTED_SHA256="34bffcd0b58695e3ee5eba2573b37f06cb5088050733ca96265815f58bd61d35" \
-    FFMPEG_DOWNLOAD="https://tubesync.sfo2.digitaloceanspaces.com/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static.tar.xz"
-
 
 # Install third party software
-RUN set -x && \
+RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
-    apt-get update && \
+  "linux/amd64")   echo "amd64"  ;; \
-    apt-get -y --no-install-recommends install locales && \
+  "linux/arm64")   echo "aarch64" ;; \
-    echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
+  *)               echo ""        ;; esac) && \
-    locale-gen en_US.UTF-8 && \
+  export S6_ARCH_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
-    # Install required distro packages
+  "linux/amd64")   echo "65d0d0f353d2ff9d0af202b268b4bf53a9948a5007650854855c729289085739" ;; \
-    apt-get -y --no-install-recommends install curl xz-utils ca-certificates binutils && \
+  "linux/arm64")   echo "3fbd14201473710a592b2189e81f00f3c8998e96d34f16bd2429c35d1bc36d00" ;; \
-    # Install s6
+  *)               echo ""        ;; esac) && \
-    curl -L ${S6_DOWNLOAD} --output /tmp/s6-overlay-${ARCH}.tar.gz && \
+  export S6_DOWNLOAD_ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
-    sha256sum /tmp/s6-overlay-${ARCH}.tar.gz && \
+  "linux/amd64")   echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-x86_64.tar.xz"   ;; \
-    echo "${S6_EXPECTED_SHA256}  /tmp/s6-overlay-${ARCH}.tar.gz" | sha256sum -c - && \
+  "linux/arm64")   echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-aarch64.tar.xz" ;; \
-    tar xzf /tmp/s6-overlay-${ARCH}.tar.gz -C / && \
+  *)               echo ""        ;; esac) && \
-    # Install ffmpeg
+  export FFMPEG_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
-    curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg-${ARCH}-static.tar.xz && \
+  "linux/amd64")   echo "71cd08ed38c33ff2625dcca68d05efda090bdae455625d3bb1e4be4a53bf7c11" ;; \
-    echo "${FFMPEG_EXPECTED_SHA256}  /tmp/ffmpeg-${ARCH}-static.tar.xz" | sha256sum -c - && \
+  "linux/arm64")   echo "b6765d97f20cecef0121559ee26a2f0dfbac6aef49c48c71eb703271cb3f527b" ;; \
-    xz --decompress /tmp/ffmpeg-${ARCH}-static.tar.xz && \
+  *)               echo ""        ;; esac) && \
-    tar -xvf /tmp/ffmpeg-${ARCH}-static.tar -C /tmp && \
+  export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \
-    install -v -s -g root -o root -m 0755 -s /tmp/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static/ffmpeg -t /usr/local/bin && \
+  "linux/amd64")   echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linux64-gpl.tar.xz"   ;; \
-    # Clean up
+  "linux/arm64")   echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linuxarm64-gpl.tar.xz" ;; \
-    rm -rf /tmp/s6-overlay-${ARCH}.tar.gz && \
+  *)               echo ""        ;; esac) && \
-    rm -rf /tmp/ffmpeg-${ARCH}-static.tar && \
+  export S6_NOARCH_EXPECTED_SHA256="fd80c231e8ae1a0667b7ae2078b9ad0e1269c4d117bf447a4506815a700dbff3" && \
-    rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static && \
+  export S6_DOWNLOAD_NOARCH="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-noarch.tar.xz" && \
-    apt-get -y autoremove --purge curl xz-utils binutils
+  echo "Building for arch: ${ARCH}|${ARCH44}, downloading S6 from: ${S6_DOWNLOAD}}, expecting S6 SHA256: ${S6_EXPECTED_SHA256}" && \
+  set -x && \
+  apt-get update && \
+  apt-get -y --no-install-recommends install locales && \
+  echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
+  locale-gen en_US.UTF-8 && \
+  # Install required distro packages
+  apt-get -y --no-install-recommends install curl ca-certificates binutils xz-utils && \
+  # Install s6
+  curl -L ${S6_DOWNLOAD_NOARCH} --output /tmp/s6-overlay-noarch.tar.xz && \
+  echo "${S6_NOARCH_EXPECTED_SHA256}  /tmp/s6-overlay-noarch.tar.xz" | sha256sum -c - && \
+  tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && \
+  curl -L ${S6_DOWNLOAD_ARCH} --output /tmp/s6-overlay-${ARCH}.tar.xz && \
+  echo "${S6_ARCH_EXPECTED_SHA256}  /tmp/s6-overlay-${ARCH}.tar.xz" | sha256sum -c - && \
+  tar -C / -Jxpf /tmp/s6-overlay-${ARCH}.tar.xz && \
+  # Install ffmpeg
+  echo "Building for arch: ${ARCH}|${ARCH44}, downloading FFMPEG from: ${FFMPEG_DOWNLOAD}, expecting FFMPEG SHA256: ${FFMPEG_EXPECTED_SHA256}" && \
+  curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg-${ARCH}.tar.xz && \
+  sha256sum /tmp/ffmpeg-${ARCH}.tar.xz && \
+  echo "${FFMPEG_EXPECTED_SHA256}  /tmp/ffmpeg-${ARCH}.tar.xz" | sha256sum -c - && \
+  tar -xf /tmp/ffmpeg-${ARCH}.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffmpeg" && \
+  tar -xf /tmp/ffmpeg-${ARCH}.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffprobe" && \
+  # Clean up
+  rm -rf /tmp/s6-overlay-${ARCH}.tar.gz && \
+  rm -rf /tmp/ffmpeg-${ARCH}.tar.xz && \
+  apt-get -y autoremove --purge curl binutils xz-utils
 
 # Copy app
 COPY tubesync /app
 COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings.py
 
-# Append container bundled software versions
+# Copy over pip.conf to use piwheels
-RUN echo "ffmpeg_version = '${FFMPEG_VERSION}-static'" >> /app/common/third_party_versions.py
+COPY pip.conf /etc/pip.conf
 
 # Add Pipfile
 COPY Pipfile /app/Pipfile
-COPY Pipfile.lock /app/Pipfile.lock
 
 # Switch workdir to the the app
 WORKDIR /app
@@ -61,29 +82,31 @@ RUN set -x && \
   # Install required distro packages
   apt-get -y install nginx-light && \
   apt-get -y --no-install-recommends install \
-    python3 \
+  python3 \
-    python3-setuptools \
+  python3-dev \
-    python3-pip \
+  python3-pip \
-    python3-dev \
+  python3-wheel \
-    gcc \
+  pipenv \
-    make \
+  gcc \
-    default-libmysqlclient-dev \
+  g++ \
-    libmariadb3 \
+  make \
-    postgresql-common \
+  pkgconf \
-    libpq-dev \
+  default-libmysqlclient-dev \
-    libpq5 \
+  libmariadb3 \
-    libjpeg62-turbo \
+  postgresql-common \
-    libwebp6 \
+  libpq-dev \
-    libjpeg-dev \
+  libpq5 \
-    zlib1g-dev \
+  libjpeg62-turbo \
-    libwebp-dev && \
+  libwebp7 \
-  # Install pipenv
+  libjpeg-dev \
-  pip3 --disable-pip-version-check install wheel pipenv && \
+  zlib1g-dev \
+  libwebp-dev \
+  redis-server && \
   # Create a 'app' user which the application will run as
   groupadd app && \
   useradd -M -d /app -s /bin/false -g app app && \
   # Install non-distro packages
-  pipenv install --system && \
+  PIPENV_VERBOSITY=64 pipenv install --system --skip-lock && \
   # Make absolutely sure we didn't accidentally bundle a SQLite dev database
   rm -rf /app/db.sqlite3 && \
   # Run any required app commands
@@ -96,20 +119,19 @@ RUN set -x && \
   mkdir -p /downloads/video && \
   # Clean up
   rm /app/Pipfile && \
-  rm /app/Pipfile.lock && \
   pipenv --clear && \
-  pip3 --disable-pip-version-check uninstall -y pipenv wheel virtualenv && \
   apt-get -y autoremove --purge \
-    python3-pip \
+  python3-pip \
-    python3-dev \
+  python3-dev \
-    gcc \
+  gcc \
-    make \
+  g++ \
-    default-libmysqlclient-dev \
+  make \
-    postgresql-common \
+  default-libmysqlclient-dev \
-    libpq-dev \
+  postgresql-common \
-    libjpeg-dev \
+  libpq-dev \
-    zlib1g-dev \
+  libjpeg-dev \
-    libwebp-dev && \
+  zlib1g-dev \
+  libwebp-dev && \
   apt-get -y autoremove && \
   apt-get -y autoclean && \
   rm -rf /var/lib/apt/lists/* && \
@@ -119,7 +141,12 @@ RUN set -x && \
   rm -rf /root && \
   mkdir -p /root && \
   chown root:root /root && \
-  chmod 0700 /root
+  chmod 0755 /root
+
+# Append software versions
+RUN set -x && \
+  FFMPEG_VERSION=$(/usr/local/bin/ffmpeg -version | head -n 1 | awk '{ print $3 }') && \
+  echo "ffmpeg_version = '${FFMPEG_VERSION}'" >> /app/common/third_party_versions.py
 
 # Copy root
 COPY config/root /
@@ -129,7 +156,7 @@ HEALTHCHECK --interval=1m --timeout=10s CMD /app/healthcheck.py http://127.0.0.1
 
 # ENVS and ports
 ENV PYTHONPATH "/app:${PYTHONPATH}"
-EXPOSE 8080
+EXPOSE 4848
 
 # Volumes
 VOLUME ["/config", "/downloads"]

Makefile

@@ -8,17 +8,17 @@ all: clean build
 
 
 dev:
-	$(python) app/manage.py runserver
+	$(python) tubesync/manage.py runserver
 
 
 build:
-	mkdir -p app/media
+	mkdir -p tubesync/media
-	mkdir -p app/static
+	mkdir -p tubesync/static
-	$(python) app/manage.py collectstatic --noinput
+	$(python) tubesync/manage.py collectstatic --noinput
 
 
 clean:
-	rm -rf app/static
+	rm -rf tubesync/static
 
 
 container: clean
@@ -29,5 +29,9 @@ runcontainer:
 	$(docker) run --rm --name $(name) --env-file dev.env --log-opt max-size=50m -ti -p 4848:4848 $(image)
 
 
-test:
+test: build
 	cd tubesync && $(python) manage.py test --verbosity=2 && cd ..
+
+
+shell:
+	cd tubesync && $(python) manage.py shell

Pipfile

@@ -4,9 +4,10 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
+autopep8 = "*"
 
 [packages]
-django = "*"
+django = "~=3.2"
 django-sass-processor = "*"
 libsass = "*"
 pillow = "*"
@@ -15,11 +16,10 @@ gunicorn = "*"
 django-compressor = "*"
 httptools = "*"
 django-background-tasks = "*"
-requests = "*"
 django-basicauth = "*"
 psycopg2-binary = "*"
 mysqlclient = "*"
 yt-dlp = "*"
-
+redis = "*"
-[requires]
+hiredis = "*"
-python_version = "3"
+requests = {extras = ["socks"], version = "*"}

Pipfile.lock

@@ -1,416 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "ac12e45a1719945b2e19d4a12b03136225f1f5e81affd1adf44a7b3c8dd36b8a"
-        },
-        "pipfile-spec": 6,
-        "requires": {
-            "python_version": "3"
-        },
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "asgiref": {
-            "hashes": [
-                "sha256:4ef1ab46b484e3c706329cedeff284a5d40824200638503f5768edb6de7d58e9",
-                "sha256:ffc141aa908e6f175673e7b1b3b7af4fdb0ecb738fc5c8b88f69f055c2415214"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.4.1"
-        },
-        "certifi": {
-            "hashes": [
-                "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
-                "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
-            ],
-            "version": "==2021.5.30"
-        },
-        "charset-normalizer": {
-            "hashes": [
-                "sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6",
-                "sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f"
-            ],
-            "markers": "python_version >= '3'",
-            "version": "==2.0.6"
-        },
-        "django": {
-            "hashes": [
-                "sha256:95b318319d6997bac3595517101ad9cc83fe5672ac498ba48d1a410f47afecd2",
-                "sha256:e93c93565005b37ddebf2396b4dc4b6913c1838baa82efdfb79acedd5816c240"
-            ],
-            "index": "pypi",
-            "version": "==3.2.7"
-        },
-        "django-appconf": {
-            "hashes": [
-                "sha256:1b1d0e1069c843ebe8ae5aa48ec52403b1440402b320c3e3a206a0907e97bb06",
-                "sha256:be58deb54a43d77d2e1621fe59f787681376d3cd0b8bd8e4758ef6c3a6453380"
-            ],
-            "version": "==1.0.4"
-        },
-        "django-background-tasks": {
-            "hashes": [
-                "sha256:e1b19e8d495a276c9d64c5a1ff8b41132f75d2f58e45be71b78650dad59af9de"
-            ],
-            "index": "pypi",
-            "version": "==1.2.5"
-        },
-        "django-basicauth": {
-            "hashes": [
-                "sha256:15e9e366f698f53c71b1e794dafea060f990a2ac556bae6b7330dd25324a091c",
-                "sha256:e5e47d1acdc1943bedcc1bf673059d6c15e257dfe9eef67a22fb824f79546c0d"
-            ],
-            "index": "pypi",
-            "version": "==0.5.3"
-        },
-        "django-compat": {
-            "hashes": [
-                "sha256:3ac9a3bedc56b9365d9eb241bc5157d0c193769bf995f9a78dc1bc24e7c2331b"
-            ],
-            "version": "==1.0.15"
-        },
-        "django-compressor": {
-            "hashes": [
-                "sha256:3358077605c146fdcca5f9eaffb50aa5dbe15f238f8854679115ebf31c0415e0",
-                "sha256:f8313f59d5e65712fc28787d084fe834997c9dfa92d064a1a3ec3d3366594d04"
-            ],
-            "index": "pypi",
-            "version": "==2.4.1"
-        },
-        "django-sass-processor": {
-            "hashes": [
-                "sha256:1f043180c47754018e803a77da003377f5ea6558de57cd6946eb27a32e9c16a2",
-                "sha256:dcaad47c591a2d52689c1bd209259e922e902d886293f0d5c9e0d1a4eb85eda2"
-            ],
-            "index": "pypi",
-            "version": "==1.0.1"
-        },
-        "gunicorn": {
-            "hashes": [
-                "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e",
-                "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"
-            ],
-            "index": "pypi",
-            "version": "==20.1.0"
-        },
-        "httptools": {
-            "hashes": [
-                "sha256:04114db99605c9b56ea22a8ec4d7b1485b908128ed4f4a8f6438489c428da794",
-                "sha256:074afd8afdeec0fa6786cd4a1676e0c0be23dc9a017a86647efa6b695168104f",
-                "sha256:113816f9af7dcfc4aa71ebb5354d77365f666ecf96ac7ff2aa1d24b6bca44165",
-                "sha256:1a8f26327023fa1a947d36e60a0582149e182fbbc949c8a65ec8665754dbbe69",
-                "sha256:2119fa619a4c53311f594f25c0205d619350fcb32140ec5057f861952e9b2b4f",
-                "sha256:21e948034f70e47c8abfa2d5e6f1a5661f87a2cddc7bcc70f61579cc87897c70",
-                "sha256:32a10a5903b5bc0eb647d01cd1e95bec3bb614a9bf53f0af1e01360b2debdf81",
-                "sha256:3787c1f46e9722ef7f07ea5c76b0103037483d1b12e34a02c53ceca5afa4e09a",
-                "sha256:3f82eb106e1474c63dba36a176067e65b48385f4cecddf3616411aa5d1fbdfec",
-                "sha256:3f9b4856d46ba1f0c850f4e84b264a9a8b4460acb20e865ec00978ad9fbaa4cf",
-                "sha256:4137137de8976511a392e27bfdcf231bd926ac13d375e0414e927b08217d779e",
-                "sha256:4687dfc116a9f1eb22a7d797f0dc6f6e17190d406ca4e729634b38aa98044b17",
-                "sha256:47dba2345aaa01b87e4981e8756af441349340708d5b60712c98c55a4d28f4af",
-                "sha256:5a836bd85ae1fb4304f674808488dae403e136d274aa5bafd0e6ee456f11c371",
-                "sha256:6e676bc3bb911b11f3d7e2144b9a53600bf6b9b21e0e4437aa308e1eef094d97",
-                "sha256:72ee0e3fb9c6437ab3ae34e9abee67fcee6876f4f58504e3f613dd5882aafdb7",
-                "sha256:79717080dc3f8b1eeb7f820b9b81528acbc04be6041f323fdd97550da2062575",
-                "sha256:8ac842df4fc3952efa7820b277961ea55e068bbc54cb59a0820400de7ae358d8",
-                "sha256:9f475b642c48b1b78584bdd12a5143e2c512485664331eade9c29ef769a17598",
-                "sha256:b8ac7dee63af4346e02b1e6d32202e3b5b3706a9928bec6da6d7a5b066217422",
-                "sha256:c0ac2e0ce6733c55858932e7d37fcc7b67ba6bb23e9648593c55f663de031b93",
-                "sha256:c14576b737d9e6e4f2a86af04918dbe9b62f57ce8102a8695c9a382dbe405c7f",
-                "sha256:cdc3975db86c29817e6d13df14e037c931fc893a710fb71097777a4147090068",
-                "sha256:eda95634027200f4b2a6d499e7c2e7fa9b8ee57e045dfda26958ea0af27c070b"
-            ],
-            "index": "pypi",
-            "version": "==0.3.0"
-        },
-        "idna": {
-            "hashes": [
-                "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a",
-                "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"
-            ],
-            "markers": "python_version >= '3'",
-            "version": "==3.2"
-        },
-        "libsass": {
-            "hashes": [
-                "sha256:06c8776417fe930714bdc930a3d7e795ae3d72be6ac883ff72a1b8f7c49e5ffb",
-                "sha256:12f39712de38689a8b785b7db41d3ba2ea1d46f9379d81ea4595802d91fa6529",
-                "sha256:1e25dd9047a9392d3c59a0b869e0404f2b325a03871ee45285ee33b3664f5613",
-                "sha256:659ae41af8708681fa3ec73f47b9735a6725e71c3b66ff570bfce78952f2314e",
-                "sha256:6b984510ed94993708c0d697b4fef2d118929bbfffc3b90037be0f5ccadf55e7",
-                "sha256:a005f298f64624f313a3ac618ab03f844c71d84ae4f4a4aec4b68d2a4ffe75eb",
-                "sha256:abc29357ee540849faf1383e1746d40d69ed5cb6d4c346df276b258f5aa8977a",
-                "sha256:d5ba529d9ce668be9380563279f3ffe988f27bc5b299c5a28453df2e0b0fbaf2",
-                "sha256:e2b1a7d093f2e76dc694c17c0c285e846d0b0deb0e8b21dc852ba1a3a4e2f1d6"
-            ],
-            "index": "pypi",
-            "version": "==0.21.0"
-        },
-        "mutagen": {
-            "hashes": [
-                "sha256:6397602efb3c2d7baebd2166ed85731ae1c1d475abca22090b7141ff5034b3e1",
-                "sha256:9c9f243fcec7f410f138cb12c21c84c64fde4195481a30c9bfb05b5f003adfed"
-            ],
-            "markers": "python_version >= '3.5' and python_version < '4'",
-            "version": "==1.45.1"
-        },
-        "mysqlclient": {
-            "hashes": [
-                "sha256:0ac0dd759c4ca02c35a9fedc24bc982cf75171651e8187c2495ec957a87dfff7",
-                "sha256:3381ca1a4f37ff1155fcfde20836b46416d66531add8843f6aa6d968982731c3",
-                "sha256:71c4b330cf2313bbda0307fc858cc9055e64493ba9bf28454d25cf8b3ee8d7f5",
-                "sha256:f6ebea7c008f155baeefe16c56cd3ee6239f7a5a9ae42396c2f1860f08a7c432",
-                "sha256:fc575093cf81b6605bed84653e48b277318b880dc9becf42dd47fa11ffd3e2b6"
-            ],
-            "index": "pypi",
-            "version": "==2.0.3"
-        },
-        "pillow": {
-            "hashes": [
-                "sha256:0412516dcc9de9b0a1e0ae25a280015809de8270f134cc2c1e32c4eeb397cf30",
-                "sha256:04835e68ef12904bc3e1fd002b33eea0779320d4346082bd5b24bec12ad9c3e9",
-                "sha256:06d1adaa284696785375fa80a6a8eb309be722cf4ef8949518beb34487a3df71",
-                "sha256:085a90a99404b859a4b6c3daa42afde17cb3ad3115e44a75f0d7b4a32f06a6c9",
-                "sha256:0b9911ec70731711c3b6ebcde26caea620cbdd9dcb73c67b0730c8817f24711b",
-                "sha256:10e00f7336780ca7d3653cf3ac26f068fa11b5a96894ea29a64d3dc4b810d630",
-                "sha256:11c27e74bab423eb3c9232d97553111cc0be81b74b47165f07ebfdd29d825875",
-                "sha256:11eb7f98165d56042545c9e6db3ce394ed8b45089a67124298f0473b29cb60b2",
-                "sha256:13654b521fb98abdecec105ea3fb5ba863d1548c9b58831dd5105bb3873569f1",
-                "sha256:15ccb81a6ffc57ea0137f9f3ac2737ffa1d11f786244d719639df17476d399a7",
-                "sha256:18a07a683805d32826c09acfce44a90bf474e6a66ce482b1c7fcd3757d588df3",
-                "sha256:19ec4cfe4b961edc249b0e04b5618666c23a83bc35842dea2bfd5dfa0157f81b",
-                "sha256:1c3ff00110835bdda2b1e2b07f4a2548a39744bb7de5946dc8e95517c4fb2ca6",
-                "sha256:27a330bf7014ee034046db43ccbb05c766aa9e70b8d6c5260bfc38d73103b0ba",
-                "sha256:2b11c9d310a3522b0fd3c35667914271f570576a0e387701f370eb39d45f08a4",
-                "sha256:2c661542c6f71dfd9dc82d9d29a8386287e82813b0375b3a02983feac69ef864",
-                "sha256:2cde7a4d3687f21cffdf5bb171172070bb95e02af448c4c8b2f223d783214056",
-                "sha256:2d5e9dc0bf1b5d9048a94c48d0813b6c96fccfa4ccf276d9c36308840f40c228",
-                "sha256:2f23b2d3079522fdf3c09de6517f625f7a964f916c956527bed805ac043799b8",
-                "sha256:35d27687f027ad25a8d0ef45dd5208ef044c588003cdcedf05afb00dbc5c2deb",
-                "sha256:35d409030bf3bd05fa66fb5fdedc39c521b397f61ad04309c90444e893d05f7d",
-                "sha256:4326ea1e2722f3dc00ed77c36d3b5354b8fb7399fb59230249ea6d59cbed90da",
-                "sha256:4abc247b31a98f29e5224f2d31ef15f86a71f79c7f4d2ac345a5d551d6393073",
-                "sha256:4d89a2e9219a526401015153c0e9dd48319ea6ab9fe3b066a20aa9aee23d9fd3",
-                "sha256:4e59e99fd680e2b8b11bbd463f3c9450ab799305d5f2bafb74fefba6ac058616",
-                "sha256:548794f99ff52a73a156771a0402f5e1c35285bd981046a502d7e4793e8facaa",
-                "sha256:56fd98c8294f57636084f4b076b75f86c57b2a63a8410c0cd172bc93695ee979",
-                "sha256:59697568a0455764a094585b2551fd76bfd6b959c9f92d4bdec9d0e14616303a",
-                "sha256:6bff50ba9891be0a004ef48828e012babaaf7da204d81ab9be37480b9020a82b",
-                "sha256:6cb3dd7f23b044b0737317f892d399f9e2f0b3a02b22b2c692851fb8120d82c6",
-                "sha256:7dbfbc0020aa1d9bc1b0b8bcf255a7d73f4ad0336f8fd2533fcc54a4ccfb9441",
-                "sha256:838eb85de6d9307c19c655c726f8d13b8b646f144ca6b3771fa62b711ebf7624",
-                "sha256:8b68f565a4175e12e68ca900af8910e8fe48aaa48fd3ca853494f384e11c8bcd",
-                "sha256:8f284dc1695caf71a74f24993b7c7473d77bc760be45f776a2c2f4e04c170550",
-                "sha256:963ebdc5365d748185fdb06daf2ac758116deecb2277ec5ae98139f93844bc09",
-                "sha256:a048dad5ed6ad1fad338c02c609b862dfaa921fcd065d747194a6805f91f2196",
-                "sha256:a1bd983c565f92779be456ece2479840ec39d386007cd4ae83382646293d681b",
-                "sha256:a66566f8a22561fc1a88dc87606c69b84fa9ce724f99522cf922c801ec68f5c1",
-                "sha256:bcb04ff12e79b28be6c9988f275e7ab69f01cc2ba319fb3114f87817bb7c74b6",
-                "sha256:bd24054aaf21e70a51e2a2a5ed1183560d3a69e6f9594a4bfe360a46f94eba83",
-                "sha256:be25cb93442c6d2f8702c599b51184bd3ccd83adebd08886b682173e09ef0c3f",
-                "sha256:c691b26283c3a31594683217d746f1dad59a7ae1d4cfc24626d7a064a11197d4",
-                "sha256:cc9d0dec711c914ed500f1d0d3822868760954dce98dfb0b7382a854aee55d19",
-                "sha256:ce2e5e04bb86da6187f96d7bab3f93a7877830981b37f0287dd6479e27a10341",
-                "sha256:ce651ca46d0202c302a535d3047c55a0131a720cf554a578fc1b8a2aff0e7d96",
-                "sha256:d0c8ebbfd439c37624db98f3877d9ed12c137cadd99dde2d2eae0dab0bbfc355",
-                "sha256:d675a876b295afa114ca8bf42d7f86b5fb1298e1b6bb9a24405a3f6c8338811c",
-                "sha256:dde3f3ed8d00c72631bc19cbfff8ad3b6215062a5eed402381ad365f82f0c18c",
-                "sha256:e5a31c07cea5edbaeb4bdba6f2b87db7d3dc0f446f379d907e51cc70ea375629",
-                "sha256:f514c2717012859ccb349c97862568fdc0479aad85b0270d6b5a6509dbc142e2",
-                "sha256:fc0db32f7223b094964e71729c0361f93db43664dd1ec86d3df217853cedda87",
-                "sha256:fd4fd83aa912d7b89b4b4a1580d30e2a4242f3936882a3f433586e5ab97ed0d5",
-                "sha256:feb5db446e96bfecfec078b943cc07744cc759893cef045aa8b8b6d6aaa8274e"
-            ],
-            "index": "pypi",
-            "version": "==8.3.2"
-        },
-        "psycopg2-binary": {
-            "hashes": [
-                "sha256:0b7dae87f0b729922e06f85f667de7bf16455d411971b2043bbd9577af9d1975",
-                "sha256:0f2e04bd2a2ab54fa44ee67fe2d002bb90cee1c0f1cc0ebc3148af7b02034cbd",
-                "sha256:123c3fb684e9abfc47218d3784c7b4c47c8587951ea4dd5bc38b6636ac57f616",
-                "sha256:1473c0215b0613dd938db54a653f68251a45a78b05f6fc21af4326f40e8360a2",
-                "sha256:14db1752acdd2187d99cb2ca0a1a6dfe57fc65c3281e0f20e597aac8d2a5bd90",
-                "sha256:1e3a362790edc0a365385b1ac4cc0acc429a0c0d662d829a50b6ce743ae61b5a",
-                "sha256:1e85b74cbbb3056e3656f1cc4781294df03383127a8114cbc6531e8b8367bf1e",
-                "sha256:20f1ab44d8c352074e2d7ca67dc00843067788791be373e67a0911998787ce7d",
-                "sha256:2f62c207d1740b0bde5c4e949f857b044818f734a3d57f1d0d0edc65050532ed",
-                "sha256:3242b9619de955ab44581a03a64bdd7d5e470cc4183e8fcadd85ab9d3756ce7a",
-                "sha256:35c4310f8febe41f442d3c65066ca93cccefd75013df3d8c736c5b93ec288140",
-                "sha256:4235f9d5ddcab0b8dbd723dca56ea2922b485ea00e1dafacf33b0c7e840b3d32",
-                "sha256:5ced67f1e34e1a450cdb48eb53ca73b60aa0af21c46b9b35ac3e581cf9f00e31",
-                "sha256:7360647ea04db2e7dff1648d1da825c8cf68dc5fbd80b8fb5b3ee9f068dcd21a",
-                "sha256:8c13d72ed6af7fd2c8acbd95661cf9477f94e381fce0792c04981a8283b52917",
-                "sha256:988b47ac70d204aed01589ed342303da7c4d84b56c2f4c4b8b00deda123372bf",
-                "sha256:995fc41ebda5a7a663a254a1dcac52638c3e847f48307b5416ee373da15075d7",
-                "sha256:a36c7eb6152ba5467fb264d73844877be8b0847874d4822b7cf2d3c0cb8cdcb0",
-                "sha256:aed4a9a7e3221b3e252c39d0bf794c438dc5453bc2963e8befe9d4cd324dff72",
-                "sha256:aef9aee84ec78af51107181d02fe8773b100b01c5dfde351184ad9223eab3698",
-                "sha256:b0221ca5a9837e040ebf61f48899926b5783668b7807419e4adae8175a31f773",
-                "sha256:b4d7679a08fea64573c969f6994a2631908bb2c0e69a7235648642f3d2e39a68",
-                "sha256:c250a7ec489b652c892e4f0a5d122cc14c3780f9f643e1a326754aedf82d9a76",
-                "sha256:ca86db5b561b894f9e5f115d6a159fff2a2570a652e07889d8a383b5fae66eb4",
-                "sha256:cfc523edecddaef56f6740d7de1ce24a2fdf94fd5e704091856a201872e37f9f",
-                "sha256:da113b70f6ec40e7d81b43d1b139b9db6a05727ab8be1ee559f3a69854a69d34",
-                "sha256:f6fac64a38f6768e7bc7b035b9e10d8a538a9fadce06b983fb3e6fa55ac5f5ce",
-                "sha256:f8559617b1fcf59a9aedba2c9838b5b6aa211ffedecabca412b92a1ff75aac1a",
-                "sha256:fbb42a541b1093385a2d8c7eec94d26d30437d0e77c1d25dae1dcc46741a385e"
-            ],
-            "index": "pypi",
-            "version": "==2.9.1"
-        },
-        "pycryptodome": {
-            "hashes": [
-                "sha256:09c1555a3fa450e7eaca41ea11cd00afe7c91fef52353488e65663777d8524e0",
-                "sha256:12222a5edc9ca4a29de15fbd5339099c4c26c56e13c2ceddf0b920794f26165d",
-                "sha256:1723ebee5561628ce96748501cdaa7afaa67329d753933296321f0be55358dce",
-                "sha256:1c5e1ca507de2ad93474be5cfe2bfa76b7cf039a1a32fc196f40935944871a06",
-                "sha256:2603c98ae04aac675fefcf71a6c87dc4bb74a75e9071ae3923bbc91a59f08d35",
-                "sha256:2dea65df54349cdfa43d6b2e8edb83f5f8d6861e5cf7b1fbc3e34c5694c85e27",
-                "sha256:31c1df17b3dc5f39600a4057d7db53ac372f492c955b9b75dd439f5d8b460129",
-                "sha256:38661348ecb71476037f1e1f553159b80d256c00f6c0b00502acac891f7116d9",
-                "sha256:3e2e3a06580c5f190df843cdb90ea28d61099cf4924334d5297a995de68e4673",
-                "sha256:3f840c49d38986f6e17dbc0673d37947c88bc9d2d9dba1c01b979b36f8447db1",
-                "sha256:501ab36aae360e31d0ec370cf5ce8ace6cb4112060d099b993bc02b36ac83fb6",
-                "sha256:60386d1d4cfaad299803b45a5bc2089696eaf6cdd56f9fc17479a6f89595cfc8",
-                "sha256:6260e24d41149268122dd39d4ebd5941e9d107f49463f7e071fd397e29923b0c",
-                "sha256:6bbf7fee7b7948b29d7e71fcacf48bac0c57fb41332007061a933f2d996f9713",
-                "sha256:6d2df5223b12437e644ce0a3be7809471ffa71de44ccd28b02180401982594a6",
-                "sha256:758949ca62690b1540dfb24ad773c6da9cd0e425189e83e39c038bbd52b8e438",
-                "sha256:77997519d8eb8a4adcd9a47b9cec18f9b323e296986528186c0e9a7a15d6a07e",
-                "sha256:7fd519b89585abf57bf47d90166903ec7b43af4fe23c92273ea09e6336af5c07",
-                "sha256:98213ac2b18dc1969a47bc65a79a8fca02a414249d0c8635abb081c7f38c91b6",
-                "sha256:99b2f3fc51d308286071d0953f92055504a6ffe829a832a9fc7a04318a7683dd",
-                "sha256:9b6f711b25e01931f1c61ce0115245a23cdc8b80bf8539ac0363bdcf27d649b6",
-                "sha256:a3105a0eb63eacf98c2ecb0eb4aa03f77f40fbac2bdde22020bb8a536b226bb8",
-                "sha256:a8eb8b6ea09ec1c2535bf39914377bc8abcab2c7d30fa9225eb4fe412024e427",
-                "sha256:a92d5c414e8ee1249e850789052608f582416e82422502dc0ac8c577808a9067",
-                "sha256:d3d6958d53ad307df5e8469cc44474a75393a434addf20ecd451f38a72fe29b8",
-                "sha256:e0a4d5933a88a2c98bbe19c0c722f5483dc628d7a38338ac2cb64a7dbd34064b",
-                "sha256:e3bf558c6aeb49afa9f0c06cee7fb5947ee5a1ff3bd794b653d39926b49077fa",
-                "sha256:e61e363d9a5d7916f3a4ce984a929514c0df3daf3b1b2eb5e6edbb131ee771cf",
-                "sha256:f977cdf725b20f6b8229b0c87acb98c7717e742ef9f46b113985303ae12a99da",
-                "sha256:fc7489a50323a0df02378bc2fff86eb69d94cc5639914346c736be981c6a02e7"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==3.10.1"
-        },
-        "pytz": {
-            "hashes": [
-                "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
-                "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
-            ],
-            "version": "==2021.1"
-        },
-        "rcssmin": {
-            "hashes": [
-                "sha256:ca87b695d3d7864157773a61263e5abb96006e9ff0e021eff90cbe0e1ba18270"
-            ],
-            "version": "==1.0.6"
-        },
-        "requests": {
-            "hashes": [
-                "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
-                "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
-            ],
-            "index": "pypi",
-            "version": "==2.26.0"
-        },
-        "rjsmin": {
-            "hashes": [
-                "sha256:0ab825839125eaca57cc59581d72e596e58a7a56fbc0839996b7528f0343a0a8",
-                "sha256:211c2fe8298951663bbc02acdffbf714f6793df54bfc50e1c6c9e71b3f2559a3",
-                "sha256:466fe70cc5647c7c51b3260c7e2e323a98b2b173564247f9c89e977720a0645f",
-                "sha256:585e75a84d9199b68056fd4a083d9a61e2a92dfd10ff6d4ce5bdb04bc3bdbfaf",
-                "sha256:6044ca86e917cd5bb2f95e6679a4192cef812122f28ee08c677513de019629b3",
-                "sha256:714329db774a90947e0e2086cdddb80d5e8c4ac1c70c9f92436378dedb8ae345",
-                "sha256:799890bd07a048892d8d3deb9042dbc20b7f5d0eb7da91e9483c561033b23ce2",
-                "sha256:975b69754d6a76be47c0bead12367a1ca9220d08e5393f80bab0230d4625d1f4",
-                "sha256:b15dc75c71f65d9493a8c7fa233fdcec823e3f1b88ad84a843ffef49b338ac32",
-                "sha256:dd0f4819df4243ffe4c964995794c79ca43943b5b756de84be92b445a652fb86",
-                "sha256:e3908b21ebb584ce74a6ac233bdb5f29485752c9d3be5e50c5484ed74169232c",
-                "sha256:e487a7783ac4339e79ec610b98228eb9ac72178973e3dee16eba0e3feef25924",
-                "sha256:ecd29f1b3e66a4c0753105baec262b331bcbceefc22fbe6f7e8bcd2067bcb4d7"
-            ],
-            "version": "==1.1.0"
-        },
-        "six": {
-            "hashes": [
-                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
-                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.16.0"
-        },
-        "sqlparse": {
-            "hashes": [
-                "sha256:0c00730c74263a94e5a9919ade150dfc3b19c574389985446148402998287dae",
-                "sha256:48719e356bb8b42991bdbb1e8b83223757b93789c00910a616a071910ca4a64d"
-            ],
-            "markers": "python_version >= '3.5'",
-            "version": "==0.4.2"
-        },
-        "urllib3": {
-            "hashes": [
-                "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4",
-                "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
-            "version": "==1.26.6"
-        },
-        "websockets": {
-            "hashes": [
-                "sha256:01db0ecd1a0ca6702d02a5ed40413e18b7d22f94afb3bbe0d323bac86c42c1c8",
-                "sha256:085bb8a6e780d30eaa1ba48ac7f3a6707f925edea787cfb761ce5a39e77ac09b",
-                "sha256:1ac35426fe3e7d3d0fac3d63c8965c76ed67a8fd713937be072bf0ce22808539",
-                "sha256:1f6b814cff6aadc4288297cb3a248614829c6e4ff5556593c44a115e9dd49939",
-                "sha256:2a43072e434c041a99f2e1eb9b692df0232a38c37c61d00e9f24db79474329e4",
-                "sha256:5b2600e01c7ca6f840c42c747ffbe0254f319594ed108db847eb3d75f4aacb80",
-                "sha256:62160772314920397f9d219147f958b33fa27a12c662d4455c9ccbba9a07e474",
-                "sha256:706e200fc7f03bed99ad0574cd1ea8b0951477dd18cc978ccb190683c69dba76",
-                "sha256:71358c7816e2762f3e4af3adf0040f268e219f5a38cb3487a9d0fc2e554fef6a",
-                "sha256:7d2e12e4f901f1bc062dfdf91831712c4106ed18a9a4cdb65e2e5f502124ca37",
-                "sha256:7f79f02c7f9a8320aff7d3321cd1c7e3a7dbc15d922ac996cca827301ee75238",
-                "sha256:82b17524b1ce6ae7f7dd93e4d18e9b9474071e28b65dbf1dfe9b5767778db379",
-                "sha256:82bd921885231f4a30d9bc550552495b3fc36b1235add6d374e7c65c3babd805",
-                "sha256:8bbf8660c3f833ddc8b1afab90213f2e672a9ddac6eecb3cde968e6b2807c1c7",
-                "sha256:9a4d889162bd48588e80950e07fa5e039eee9deb76a58092e8c3ece96d7ef537",
-                "sha256:b4ade7569b6fd17912452f9c3757d96f8e4044016b6d22b3b8391e641ca50456",
-                "sha256:b8176deb6be540a46695960a765a77c28ac8b2e3ef2ec95d50a4f5df901edb1c",
-                "sha256:c4fc9a1d242317892590abe5b61a9127f1a61740477bfb121743f290b8054002",
-                "sha256:c5880442f5fc268f1ef6d37b2c152c114deccca73f48e3a8c48004d2f16f4567",
-                "sha256:cd8c6f2ec24aedace251017bc7a414525171d4e6578f914acab9349362def4da",
-                "sha256:d67646ddd17a86117ae21c27005d83c1895c0cef5d7be548b7549646372f868a",
-                "sha256:e42a1f1e03437b017af341e9bbfdc09252cd48ef32a8c3c3ead769eab3b17368",
-                "sha256:eb282127e9c136f860c6068a4fba5756eb25e755baffb5940b6f1eae071928b2",
-                "sha256:fe83b3ec9ef34063d86dfe1029160a85f24a5a94271036e5714a57acfdd089a1",
-                "sha256:ff59c6bdb87b31f7e2d596f09353d5a38c8c8ff571b0e2238e8ee2d55ad68465"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==10.0"
-        },
-        "whitenoise": {
-            "hashes": [
-                "sha256:d234b871b52271ae7ed6d9da47ffe857c76568f11dd30e28e18c5869dbd11e12",
-                "sha256:d963ef25639d1417e8a247be36e6aedd8c7c6f0a08adcb5a89146980a96b577c"
-            ],
-            "index": "pypi",
-            "version": "==5.3.0"
-        },
-        "yt-dlp": {
-            "hashes": [
-                "sha256:c97716a715261657345176ab8190a19efa51db0e5b174a6629956548750245e1",
-                "sha256:ca7e77cdb055ba2683df5b0807aab1c1e120cbe02c8f35d9d3293d94dbdaea63"
-            ],
-            "index": "pypi",
-            "version": "==2021.9.2"
-        }
-    },
-    "develop": {}
-}

README.md

@@ -9,9 +9,9 @@ downloaded.
 
 If you want to watch YouTube videos in particular quality or settings from your local
 media server, then TubeSync is for you. Internally, TubeSync is a web interface wrapper
-on `youtube-dl` and `ffmpeg` with a task scheduler.
+on `yt-dlp` and `ffmpeg` with a task scheduler.
 
-There are several other web interfaces to YouTube and `youtube-dl` all with varying
+There are several other web interfaces to YouTube and `yt-dlp` all with varying
 features and implementations. TubeSync's largest difference is full PVR experience of
 updating media servers and better selection of media formats. Additionally, to be as
 hands-free as possible, TubeSync has gradual retrying of failures with back-off timers
@@ -69,11 +69,12 @@ currently just Plex, to complete the PVR experience.
 # Installation
 
 TubeSync is designed to be run in a container, such as via Docker or Podman. It also
-works in a Docker Compose stack. Only `amd64` is initially supported.
+works in a Docker Compose stack. `amd64` (most desktop PCs and servers) and `arm64`
+(modern ARM computers, such as the Rasperry Pi 3 or later) are supported.
 
 Example (with Docker on *nix):
 
-First find your the user ID and group ID you want to run TubeSync as, if you're not
+First find the user ID and group ID you want to run TubeSync as, if you're not
 sure what this is it's probably your current user ID and group ID:
 
 ```bash
@@ -116,11 +117,13 @@ $ docker run \
 Once running, open `http://localhost:4848` in your browser and you should see the
 TubeSync dashboard. If you do, you can proceed to adding some sources (YouTube channels
 and playlists). If not, check `docker logs tubesync` to see what errors might be
-occuring, typical ones are file permission issues.
+occurring, typical ones are file permission issues.
 
 Alternatively, for Docker Compose, you can use something like:
 
-```yaml
+```yml
+version: '3.7'
+services:
   tubesync:
     image: ghcr.io/meeb/tubesync:latest
     container_name: tubesync
@@ -148,7 +151,7 @@ HTTP_USER
 HTTP_PASS
 ```
 
-For example in the `docker run ...` line add in:
+For example, in the `docker run ...` line add in:
 
 ```bash
 ...
@@ -233,13 +236,12 @@ $ docker logs --follow tubesync
 Once you're happy using TubeSync there are some advanced usage guides for more complex
 and less common features:
 
-![Import existing media into TubeSync](https://github.com/meeb/tubesync/blob/main/docs/import-existing-media.md)
+ * [Import existing media into TubeSync](https://github.com/meeb/tubesync/blob/main/docs/import-existing-media.md)
-
+ * [Sync or create missing metadata files](https://github.com/meeb/tubesync/blob/main/docs/create-missing-metadata.md)
-![Sync or create missing metadata files](https://github.com/meeb/tubesync/blob/main/docs/create-missing-metadata.md)
+ * [Reset tasks from the command line](https://github.com/meeb/tubesync/blob/main/docs/reset-tasks.md)
-
+ * [Using PostgreSQL, MySQL or MariaDB as database backends](https://github.com/meeb/tubesync/blob/main/docs/other-database-backends.md)
-![Reset tasks from the command line](https://github.com/meeb/tubesync/blob/main/docs/reset-tasks.md)
+ * [Using cookies](https://github.com/meeb/tubesync/blob/main/docs/using-cookies.md)
-
+ * [Reset metadata](https://github.com/meeb/tubesync/blob/main/docs/reset-metadata.md)
-![Using PostgreSQL, MySQL or MariaDB as database backends](https://github.com/meeb/tubesync/blob/main/docs/other-database-backends.md)
 
 
 # Warnings
@@ -281,7 +283,7 @@ automatically.
 ### Does TubeSync support any other video platforms?
 
 At the moment, no. This is a pre-release. The library TubeSync uses that does most
-of the downloading work, `youtube-dl`, supports many hundreds of video sources so it's
+of the downloading work, `yt-dlp`, supports many hundreds of video sources so it's
 likely more will be added to TubeSync if there is demand for it.
 
 ### Is there a progress bar?
@@ -293,27 +295,27 @@ your install is doing check the container logs.
 
 ### Are there alerts when a download is complete?
 
-No, this feature is best served by existing services such as the execelent
+No, this feature is best served by existing services such as the excellent 
 [Tautulli](https://tautulli.com/) which can monitor your Plex server and send alerts
 that way.
 
-### There's errors in my "tasks" tab!
+### There are errors in my "tasks" tab!
 
 You only really need to worry about these if there is a permanent failure. Some errors
-are temproary and will be retried for you automatically, such as a download got
+are temporary and will be retried for you automatically, such as a download got
-interrupted and will be tried again later. Sources with permanet errors (such as no
+interrupted and will be tried again later. Sources with permanent errors (such as no
 media available because you got a channel name wrong) will be shown as errors on the
 "sources" tab.
 
 ### What is TubeSync written in?
 
-Python3 using Django, embedding youtube-dl. It's pretty much glue between other much
+Python3 using Django, embedding yt-dlp. It's pretty much glue between other much
 larger libraries.
 
 Notable libraries and software used:
 
  * [Django](https://www.djangoproject.com/)
- * [youtube-dl](https://yt-dl.org/)
+ * [yt-dlp](https://github.com/yt-dlp/yt-dlp)
  * [ffmpeg](https://ffmpeg.org/)
  * [Django Background Tasks](https://github.com/arteria/django-background-tasks/)
  * [django-sass](https://github.com/coderedcorp/django-sass/)
@@ -323,7 +325,7 @@ See the [Pipefile](https://github.com/meeb/tubesync/blob/main/Pipfile) for a ful
 
 ### Can I get access to the full Django admin?
 
-Yes, although pretty much all operations are available through the front end interface
+Yes, although pretty much all operations are available through the front-end interface
 and you can probably break things by playing in the admin. If you still want to access
 it you can run:
 
@@ -349,6 +351,10 @@ etc.). Configuration of this is beyond the scope of this README.
 
 Just `amd64` for the moment. Others may be made available if there is demand.
 
+### The pipenv install fails with "Locking failed"!
+
+Make sure that you have `mysql_config` or `mariadb_config` available, as required by the python module `mysqlclient`. On Debian-based systems this is usually found in the package `libmysqlclient-dev`
+
 
 # Advanced configuration
 
@@ -356,25 +362,26 @@ There are a number of other environment variables you can set. These are, mostly
 **NOT** required to be set in the default container installation, they are really only
 useful if you are manually installing TubeSync in some other environment. These are:
 
-| Name                     | What                                                         | Example                              |
+| Name                        | What                                                         | Example                              |
-| ------------------------ | ------------------------------------------------------------ | ------------------------------------ |
+| --------------------------- | ------------------------------------------------------------ | ------------------------------------ |
-| DJANGO_SECRET_KEY        | Django's SECRET_KEY                                          | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l     |
+| DJANGO_SECRET_KEY           | Django's SECRET_KEY                                          | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l     |
-| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME                                   | /somepath                            |
+| DJANGO_URL_PREFIX           | Run TubeSync in a sub-URL on the web server                  | /somepath/                           |
-| TUBESYNC_DEBUG           | Enable debugging                                             | True                                 |
+| TUBESYNC_DEBUG              | Enable debugging                                             | True                                 |
-| TUBESYNC_WORKERS         | Number of background workers, default is 2, max allowed is 8 | 2                                    |
+| TUBESYNC_WORKERS            | Number of background workers, default is 2, max allowed is 8 | 2                                    |
-| TUBESYNC_HOSTS           | Django's ALLOWED_HOSTS                                       | tubesync.example.com,otherhost.com   |
+| TUBESYNC_HOSTS              | Django's ALLOWED_HOSTS, defaults to `*`                      | tubesync.example.com,otherhost.com   |
-| GUNICORN_WORKERS         | Number of gunicorn workers to spawn                          | 3                                    |
+| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True  | True
-| LISTEN_HOST              | IP address for gunicorn to listen on                         | 127.0.0.1                            |
+| GUNICORN_WORKERS            | Number of gunicorn workers to spawn                          | 3                                    |
-| LISTEN_PORT              | Port number for gunicorn to listen on                        | 8080                                 |
+| LISTEN_HOST                 | IP address for gunicorn to listen on                         | 127.0.0.1                            |
-| HTTP_USER                | Sets the username for HTTP basic authentication              | some-username                        |
+| LISTEN_PORT                 | Port number for gunicorn to listen on                        | 8080                                 |
-| HTTP_PASS                | Sets the password for HTTP basic authentication              | some-secure-password                 |
+| HTTP_USER                   | Sets the username for HTTP basic authentication              | some-username                        |
-| DATABASE_CONNECTION      | Optional external database connection details                | mysql://user:pass@host:port/database |
+| HTTP_PASS                   | Sets the password for HTTP basic authentication              | some-secure-password                 |
+| DATABASE_CONNECTION         | Optional external database connection details                | mysql://user:pass@host:port/database |
 
 
 # Manual, non-containerised, installation
 
 As a relatively normal Django app you can run TubeSync without the container. Beyond
-following this rough guide you are on your own and should be knowledgeable about
+following this rough guide, you are on your own and should be knowledgeable about
 installing and running WSGI-based Python web applications before attempting this.
 
 1. Clone or download this repo
@@ -397,7 +404,7 @@ installing and running WSGI-based Python web applications before attempting this
 
 # Tests
 
-There is a moderately comprehensive test suite focussing on the custom media format
+There is a moderately comprehensive test suite focusing on the custom media format
 matching logic and that the front-end interface works. You can run it via Django:
 
 ```bash

config/root/etc/cont-init.d/10-tubesync

@@ -1,27 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-# Change runtime user UID and GID
-PUID=${PUID:-911}
-PGID=${PGID:-911}
-groupmod -o -g "$PGID" app
-usermod -o -u "$PUID" app
-
-# Reset permissions
-chown -R app:app /run/app && \
-chmod -R 0700 /run/app && \
-chown -R app:app /config && \
-chmod -R 0755 /config && \
-chown -R app:app /downloads && \
-chmod -R 0755 /downloads && \
-chown -R root:app /app && \
-chmod -R 0750 /app && \
-chown -R app:app /app/common/static && \
-chmod -R 0750 /app/common/static && \
-chown -R app:app /app/static && \
-chmod -R 0750 /app/static && \
-find /app -type f ! -iname healthcheck.py -exec chmod 640 {} \; && \
-chmod 0755 /app/healthcheck.py
-
-# Run migrations
-exec s6-setuidgid app \
-    /usr/bin/python3 /app/manage.py migrate

config/root/etc/nginx/nginx.conf

@@ -79,6 +79,11 @@ http {
             proxy_connect_timeout 10;
         }
 
+        # File dwnload and streaming
+        location /media-data/ {
+            internal;
+            alias /downloads/;
+        }
     }
 
 }

config/root/etc/redis/redis.conf

@@ -0,0 +1,46 @@
+bind 127.0.0.1
+protected-mode yes
+port 6379
+tcp-backlog 511
+timeout 0
+tcp-keepalive 300
+daemonize no
+supervised no
+loglevel notice
+logfile ""
+databases 1
+always-show-logo no
+save ""
+dir /var/lib/redis
+maxmemory 64mb
+maxmemory-policy noeviction
+lazyfree-lazy-eviction no
+lazyfree-lazy-expire no
+lazyfree-lazy-server-del no
+replica-lazy-flush no
+lazyfree-lazy-user-del no
+oom-score-adj no
+oom-score-adj-values 0 200 800
+appendonly no
+appendfsync no
+lua-time-limit 5000
+slowlog-log-slower-than 10000
+slowlog-max-len 128
+latency-monitor-threshold 0
+notify-keyspace-events ""
+hash-max-ziplist-entries 512
+hash-max-ziplist-value 64
+list-max-ziplist-size -2
+list-compress-depth 0
+set-max-intset-entries 512
+zset-max-ziplist-entries 128
+zset-max-ziplist-value 64
+hll-sparse-max-bytes 3000
+stream-node-max-bytes 4096
+stream-node-max-entries 100
+activerehashing yes
+client-output-buffer-limit normal 0 0 0
+client-output-buffer-limit replica 256mb 64mb 60
+client-output-buffer-limit pubsub 32mb 8mb 60
+hz 10
+dynamic-hz yes

config/root/etc/s6-overlay/s6-rc.d/celery-beat/dependencies

@@ -0,0 +1 @@
+gunicorn

config/root/etc/s6-overlay/s6-rc.d/celery-beat/run

@@ -0,0 +1,25 @@
+#!/usr/bin/with-contenv bash
+
+UMASK_SET=${UMASK_SET:-022}
+umask "$UMASK_SET"
+
+cd /app || exit
+
+PIDFILE=/run/app/celery-beat.pid
+SCHEDULE=/tmp/tubesync-celerybeat-schedule
+
+if [ -f "${PIDFILE}" ]
+then
+    PID=$(cat $PIDFILE)
+    echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
+    if kill -0 $PID
+    then
+        echo "Killing old gunicorn process with PID: ${PID}"
+        kill -9 $PID
+    fi
+    echo "Removing stale PID file: ${PIDFILE}"
+    rm ${PIDFILE}
+fi
+
+#exec s6-setuidgid app \
+#    /usr/local/bin/celery --workdir /app -A tubesync beat --pidfile ${PIDFILE} -s ${SCHEDULE}

config/root/etc/s6-overlay/s6-rc.d/celery-beat/type

@@ -0,0 +1 @@
+longrun

config/root/etc/s6-overlay/s6-rc.d/celery-worker/dependencies

@@ -0,0 +1 @@
+gunicorn

config/root/etc/s6-overlay/s6-rc.d/celery-worker/run

@@ -0,0 +1,24 @@
+#!/usr/bin/with-contenv bash
+
+UMASK_SET=${UMASK_SET:-022}
+umask "$UMASK_SET"
+
+cd /app || exit
+
+PIDFILE=/run/app/celery-worker.pid
+
+if [ -f "${PIDFILE}" ]
+then
+    PID=$(cat $PIDFILE)
+    echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
+    if kill -0 $PID
+    then
+        echo "Killing old gunicorn process with PID: ${PID}"
+        kill -9 $PID
+    fi
+    echo "Removing stale PID file: ${PIDFILE}"
+    rm ${PIDFILE}
+fi
+
+#exec s6-setuidgid app \
+#    /usr/local/bin/celery --workdir /app -A tubesync worker --pidfile ${PIDFILE} -l INFO

config/root/etc/s6-overlay/s6-rc.d/celery-worker/type

@@ -0,0 +1 @@
+longrun

config/root/etc/s6-overlay/s6-rc.d/gunicorn/dependencies

@@ -0,0 +1 @@
+tubesync-init

config/root/etc/s6-overlay/s6-rc.d/gunicorn/run

@@ -1,4 +1,4 @@
-#!/usr/bin/with-contenv bash
+#!/command/with-contenv bash
 
 UMASK_SET=${UMASK_SET:-022}
 umask "$UMASK_SET"

config/root/etc/s6-overlay/s6-rc.d/gunicorn/type

@@ -0,0 +1 @@
+longrun

config/root/etc/s6-overlay/s6-rc.d/nginx/dependencies

@@ -0,0 +1 @@
+gunicorn

config/root/etc/s6-overlay/s6-rc.d/nginx/run

@@ -0,0 +1,5 @@
+#!/command/with-contenv bash
+
+cd /
+
+/usr/sbin/nginx

config/root/etc/s6-overlay/s6-rc.d/nginx/type

@@ -0,0 +1 @@
+longrun

config/root/etc/s6-overlay/s6-rc.d/redis/run

@@ -0,0 +1,4 @@
+#!/command/with-contenv bash
+
+exec s6-setuidgid redis \
+    /usr/bin/redis-server /etc/redis/redis.conf

config/root/etc/s6-overlay/s6-rc.d/redis/type

@@ -0,0 +1 @@
+longrun

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run

@@ -0,0 +1,34 @@
+#!/command/with-contenv bash
+
+# Change runtime user UID and GID
+PUID="${PUID:-911}"
+PUID="${PUID:-911}"
+groupmod -o -g "$PGID" app
+usermod -o -u "$PUID" app
+
+# Reset permissions
+chown -R app:app /run/app
+chmod -R 0700 /run/app
+chown -R app:app /config
+chmod -R 0755 /config
+chown -R root:app /app
+chmod -R 0750 /app
+chown -R app:app /app/common/static
+chmod -R 0750 /app/common/static
+chown -R app:app /app/static
+chmod -R 0750 /app/static
+find /app -type f ! -iname healthcheck.py -exec chmod 640 {} \;
+chmod 0755 /app/healthcheck.py
+
+# Optionally reset the download dir permissions
+TUBESYNC_RESET_DOWNLOAD_DIR="${TUBESYNC_RESET_DOWNLOAD_DIR:-True}"
+if [ "$TUBESYNC_RESET_DOWNLOAD_DIR" == "True" ]
+then
+    echo "TUBESYNC_RESET_DOWNLOAD_DIR=True, Resetting /downloads directory permissions"
+    chown -R app:app /downloads
+    chmod -R 0755 /downloads
+fi
+
+# Run migrations
+exec s6-setuidgid app \
+    /usr/bin/python3 /app/manage.py migrate

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/type

@@ -0,0 +1 @@
+oneshot

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/up

@@ -0,0 +1,3 @@
+#!/command/execlineb -P
+
+/etc/s6-overlay/s6-rc.d/tubesync-init/run

config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/dependencies

@@ -0,0 +1 @@
+gunicorn

config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run

@@ -1,4 +1,4 @@
-#!/usr/bin/with-contenv bash
+#!/command/with-contenv bash
 
 exec s6-setuidgid app \
     /usr/bin/python3 /app/manage.py process_tasks

config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/type

@@ -0,0 +1 @@
+longrun

config/root/etc/services.d/nginx/run

@@ -1,5 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-cd /
-
-/usr/sbin/nginx

docs/other-database-backends.md

@@ -13,18 +13,18 @@ become an issue.
 TubeSync supports SQLite (the automatic default) as well as PostgreSQL, MySQL and
 MariaDB. For MariaDB just follow the MySQL instructions as the driver is the same.
 
-You should a blank install of TubeSync. Migrating to a new database will reset your
+You should start with a blank install of TubeSync. Migrating to a new database will
-database. If you are comfortable with Django you can export and re-import existing
+reset your database. If you are comfortable with Django you can export and re-import
-database data with:
+existing database data with:
 
 ```bash
-$ docker exec -ti tubesync python3 /app/manage.py dumpdata > some-file.json
+$ docker exec -i tubesync python3 /app/manage.py dumpdata > some-file.json
 ```
 
 Then change you database backend over, then use
 
 ```bash
-$ cat some-file.json | docker exec -ti tubesync python3 /app/manage.py loaddata --format=json -
+$ cat some-file.json | docker exec -i tubesync python3 /app/manage.py loaddata --format=json -
 ```
 
 As detailed in the Django documentation:

docs/reset-metadata.md

@@ -0,0 +1,30 @@
+# TubeSync
+
+## Advanced usage guide - reset media metadata from the command line
+
+This command allows you to reset all media item metadata. You might want to use
+this if you have a lot of media items with invalid metadata and you want to
+wipe it which triggers the metadata to be redownloaded.
+
+
+## Requirements
+
+You have added some sources and media
+
+## Steps
+
+### 1. Run the reset tasks command
+
+Execute the following Django command:
+
+`./manage.py reset-metadata`
+
+When deploying TubeSync inside a container, you can execute this with:
+
+`docker exec -ti tubesync python3 /app/manage.py reset-metadata`
+
+This command will log what its doing to the terminal when you run it.
+
+When this is run, new tasks will be immediately created so all your media
+items will start downloading updated metadata straight away, any missing information
+such as thumbnails will be redownloaded, etc.

docs/using-cookies.md

@@ -0,0 +1,50 @@
+# TubeSync
+
+## Advanced usage guide - using exported cookies
+
+This is a new feature in v0.10 of TubeSync and later. It allows you to use the cookies
+file exported from your browser in "Netscape" format with TubeSync to authenticate
+to YouTube. This can bypass some throttling, age restrictions and other blocks at
+YouTube.
+
+**IMPORTANT NOTE**: Using cookies exported from your browser that is authenticated
+to YouTube identifes your Google account as using TubeSync. This may result in
+potential account impacts and is entirely at your own risk. Do not use this
+feature unless you really know what you're doing.
+
+## Requirements
+
+Have a browser that supports exporting your cookies and be logged into YouTube.
+
+## Steps
+
+### 1. Export your cookies
+
+You need to export cookies for youtube.com from your browser, you can either do
+this manually or there are plug-ins to automate this for you. This file must be
+in the "Netscape" cookie export format.
+
+Save your cookies as a `cookies.txt` file.
+
+### 2. Import into TubeSync
+
+Drop the `cookies.txt` file into your TubeSync `config` directory.
+
+If detected correctly, you will see something like this in the worker or container
+logs:
+
+```
+YYYY-MM-DD HH:MM:SS,mmm [tubesync/INFO] [youtube-dl] using cookies.txt from: /config/cookies.txt
+```
+
+If you see that line it's working correctly.
+
+If you see errors in your logs like this:
+
+```
+http.cookiejar.LoadError: '/config/cookies.txt' does not look like a Netscape format cookies file
+```
+
+Then your `cookies.txt` file was not generated or created correctly as it's not
+in the required "Netscape" format. You can fix this by exporting your `cookies.txt`
+in the correct "Netscape" format.

pip.conf

@@ -0,0 +1,2 @@
+[global]
+extra-index-url=https://www.piwheels.org/simple

tubesync/common/static/styles/_fonts.scss

@@ -1,20 +1,20 @@
 @font-face {
-    font-family: 'roboto-light';
+    font-family: 'roboto';
-    src: url('/static/fonts/roboto/roboto-light.woff') format('woff');
+    src: url('../fonts/roboto/roboto-light.woff') format('woff');
+    font-weight: lighter;
+    font-style: normal;
+}
+
+@font-face {
+    font-family: 'roboto';
+    src: url('../fonts/roboto/roboto-regular.woff') format('woff');
     font-weight: normal;
     font-style: normal;
 }
 
 @font-face {
-    font-family: 'roboto-regular';
+    font-family: 'roboto';
-    src: url('/static/fonts/roboto/roboto-regular.woff') format('woff');
+    src: url('../fonts/roboto/roboto-bold.woff') format('woff');
-    font-weight: normal;
-    font-style: normal;
-}
-
-@font-face {
-    font-family: 'roboto-bold';
-    src: url('/static/fonts/roboto/roboto-bold.woff') format('woff');
     font-weight: bold;
     font-style: normal;
 }

tubesync/common/static/styles/_variables.scss

@@ -1,2 +1,2 @@
-$font-family: 'roboto-regular', Arial, Helvetica, sans-serif;
+$font-family: 'roboto', Arial, Helvetica, sans-serif;
 $font-size: 1.05rem;

tubesync/common/static/styles/fontawesome/_icons.scss

@@ -65,6 +65,7 @@ readers do not read off random characters that represent icons */
 .#{$fa-css-prefix}-arrows-alt-h:before { content: fa-content($fa-var-arrows-alt-h); }
 .#{$fa-css-prefix}-arrows-alt-v:before { content: fa-content($fa-var-arrows-alt-v); }
 .#{$fa-css-prefix}-artstation:before { content: fa-content($fa-var-artstation); }
+.#{$fa-css-prefix}-arrow-rotate-right:before { content: fa-content($fa-var-arrow-rotate-right); }
 .#{$fa-css-prefix}-assistive-listening-systems:before { content: fa-content($fa-var-assistive-listening-systems); }
 .#{$fa-css-prefix}-asterisk:before { content: fa-content($fa-var-asterisk); }
 .#{$fa-css-prefix}-asymmetrik:before { content: fa-content($fa-var-asymmetrik); }

tubesync/common/static/styles/fontawesome/_variables.scss

@@ -80,6 +80,7 @@ $fa-var-arrow-right: \f061;
 $fa-var-arrow-up: \f062;
 $fa-var-arrows-alt: \f0b2;
 $fa-var-arrows-alt-h: \f337;
+$fa-var-arrow-rotate-right: \f01e;
 $fa-var-arrows-alt-v: \f338;
 $fa-var-artstation: \f77a;
 $fa-var-assistive-listening-systems: \f2a2;

tubesync/common/static/styles/materializecss/components/forms/_checkboxes.scss

@@ -14,7 +14,7 @@
   // Text Label Style
   + span:not(.lever) {
     position: relative;
-    padding-left: 35px;
+    padding-left: 27px;
     cursor: pointer;
     display: inline-block;
     height: 25px;

tubesync/common/static/styles/tubesync.scss

@@ -17,3 +17,16 @@ html {
     visibility: visible;
     opacity: 1;
 }
+
+.flex-collection-container {
+    display: flex !important;
+    align-items: center;
+}
+
+.flex-grow {
+    flex-grow: 1;
+}
+
+.help-text > i {
+    padding-right: 6px;
+}

tubesync/common/tests.py

@@ -2,7 +2,7 @@ import os.path
 from django.conf import settings
 from django.test import TestCase, Client
 from .testutils import prevent_request_warnings
-from .utils import parse_database_connection_string
+from .utils import parse_database_connection_string, clean_filename
 from .errors import DatabaseConnectionError
 
 
@@ -65,7 +65,7 @@ class CommonStaticTestCase(TestCase):
         self.assertTrue(os.path.exists(favicon_real_path))
 
 
-class DatabaseConnectionTestCase(TestCase):
+class UtilsTestCase(TestCase):
 
     def test_parse_database_connection_string(self):
         database_dict = parse_database_connection_string(
@@ -126,3 +126,12 @@ class DatabaseConnectionTestCase(TestCase):
         with self.assertRaises(DatabaseConnectionError):
             parse_database_connection_string(
                 'postgresql://tubesync:password@localhost:5432/tubesync/test')
+
+    def test_clean_filename(self):
+        self.assertEqual(clean_filename('a'), 'a')
+        self.assertEqual(clean_filename('a\t'), 'a')
+        self.assertEqual(clean_filename('a\n'), 'a')
+        self.assertEqual(clean_filename('a a'), 'a a')
+        self.assertEqual(clean_filename('a  a'), 'a  a')
+        self.assertEqual(clean_filename('a\t\t\ta'), 'a   a')
+        self.assertEqual(clean_filename('a\t\t\ta\t\t\t'), 'a   a')

tubesync/common/utils.py

@@ -1,3 +1,4 @@
+import string
 from datetime import datetime
 from urllib.parse import urlunsplit, urlencode, urlparse
 from yt_dlp.utils import LazyList
@@ -113,8 +114,13 @@ def clean_filename(filename):
     to_scrub = '<>\/:*?"|%'
     for char in to_scrub:
         filename = filename.replace(char, '')
-    filename = ''.join([c for c in filename if ord(c) > 30])
+    clean_filename = ''
-    return ' '.join(filename.split())
+    for c in filename:
+        if c in string.whitespace:
+            c = ' '
+        if ord(c) > 30:
+            clean_filename += c
+    return clean_filename.strip()
 
 
 def json_serial(obj):

tubesync/sync/fields.py

@@ -0,0 +1,109 @@
+from django.forms import MultipleChoiceField,  CheckboxSelectMultiple, Field, TypedMultipleChoiceField
+from django.db import models
+from typing import Any, Optional, Dict
+from django.utils.translation import gettext_lazy as _
+
+# this is a form field!
+class CustomCheckboxSelectMultiple(CheckboxSelectMultiple):
+    template_name = 'widgets/checkbox_select.html'
+    option_template_name = 'widgets/checkbox_option.html'
+
+    def get_context(self, name: str, value: Any, attrs) -> Dict[str, Any]:
+        ctx = super().get_context(name, value, attrs)['widget']
+        ctx["multipleChoiceProperties"] = []
+        for _group, options, _index in ctx["optgroups"]:
+            for option in options:
+                if not isinstance(value,str) and not isinstance(value,list) and ( option["value"] in value.selected_choices or ( value.allow_all and value.all_choice in value.selected_choices ) ):
+                    checked = True
+                else:
+                    checked = False
+
+                ctx["multipleChoiceProperties"].append({
+                    "template_name": option["template_name"],
+                    "type": option["type"],
+                    "value": option["value"],
+                    "label": option["label"],
+                    "name": option["name"],
+                    "checked": checked})
+
+        return { 'widget': ctx }
+
+# this is a database field!
+class CommaSepChoiceField(models.Field):
+    "Implements comma-separated storage of lists"
+
+    def __init__(self, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, *args, **kwargs):
+        self.separator = separator
+        self.possible_choices = possible_choices
+        self.selected_choices = []
+        self.allow_all = allow_all
+        self.all_label = all_label
+        self.all_choice = all_choice
+        super().__init__(*args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        if self.separator != ",":
+            kwargs['separator'] = self.separator
+        kwargs['possible_choices'] = self.possible_choices
+        return name, path, args, kwargs
+
+    def db_type(self, connection):
+        return 'text'
+
+    def get_my_choices(self):
+        choiceArray = []
+        if self.possible_choices is None:
+            return choiceArray
+        if self.allow_all:
+            choiceArray.append((self.all_choice, _(self.all_label)))
+
+        for t in self.possible_choices:
+            choiceArray.append(t)
+        
+        return choiceArray
+
+    def formfield(self, **kwargs):
+        # This is a fairly standard way to set up some defaults
+        # while letting the caller override them.
+        defaults = {'form_class': MultipleChoiceField, 
+                    'choices': self.get_my_choices,
+                    'widget': CustomCheckboxSelectMultiple,
+                    'label': '',
+                    'required': False}
+        defaults.update(kwargs)
+        #del defaults.required
+        return super().formfield(**defaults)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        # Only include kwarg if it's not the default
+        if self.separator != ",":
+            kwargs['separator'] = self.separator
+        return name, path, args, kwargs
+
+    def from_db_value(self, value, expr, conn):
+        if value is None:
+            self.selected_choices = []
+        else:
+            self.selected_choices = value.split(",")
+
+        return self
+
+    def get_prep_value(self, value):
+        if value is None:
+            return ""
+        if not isinstance(value,list):
+            return ""
+
+        if self.all_choice not in value:
+            return ",".join(value)
+        else:
+            return self.all_choice
+
+    def get_text_for_value(self, val):
+        fval = [i for i in self.possible_choices if i[0] == val]
+        if len(fval) <= 0:
+            return []
+        else:
+            return fval[0][1]

tubesync/sync/management/commands/reset-metadata.py

@@ -0,0 +1,19 @@
+from django.core.management.base import BaseCommand
+from sync.models import Media
+
+
+from common.logger import log
+
+
+class Command(BaseCommand):
+
+    help = 'Resets all media item metadata'
+
+    def handle(self, *args, **options):
+        log.info('Resettings all media metadata...')
+        # Delete all metadata
+        Media.objects.update(metadata=None)
+        # Trigger the save signal on each media item
+        for item in Media.objects.all():
+            item.save()
+        log.info('Done')

tubesync/sync/management/commands/youtube-dl-info.py

@@ -1,6 +1,7 @@
 import json
 from django.core.management.base import BaseCommand, CommandError
 from sync.youtube import get_media_info
+from common.utils import json_serial
 
 
 class Command(BaseCommand):
@@ -14,5 +15,6 @@ class Command(BaseCommand):
         url = options['url']
         self.stdout.write(f'Showing information for URL: {url}')
         info = get_media_info(url)
-        self.stdout.write(json.dumps(info, indent=4, sort_keys=True))
+        d = json.dumps(info, indent=4, sort_keys=True, default=json_serial)
+        self.stdout.write(d)
         self.stdout.write('Done')

tubesync/sync/matching.py

@@ -53,6 +53,8 @@ def get_best_audio_format(media):
         # If the format has a video stream, skip it
         if fmt['vcodec'] is not None:
             continue
+        if not fmt['acodec']:
+            continue
         audio_formats.append(fmt)
     audio_formats = list(reversed(sorted(audio_formats, key=lambda k: k['abr'])))
     if not audio_formats:
@@ -88,6 +90,8 @@ def get_best_video_format(media):
         # If the format has an audio stream, skip it
         if fmt['acodec'] is not None:
             continue
+        if not fmt['vcodec']:
+            continue
         if media.source.source_resolution.strip().upper() == fmt['format']:
             video_formats.append(fmt)
     # Check we matched some streams

tubesync/sync/mediaservers.py

@@ -44,7 +44,9 @@ class PlexMediaServer(MediaServer):
              '<p>The <strong>libraries</strong> is a comma-separated list of Plex '
              'library or section IDs, you can find out how to get your library or '
              'section IDs <a href="https://support.plex.tv/articles/201242707-plex-'
-             'media-scanner-via-command-line/#toc-1" target="_blank">here</a>.</p>')
+             'media-scanner-via-command-line/#toc-1" target="_blank">here</a> or '
+             '<a href="https://www.plexopedia.com/plex-media-server/api/server/libraries/" '
+             'target="_blank">here</a></p>.')
 
     def make_request(self, uri='/', params={}):
         headers = {'User-Agent': 'TubeSync'}
@@ -124,7 +126,7 @@ class PlexMediaServer(MediaServer):
         # Seems we have a valid library sections page, get the library IDs
         remote_libraries = {}
         try:
-            for parent in parsed_response.getiterator('MediaContainer'):
+            for parent in parsed_response.iter('MediaContainer'):
                 for d in parent:
                     library_id = d.attrib['key']
                     library_name = d.attrib['title']

tubesync/sync/migrations/0010_auto_20210924_0554.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.2.7 on 2021-09-24 05:54
+
+import django.core.files.storage
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0009_auto_20210218_0442'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='media',
+            name='media_file',
+            field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='index_schedule',
+            field=models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours'), (259200, 'Every 3 days'), (604800, 'Every 7 days'), (0, 'Never')], db_index=True, default=86400, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='media_format',
+            field=models.CharField(default='{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'),
+        ),
+    ]

tubesync/sync/migrations/0011_auto_20220201_1654.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.11 on 2022-02-01 16:54
+
+import django.core.files.storage
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0010_auto_20210924_0554'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='write_json',
+            field=models.BooleanField(
+                default=False, help_text='Write a JSON file with the media info, these may be detected and used by some media servers', verbose_name='write json'),
+        ),
+    ]

tubesync/sync/migrations/0012_alter_media_downloaded_format.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.12 on 2022-04-06 06:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0011_auto_20220201_1654'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='media',
+            name='downloaded_format',
+            field=models.CharField(blank=True, help_text='Video format (resolution) of the downloaded media', max_length=30, null=True, verbose_name='downloaded format'),
+        ),
+    ]

tubesync/sync/migrations/0013_fix_elative_media_file.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.2.12 on 2022-04-06 06:19
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+def fix_media_file(apps, schema_editor):
+    Media = apps.get_model('sync', 'Media')
+    for media in Media.objects.filter(downloaded=True):
+        download_dir = str(settings.DOWNLOAD_ROOT)
+
+        if media.media_file.name.startswith(download_dir):
+            media.media_file.name = media.media_file.name[len(download_dir) + 1:]
+            media.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0012_alter_media_downloaded_format'),
+    ]
+
+    operations = [
+        migrations.RunPython(fix_media_file)
+    ]

tubesync/sync/migrations/0014_alter_media_media_file.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.15 on 2022-12-28 20:33
+
+import django.core.files.storage
+from django.conf import settings
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0013_fix_elative_media_file'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='media',
+            name='media_file',
+            field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=str(settings.DOWNLOAD_ROOT)), upload_to=sync.models.get_media_file_path, verbose_name='media file'),
+        ),
+    ]

tubesync/sync/migrations/0015_auto_20230213_0603.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.17 on 2023-02-13 06:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0014_alter_media_media_file'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='media',
+            name='manual_skip',
+            field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\' be downloaded', verbose_name='manual_skip'),
+        ),
+        migrations.AlterField(
+            model_name='media',
+            name='skip',
+            field=models.BooleanField(db_index=True, default=False, help_text='INTERNAL FLAG - Media will be skipped and not downloaded', verbose_name='skip'),
+        ),
+    ]

tubesync/sync/migrations/0016_auto_20230214_2052.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.2.18 on 2023-02-14 20:52
+
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0015_auto_20230213_0603'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='embed_metadata',
+            field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='embed_thumbnail',
+            field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='enable_sponsorblock',
+            field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='sponsorblock_categories',
+            field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))),
+        ),
+    ]

tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.18 on 2023-02-20 02:23
+
+from django.db import migrations
+import sync.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0016_auto_20230214_2052'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='source',
+            name='sponsorblock_categories',
+            field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', separator=''),
+        ),
+    ]

tubesync/sync/migrations/0018_source_subtitles.py

@@ -0,0 +1,27 @@
+# Generated by pac
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0017_alter_source_sponsorblock_categories'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='write_subtitles',
+            field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='auto_subtitles',
+            field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto subtitles'),
+        ),
+        migrations.AddField(
+            model_name='source',
+            name='sub_langs',
+            field=models.CharField(default='en', help_text='List of subtitles langs to download comma-separated. Example: en,fr',max_length=30),
+        ),
+    ]

tubesync/sync/models.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from django.conf import settings
 from django.db import models
 from django.core.files.storage import FileSystemStorage
+from django.core.validators import RegexValidator
 from django.utils.text import slugify
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
@@ -19,10 +20,9 @@ from .utils import seconds_to_timestr, parse_media_format
 from .matching import (get_best_combined_format, get_best_audio_format, 
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
+from .fields import CommaSepChoiceField
 
-
+media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/')
-media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT))
-
 
 class Source(models.Model):
     '''
@@ -106,6 +106,47 @@ class Source(models.Model):
     EXTENSION_MKV = 'mkv'
     EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
 
+
+    # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
+    SPONSORBLOCK_CATEGORIES_CHOICES = (
+        ('sponsor', 'Sponsor'),
+        ('intro', 'Intermission/Intro Animation'),
+        ('outro', 'Endcards/Credits'),
+        ('selfpromo', 'Unpaid/Self Promotion'),
+        ('preview', 'Preview/Recap'),
+        ('filler', 'Filler Tangent'),
+        ('interaction', 'Interaction Reminder'),
+        ('music_offtopic', 'Non-Music Section'),
+    )
+    
+    sponsorblock_categories = CommaSepChoiceField(
+            _(''),
+            possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
+            all_choice="all",
+            allow_all=True,
+            all_label="(all options)",
+            default="all",
+            help_text=_("Select the sponsorblocks you want to enforce")
+        )
+
+    embed_metadata = models.BooleanField(
+        _('embed metadata'),
+        default=False,
+        help_text=_('Embed metadata from source into file')
+    )
+    embed_thumbnail = models.BooleanField(
+        _('embed thumbnail'),
+        default=False,
+        help_text=_('Embed thumbnail into the file')
+    )
+
+    enable_sponsorblock = models.BooleanField(
+        _('enable sponsorblock'),
+        default=True,
+        help_text=_('Use SponsorBlock?')
+    )
+
+
     # Fontawesome icons used for the source on the front end
     ICONS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
@@ -298,12 +339,41 @@ class Source(models.Model):
         default=False,
         help_text=_('Write an NFO file in XML with the media info, these may be detected and used by some media servers')
     )
+    write_json = models.BooleanField(
+        _('write json'),
+        default=False,
+        help_text=_('Write a JSON file with the media info, these may be detected and used by some media servers')
+    )
     has_failed = models.BooleanField(
         _('has failed'),
         default=False,
         help_text=_('Source has failed to index media')
     )
 
+    write_subtitles = models.BooleanField(
+        _('write subtitles'),
+        default=False,
+        help_text=_('Download video subtitles')
+    )
+
+    auto_subtitles = models.BooleanField(
+        _('accept auto-generated subs'),
+        default=False,
+        help_text=_('Accept auto-generated subtitles')
+    )
+    sub_langs = models.CharField(
+        _('subs langs'),
+        max_length=30,
+        default='en',
+        help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'),
+        validators=[
+            RegexValidator(
+                regex=r"^(\-?[\_\.a-zA-Z]+,)*(\-?[\_\.a-zA-Z]+){1}$",
+                message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat')
+            )
+        ]
+    )
+
     def __str__(self):
         return self.name
 
@@ -387,10 +457,14 @@ class Source(models.Model):
     @property
     def directory_path(self):
         download_dir = Path(media_file_storage.location)
+        return download_dir / self.type_directory_path
+
+    @property
+    def type_directory_path(self):
         if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO:
-            return download_dir / settings.DOWNLOAD_AUDIO_DIR / self.directory
+            return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory
         else:
-            return download_dir / settings.DOWNLOAD_VIDEO_DIR / self.directory
+            return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory
 
     def make_directory(self):
         return os.makedirs(self.directory_path, exist_ok=True)
@@ -428,12 +502,13 @@ class Source(models.Model):
             fmt.append('60fps')
         if self.prefer_hdr:
             fmt.append('hdr')
+        now = timezone.now()
         return {
-            'yyyymmdd': timezone.now().strftime('%Y%m%d'),
+            'yyyymmdd': now.strftime('%Y%m%d'),
-            'yyyy_mm_dd': timezone.now().strftime('%Y-%m-%d'),
+            'yyyy_mm_dd': now.strftime('%Y-%m-%d'),
-            'yyyy': timezone.now().strftime('%Y'),
+            'yyyy': now.strftime('%Y'),
-            'mm': timezone.now().strftime('%m'),
+            'mm': now.strftime('%m'),
-            'dd': timezone.now().strftime('%d'),
+            'dd': now.strftime('%d'),
             'source': self.slugname,
             'source_full': self.name,
             'title': 'some-media-title-name',
@@ -467,7 +542,11 @@ class Source(models.Model):
         response = indexer(self.index_url)
         if not isinstance(response, dict):
             return []
-        return response.get('entries', [])
+        entries = response.get('entries', [])
+
+        if settings.MAX_ENTRIES_PROCESSING:
+            entries = entries[:settings.MAX_ENTRIES_PROCESSING]
+        return entries
 
 
 def get_media_thumb_path(instance, filename):
@@ -655,7 +734,7 @@ class Media(models.Model):
     media_file = models.FileField(
         _('media file'),
         upload_to=get_media_file_path,
-        max_length=200,
+        max_length=255,
         blank=True,
         null=True,
         storage=media_file_storage,
@@ -665,7 +744,13 @@ class Media(models.Model):
         _('skip'),
         db_index=True,
         default=False,
-        help_text=_('Media will be skipped and not downloaded')
+        help_text=_('INTERNAL FLAG - Media will be skipped and not downloaded')
+    )
+    manual_skip = models.BooleanField(
+        _('manual_skip'),
+        db_index=True,
+        default=False,
+        help_text=_('Media marked as "skipped", won\' be downloaded')
     )
     downloaded = models.BooleanField(
         _('downloaded'),
@@ -685,7 +770,7 @@ class Media(models.Model):
         max_length=30,
         blank=True,
         null=True,
-        help_text=_('Audio codec of the downloaded media')
+        help_text=_('Video format (resolution) of the downloaded media')
     )
     downloaded_height = models.PositiveIntegerField(
         _('downloaded height'),
@@ -826,7 +911,10 @@ class Media(models.Model):
                     'hdr': hdr,
                     'format': tuple(fmt),
                 }
-            resolution = f'{self.downloaded_height}p'
+            if self.downloaded_format:
+                resolution = self.downloaded_format.lower()
+            elif self.downloaded_height:
+                resolution = f'{self.downloaded_height}p'
             if self.downloaded_format != 'audio':
                 vcodec = self.downloaded_video_codec.lower()
                 fmt.append(vcodec)
@@ -853,7 +941,7 @@ class Media(models.Model):
         # Otherwise, calculate from matched format codes
         vformat = None
         aformat = None
-        if '+' in format_str:
+        if format_str and '+' in format_str:
             # Seperate audio and video streams
             vformat_code, aformat_code = format_str.split('+')
             vformat = self.get_format_by_code(vformat_code)
@@ -862,7 +950,7 @@ class Media(models.Model):
             # Combined stream or audio only
             cformat = self.get_format_by_code(format_str)
             aformat = cformat
-            if cformat['vcodec']:
+            if cformat and cformat['vcodec']:
                 # Combined
                 vformat = cformat
         if vformat:
@@ -993,7 +1081,12 @@ class Media(models.Model):
     @property
     def duration(self):
         field = self.get_metadata_field('duration')
-        return int(self.loaded_metadata.get(field, 0))
+        duration = self.loaded_metadata.get(field, 0)
+        try:
+            duration = int(duration)
+        except (TypeError, ValueError):
+            duration = 0
+        return duration
 
     @property
     def duration_formatted(self):
@@ -1053,7 +1146,10 @@ class Media(models.Model):
 
     @property
     def thumbname(self):
-        filename = self.filename
+        if self.downloaded and self.media_file:
+            filename = os.path.basename(self.media_file.path)
+        else:
+            filename = self.filename
         prefix, ext = os.path.splitext(filename)
         return f'{prefix}.jpg'
 
@@ -1063,7 +1159,10 @@ class Media(models.Model):
 
     @property
     def nfoname(self):
-        filename = self.filename
+        if self.downloaded and self.media_file:
+            filename = os.path.basename(self.media_file.path)
+        else:
+            filename = self.filename
         prefix, ext = os.path.splitext(filename)
         return f'{prefix}.nfo'
     
@@ -1071,6 +1170,19 @@ class Media(models.Model):
     def nfopath(self):
         return self.source.directory_path / self.nfoname
 
+    @property
+    def jsonname(self):
+        if self.downloaded and self.media_file:
+            filename = os.path.basename(self.media_file.path)
+        else:
+            filename = self.filename
+        prefix, ext = os.path.splitext(filename)
+        return f'{prefix}.info.json'
+    
+    @property
+    def jsonpath(self):
+        return self.source.directory_path / self.jsonname
+
     @property
     def directory_path(self):
         # Otherwise, create a suitable filename from the source media_format
@@ -1095,6 +1207,31 @@ class Media(models.Model):
             return False
         return os.path.exists(self.media_file.path)
 
+    @property
+    def content_type(self):
+        if not self.downloaded:
+            return 'video/mp4'
+        vcodec = self.downloaded_video_codec
+        if vcodec is None:
+            acodec = self.downloaded_audio_codec
+            if acodec is None:
+                raise TypeError() # nothing here.
+            
+            acodec = acodec.lower()
+            if acodec == "mp4a":
+                return "audio/mp4"
+            elif acodec == "opus":
+                return "audio/opus"
+            else:
+                # fall-fall-back.
+                return 'audio/ogg'
+
+        vcodec = vcodec.lower()
+        if vcodec == 'vp9':
+            return 'video/webm'
+        else:
+            return 'video/mp4'
+
     @property
     def nfoxml(self):
         '''
@@ -1219,7 +1356,10 @@ class Media(models.Model):
                                     f'no valid format available')
         # Download the media with youtube-dl
         download_youtube_media(self.url, format_str, self.source.extension,
-                               str(self.filepath))
+                               str(self.filepath), self.source.write_json, 
+                               self.source.sponsorblock_categories, self.source.embed_thumbnail,
+                               self.source.embed_metadata, self.source.enable_sponsorblock,
+                              self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs )
         # Return the download paramaters
         return format_str, self.source.extension
 
@@ -1229,7 +1369,7 @@ class Media(models.Model):
         '''
         indexer = self.INDEXERS.get(self.source.source_type, None)
         if not callable(indexer):
-            raise Exception(f'Meida with source type f"{self.source.source_type}" '
+            raise Exception(f'Media with source type f"{self.source.source_type}" '
                             f'has no indexer')
         return indexer(self.url)
 

tubesync/sync/signals.py

@@ -93,6 +93,10 @@ def task_task_failed(sender, task_id, completed_task, **kwargs):
 
 @receiver(post_save, sender=Media)
 def media_post_save(sender, instance, created, **kwargs):
+    # If the media is skipped manually, bail.
+    if instance.manual_skip:
+        return
+
     # Triggered after media is saved
     cap_changed = False
     can_download_changed = False

tubesync/sync/tasks.py

@@ -132,16 +132,14 @@ def cleanup_completed_tasks():
 
 
 def cleanup_old_media():
-    for media in Media.objects.filter(download_date__isnull=False):
+    for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0):
-        if media.source.delete_old_media and media.source.days_to_keep > 0:
+        delta = timezone.now() - timedelta(days=source.days_to_keep)
-            delta = timezone.now() - timedelta(days=media.source.days_to_keep)
+        for media in source.media_source.filter(downloaded=True, download_date__lt=delta):
-            if media.downloaded and media.download_date < delta:
+            log.info(f'Deleting expired media: {source} / {media} '
-                # Media was downloaded after the cutoff date, delete it
+                     f'(now older than {source.days_to_keep} days / '
-                log.info(f'Deleting expired media: {media.source} / {media} '
+                     f'download_date before {delta})')
-                         f'(now older than {media.source.days_to_keep} days / '
+            # .delete() also triggers a pre_delete signal that removes the files
-                         f'download_date before {delta})')
+            media.delete()
-                # .delete() also triggers a pre_delete signal that removes the files
-                media.delete()
 
 
 @background(schedule=0)
@@ -153,7 +151,6 @@ def index_source_task(source_id):
         source = Source.objects.get(pk=source_id)
     except Source.DoesNotExist:
         # Task triggered but the Source has been deleted, delete the task
-        delete_index_source_task(source_id)
         return
     # Reset any errors
     source.has_failed = False
@@ -202,7 +199,6 @@ def check_source_directory_exists(source_id):
         source = Source.objects.get(pk=source_id)
     except Source.DoesNotExist:
         # Task triggered but the Source has been deleted, delete the task
-        delete_index_source_task(source_id)
         return
     # Check the source output directory exists
     if not source.directory_exists():
@@ -223,6 +219,11 @@ def download_media_metadata(media_id):
         log.error(f'Task download_media_metadata(pk={media_id}) called but no '
                   f'media exists with ID: {media_id}')
         return
+    
+    if media.manual_skip:
+        log.info(f'Task for ID: {media_id} skipped, due to task being manually skipped.')
+        return
+
     source = media.source
     metadata = media.index_metadata()
     media.metadata = json.dumps(metadata, default=json_serial)
@@ -235,7 +236,7 @@ def download_media_metadata(media_id):
         media.skip = True
     # If the source has a download cap date check the upload date is allowed
     max_cap_age = source.download_cap_date
-    if max_cap_age:
+    if media.published and max_cap_age:
         if media.published < max_cap_age:
             # Media was published after the cap date, skip it
             log.warn(f'Media: {source} / {media} is older than cap age '
@@ -341,7 +342,7 @@ def download_media(media_id):
         log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: '
                  f'"{filepath}"')
         # Link the media file to the object and update info about the download
-        media.media_file.name = str(filepath)
+        media.media_file.name = str(media.source.type_directory_path / media.filename)
         media.downloaded = True
         media.download_date = timezone.now()
         media.downloaded_filesize = os.path.getsize(filepath)

tubesync/sync/templates/sync/_mediaformatvars.html

@@ -25,12 +25,12 @@
     </tr>
     <tr>
       <td>{mm}</td>
-      <td>Media publish year in MM</td>
+      <td>Media publish month in MM</td>
       <td>01</td>
     </tr>
     <tr>
       <td>{dd}</td>
-      <td>Media publish year in DD</td>
+      <td>Media publish day in DD</td>
       <td>31</td>
     </tr>
     <tr>

tubesync/sync/templates/sync/dashboard.html

@@ -101,7 +101,7 @@
 </div>
 <div class="row">
   <div class="col s12">
-    <h2 class="truncate">Runtime infomation</h2>
+    <h2 class="truncate">Runtime information</h2>
   </div>
 </div>
 <div class="row">

tubesync/sync/templates/sync/media-item.html

@@ -9,10 +9,24 @@
     {% if media.title %}<h2 class="truncate"><strong>{{ media.title }}</strong></h2>{% endif %}
     <p class="truncate"><strong><a href="{{ media.url }}" target="_blank"><i class="fas fa-link"></i> {{ media.url }}</a></strong></p>
     <p class="truncate">Downloading to: <strong>{{ media.source.directory_path }}</strong></p>
+    {% if download_state == 'downloaded' %}
+    {% if media.source.is_audio %}
+    <audio controls src="{% url 'sync:media-content' pk=media.pk %}"></audio>
+    {% else %}
+    <video controls style="width: 100%">
+      <source src="{% url 'sync:media-content' pk=media.pk %}">
+    </video>
+    {% endif %}
+
+    <p class="truncate"><a href="{% url 'sync:media-content' pk=media.pk %}" download="{{ media.filename }}"><strong><i class="fas fa-download"></i> Download</strong></a></p>
+    {% endif %}
   </div>
 </div>
-{% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
+{% if media.manual_skip %}{% include 'errorbox.html' with message='Media is marked to be skipped and will not be downloaded.' %}
-{% if media.skip %}{% include 'errorbox.html' with message='Media is marked to be skipped and will not be downloaded.' %}{% endif %}
+{% else %}
+  {% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
+  {% if media.skip %}{% include 'errorbox.html' with message='This media may be skipped due to error(s).' %}{% endif %}
+{% endif %}
 {% include 'infobox.html' with message=message %}
 <div class="row">
   <div class="col s12 m7">
@@ -156,10 +170,10 @@
 {% else %}
 <div class="row">
   <div class="col s12">
-    {% if media.skip %}
+    {% if media.manual_skip %}
-      <a href="{% url 'sync:enable-media' pk=media.pk %}" class="btn">Enable (unskip) media <i class="fas fa-cloud-download-alt"></i></a>
+      <a href="{% url 'sync:enable-media' pk=media.pk %}" class="btn">Unskip media (manually) <i class="fas fa-cloud-download-alt"></i></a>
     {% else %}
-      <a href="{% url 'sync:skip-media' pk=media.pk %}" class="btn delete-button">Skip media <i class="fas fa-times-circle"></i></a>
+      <a href="{% url 'sync:skip-media' pk=media.pk %}" class="btn delete-button">Manually mark media to be skipped <i class="fas fa-times-circle"></i></a>
     {% endif %}
   </div>
 </div>

tubesync/sync/templates/sync/media.html

@@ -4,7 +4,7 @@
 
 {% block content %}
 <div class="row">
-  <div class="col s12 m9">
+  <div class="col s12 m6">
     <h1 class="truncate">Media</h1>
   </div>
   <div class="col s12 m3">
@@ -14,6 +14,13 @@
     <a href="{% url 'sync:media' %}?show_skipped=yes{% if source %}&filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye"></i> Show skipped media</a>
     {% endif %}
   </div>
+  <div class="col s12 m3">
+    {% if only_skipped %}
+    <a href="{% url 'sync:media' %}{% if source %}?filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye-slash"></i> Only skipped media</a>
+    {% else %}
+    <a href="{% url 'sync:media' %}?only_skipped=yes{% if source %}&filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye"></i> Only skipped media</a>
+    {% endif %}
+  </div>
 </div>
 {% include 'infobox.html' with message=message %}
 <div class="row no-margin-bottom">
@@ -29,8 +36,10 @@
             {% if m.downloaded %}
               <i class="fas fa-check-circle" title="Downloaded"></i> {{ m.download_date|date:'Y-m-d' }}
             {% else %}
-              {% if m.skip %}
+              {% if m.manual_skip %}
-              <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped</span>
+              <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Manually skipped</span>
+              {% elif m.skip %}
+              <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped by system</span>
               {% elif not m.source.download_media %}
               <span class="error-text"><i class="fas fa-times" title="Not downloading media for this source"></i> Disabled at source</span>
               {% elif not m.has_metadata %}

tubesync/sync/templates/sync/source.html

@@ -111,6 +111,10 @@
         <td class="hide-on-small-only">Write NFO?</td>
         <td><span class="hide-on-med-and-up">Write NFO?<br></span><strong>{% if source.write_nfo %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
+      <tr title="Should a JSON file be written with the media?">
+        <td class="hide-on-small-only">Write JSON?</td>
+        <td><span class="hide-on-med-and-up">Write JSON?<br></span><strong>{% if source.write_json %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
       {% if source.delete_old_media and source.days_to_keep > 0 %}
       <tr title="Days after which your media from this source will be locally deleted">
         <td class="hide-on-small-only">Delete old media</td>
@@ -126,6 +130,55 @@
         <td class="hide-on-small-only">UUID</td>
         <td><span class="hide-on-med-and-up">UUID<br></span><strong>{{ source.uuid }}</strong></td>
       </tr>
+      <tr title="{{ _('Embedding thumbnail?') }}">
+        <td class="hide-on-small-only">{{ _("Embed thumbnail?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Embed thumbnail?") }}<br></span><strong><i class="fas {% if source.embed_thumbnail %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+      <tr title="{{ _('Embedding metadata?') }}">
+        <td class="hide-on-small-only">{{ _("Embed metadata?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Embed metadata?") }}<br></span><strong><i class="fas {% if source.embed_metadata %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+
+      <tr title="{{ _('Is sponsorblock enabled?') }}">
+        <td class="hide-on-small-only">{{ _("SponsorBlock?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Sponsorblock enabled?") }}<br></span><strong><i class="fas {% if source.enable_sponsorblock %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+
+      {% if source.enable_sponsorblock %}
+      <tr title="{{ _('SponsorBlock: What to block?') }}">
+        <td class="hide-on-small-only">{{ _("What blocked?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("What blocked?") }}<br></span><strong>
+          {% if source.sponsorblock_categories.all_choice in source.sponsorblock_categories.selected_choices %}
+            {% for k,v in source.sponsorblock_categories.possible_choices %}
+              {{ v }}: <i class="fas fa-check"></i><BR>
+            {% endfor %}
+          {% else %}
+            {% for c in source.sponsorblock_categories.selected_choices %} 
+              {% for k,v in source.sponsorblock_categories.possible_choices %}
+                {% if k == c %} {{ v }}: <i class="fas fa-check"></i><BR>{% endif %}
+              {% endfor %}
+            {% endfor %}
+          {% endif %}
+        </strong></td>
+      </tr>
+      {% endif %}
+
+      <tr title="{{ _('Are Subtitles downloaded?') }}">
+        <td class="hide-on-small-only">{{ _("Download subtitles?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Download subtitles?") }}:</span><strong><i class="fas {% if source.write_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+
+      {% if source.write_subtitles %}
+      <tr title="{{ _('Are auto subs accepted?') }}">
+        <td class="hide-on-small-only">{{ _("Auto-generated subtitles?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Auto-generated subtitles?") }}:</span><strong><i class="fas {% if source.auto_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
+      </tr>
+      <tr title="{{ _('Subs langs?') }}">
+        <td class="hide-on-small-only">{{ _("Subs langs?") }}:</td>
+        <td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
+      </tr>
+      {% endif %}
+      
     </table>
   </div>
 </div>

tubesync/sync/templates/sync/sources.html

@@ -24,15 +24,18 @@
   <div class="col s12">
     <div class="collection">
     {% for source in sources %}
-      <a href="{% url 'sync:source' pk=source.pk %}" class="collection-item">
+      <span class="collection-item flex-collection-container">
-        {{ source.icon|safe }} <strong>{{ source.name }}</strong> ({{ source.get_source_type_display }} &quot;{{ source.key }}&quot;)<br>
+        <a href="{% url 'sync:source' pk=source.pk %}" class="flex-grow">
-        {{ source.format_summary }}<br>
+          {{ source.icon|safe }} <strong>{{ source.name }}</strong> ({{ source.get_source_type_display }} &quot;{{ source.key }}&quot;)<br>
-        {% if source.has_failed %}
+          {{ source.format_summary }}<br>
-        <span class="error-text"><i class="fas fa-exclamation-triangle"></i> <strong>Source has permanent failures</strong></span>
+          {% if source.has_failed %}
-        {% else %}
+          <span class="error-text"><i class="fas fa-exclamation-triangle"></i> <strong>Source has permanent failures</strong></span>
-        <strong>{{ source.media_count }}</strong> media items, <strong>{{ source.downloaded_count }}</strong> downloaded{% if source.delete_old_media and source.days_to_keep > 0 %}, keeping {{ source.days_to_keep }} days of media{% endif %}
+          {% else %}
-        {% endif %}
+          <strong>{{ source.media_count }}</strong> media items, <strong>{{ source.downloaded_count }}</strong> downloaded{% if source.delete_old_media and source.days_to_keep > 0 %}, keeping {{ source.days_to_keep }} days of media{% endif %}
-      </a>
+          {% endif %}
+        </a>
+        <a href="{% url 'sync:source-sync-now' pk=source.pk %}" class="collection-item"><i class="fas fa-arrow-rotate-right"></i></a>
+      </span>
     {% empty %}
       <span class="collection-item no-items"><i class="fas fa-info-circle"></i> You haven't added any sources.</span>
     {% endfor %}

tubesync/sync/templates/sync/tasks.html

@@ -66,7 +66,7 @@
       {% for task in scheduled %}
         <a href="{% url task.url pk=task.instance.pk %}" class="collection-item">
           <i class="fas fa-stopwatch"></i> <strong>{{ task }}</strong><br>
-          {% if task.instance.index_schedule %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
+          {% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
           <i class="fas fa-redo"></i> Task will run {% if task.run_now %}<strong>immediately</strong>{% else %}at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>{% endif %}
         </a>
       {% empty %}

tubesync/sync/templates/widgets/checkbox_option.html

@@ -0,0 +1,7 @@
+<!--<input type="{{ option.type }}" name="{{ option.name }}" value="{{ option.value }}" id="{{ option.value }}"><BR>
+    <label for="{{ option.value }}">{{option.label}}</label>-->
+
+<label>
+    <input type="{{ option.type }}" name="{{ option.name }}" value="{{ option.value }}" id="{{ option.value }}" {% if option.checked %}checked{% endif %}>
+    <span>{{option.label}}</span>
+</label>

tubesync/sync/templates/widgets/checkbox_select.html

@@ -0,0 +1,5 @@
+</label>
+{% for option in widget.multipleChoiceProperties %}
+    {% include option.template_name with option=option %}
+{% endfor %}
+<label>

tubesync/sync/tests.py

@@ -6,7 +6,7 @@
 
 
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
 from urllib.parse import urlsplit
 from xml.etree import ElementTree
 from django.conf import settings
@@ -14,6 +14,7 @@ from django.test import TestCase, Client
 from django.utils import timezone
 from background_task.models import Task
 from .models import Source, Media
+from .tasks import cleanup_old_media
 
 
 class FrontEndTestCase(TestCase):
@@ -36,6 +37,9 @@ class FrontEndTestCase(TestCase):
         test_sources = {
             'youtube-channel': {
                 'valid': (
+                    'https://m.youtube.com/testchannel',
+                    'https://m.youtube.com/c/testchannel',
+                    'https://m.youtube.com/c/testchannel/videos',
                     'https://www.youtube.com/testchannel',
                     'https://www.youtube.com/c/testchannel',
                     'https://www.youtube.com/c/testchannel/videos',
@@ -47,6 +51,7 @@ class FrontEndTestCase(TestCase):
                 'invalid_domain': (
                     'https://www.test.com/c/testchannel',
                     'https://www.example.com/c/testchannel',
+                    'https://n.youtube.com/c/testchannel',
                 ),
                 'invalid_path': (
                     'https://www.youtube.com/test/invalid',
@@ -62,6 +67,8 @@ class FrontEndTestCase(TestCase):
             },
             'youtube-channel-id': {
                 'valid': (
+                    'https://m.youtube.com/channel/channelid',
+                    'https://m.youtube.com/channel/channelid/videos',
                     'https://www.youtube.com/channel/channelid',
                     'https://www.youtube.com/channel/channelid/videos',
                 ),
@@ -72,6 +79,7 @@ class FrontEndTestCase(TestCase):
                 'invalid_domain': (
                     'https://www.test.com/channel/channelid',
                     'https://www.example.com/channel/channelid',
+                    'https://n.youtube.com/channel/channelid',
                 ),
                 'invalid_path': (
                     'https://www.youtube.com/test/invalid',
@@ -83,6 +91,8 @@ class FrontEndTestCase(TestCase):
             },
             'youtube-playlist': {
                 'valid': (
+                    'https://m.youtube.com/playlist?list=testplaylist',
+                    'https://m.youtube.com/watch?v=testvideo&list=testplaylist',
                     'https://www.youtube.com/playlist?list=testplaylist',
                     'https://www.youtube.com/watch?v=testvideo&list=testplaylist',
                 ),
@@ -93,6 +103,7 @@ class FrontEndTestCase(TestCase):
                 'invalid_domain': (
                     'https://www.test.com/playlist?list=testplaylist',
                     'https://www.example.com/playlist?list=testplaylist',
+                    'https://n.youtube.com/playlist?list=testplaylist',
                 ),
                 'invalid_path': (
                     'https://www.youtube.com/notplaylist?list=testplaylist',
@@ -172,7 +183,8 @@ class FrontEndTestCase(TestCase):
             'source_acodec': 'OPUS',
             'prefer_60fps': False,
             'prefer_hdr': False,
-            'fallback': 'f'
+            'fallback': 'f',
+            'sub_langs': 'en',
         }
         response = c.post('/source-add', data)
         self.assertEqual(response.status_code, 302)
@@ -213,7 +225,8 @@ class FrontEndTestCase(TestCase):
             'source_acodec': Source.SOURCE_ACODEC_OPUS,
             'prefer_60fps': False,
             'prefer_hdr': False,
-            'fallback': Source.FALLBACK_FAIL
+            'fallback': Source.FALLBACK_FAIL,
+            'sub_langs': 'en',
         }
         response = c.post(f'/source-update/{source_uuid}', data)
         self.assertEqual(response.status_code, 302)
@@ -242,7 +255,8 @@ class FrontEndTestCase(TestCase):
             'source_acodec': Source.SOURCE_ACODEC_OPUS,
             'prefer_60fps': False,
             'prefer_hdr': False,
-            'fallback': Source.FALLBACK_FAIL
+            'fallback': Source.FALLBACK_FAIL,
+            'sub_langs': 'en',
         }
         response = c.post(f'/source-update/{source_uuid}', data)
         self.assertEqual(response.status_code, 302)
@@ -454,11 +468,14 @@ metadata_60fps_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_60
 metadata_60fps = open(metadata_60fps_filepath, 'rt').read()
 metadata_60fps_hdr_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_60fps_hdr.json'
 metadata_60fps_hdr = open(metadata_60fps_hdr_filepath, 'rt').read()
+metadata_20230629_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_2023-06-29.json'
+metadata_20230629 = open(metadata_20230629_filepath, 'rt').read()
 all_test_metadata = {
     'boring': metadata,
     'hdr': metadata_hdr,
     '60fps': metadata_60fps,
     '60fps+hdr': metadata_60fps_hdr,
+    '20230629': metadata_20230629,
 }
 
 
@@ -491,7 +508,7 @@ class FilepathTestCase(TestCase):
             metadata=metadata,
         )
 
-    def test_source_dirname(self):
+    def test_source_media_format(self):
         # Check media format validation is working
         # Empty
         self.source.media_format = ''
@@ -1387,3 +1404,95 @@ class FormatMatchingTestCase(TestCase):
             match_type, format_code = self.media.get_best_video_format()
             self.assertEqual(format_code, expected_format_code)
             self.assertEqual(match_type, expeceted_match_type)
+
+    def test_metadata_20230629(self):
+        self.media.metadata = all_test_metadata['20230629']
+        expected_matches = {
+            # (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
+            ('360p', 'AVC1', False, True): (False, '134'),             # Fallback match, no hdr
+            ('360p', 'AVC1', True, False): (False, '134'),             # Fallback match, no 60fps
+            ('360p', 'AVC1', True, True): (False, '332'),              # Fallback match, 60fps+hdr, switched to VP9
+            ('360p', 'VP9', False, False): (True, '243'),              # Exact match
+            ('360p', 'VP9', False, True): (True, '332'),               # Exact match, hdr
+            ('360p', 'VP9', True, False): (False, '332'),              # Fallback match, 60fps, extra hdr
+            ('360p', 'VP9', True, True): (True, '332'),                # Exact match, 60fps+hdr
+            ('480p', 'AVC1', False, False): (True, '135'),             # Exact match
+            ('480p', 'AVC1', False, True): (False, '135'),             # Fallback match, no hdr
+            ('480p', 'AVC1', True, False): (False, '135'),             # Fallback match, no 60fps
+            ('480p', 'AVC1', True, True): (False, '333'),              # Fallback match, 60fps+hdr, switched to VP9
+            ('480p', 'VP9', False, False): (True, '244'),              # Exact match
+            ('480p', 'VP9', False, True): (True, '333'),               # Exact match, hdr
+            ('480p', 'VP9', True, False): (False, '333'),              # Fallback match, 60fps, extra hdr
+            ('480p', 'VP9', True, True): (True, '333'),                # Exact match, 60fps+hdr
+            ('720p', 'AVC1', False, False): (True, '136'),             # Exact match
+            ('720p', 'AVC1', False, True): (False, '136'),             # Fallback match, no hdr
+            ('720p', 'AVC1', True, False): (True, '298'),              # Exact match, 60fps
+            ('720p', 'AVC1', True, True): (False, '334'),              # Fallback match, 60fps+hdr, switched to VP9
+            ('720p', 'VP9', False, False): (True, '247'),              # Exact match
+            ('720p', 'VP9', False, True): (True, '334'),               # Exact match, hdr
+            ('720p', 'VP9', True, False): (True, '302'),               # Exact match, 60fps
+            ('720p', 'VP9', True, True): (True, '334'),                # Exact match, 60fps+hdr
+            ('1440p', 'AVC1', False, False): (False, '308'),           # Fallback match, 60fps, switched to VP9 (no 1440p AVC1)
+            ('1440p', 'AVC1', False, True): (False, '336'),            # Fallback match, 60fps+hdr, switched to VP9 (no 1440p AVC1)
+            ('1440p', 'AVC1', True, False): (False, '308'),            # Fallback match, 60fps, switched to VP9 (no 1440p AVC1)
+            ('1440p', 'AVC1', True, True): (False, '336'),             # Fallback match, 60fps+hdr, switched to VP9 (no 1440p AVC1)
+            ('1440p', 'VP9', False, False): (False, '308'),            # Fallback, 60fps
+            ('1440p', 'VP9', False, True): (True, '336'),              # Exact match, hdr
+            ('1440p', 'VP9', True, False): (True, '308'),              # Exact match, 60fps
+            ('1440p', 'VP9', True, True): (True, '336'),               # Exact match, 60fps+hdr
+            ('2160p', 'AVC1', False, False): (False, '315'),           # Fallback, 60fps, switched to VP9 (no 2160p AVC1)
+            ('2160p', 'AVC1', False, True): (False, '337'),            # Fallback match, 60fps+hdr, switched to VP9 (no 2160p AVC1)
+            ('2160p', 'AVC1', True, False): (False, '315'),            # Fallback, switched to VP9 (no 2160p AVC1)
+            ('2160p', 'AVC1', True, True): (False, '337'),             # Fallback match, 60fps+hdr, switched to VP9 (no 2160p AVC1)
+            ('2160p', 'VP9', False, False): (False, '315'),            # Fallback, 60fps
+            ('2160p', 'VP9', False, True): (True, '337'),              # Exact match, hdr
+            ('2160p', 'VP9', True, False): (True, '315'),              # Exact match, 60fps
+            ('2160p', 'VP9', True, True): (True, '337'),               # Exact match, 60fps+hdr
+            ('4320p', 'AVC1', False, False): (False, '272'),           # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'AVC1', False, True): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'AVC1', True, False): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'AVC1', True, True): (False, '272'),             # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320p', 'VP9', False, False): (False, '272'),            # Fallback, 60fps (no other 8k streams)
+            ('4320p', 'VP9', False, True): (False, '272'),             # Fallback, 60fps (no other 8k streams)
+            ('4320p', 'VP9', True, False): (True, '272'),              # Exact match, 60fps
+            ('4320p', 'VP9', True, True): (False, '272'),              # Fallback, 60fps (no other 8k streams)
+        }
+        for params, expected in expected_matches.items():
+            resolution, vcodec, prefer_60fps, prefer_hdr = params
+            expeceted_match_type, expected_format_code = expected
+            self.source.source_resolution = resolution
+            self.source.source_vcodec = vcodec
+            self.source.prefer_60fps = prefer_60fps
+            self.source.prefer_hdr = prefer_hdr
+            # The aim here is to execute the matching code to find error paths, specific testing isn't required
+            self.media.get_best_video_format()
+            self.media.get_best_audio_format()
+
+
+class TasksTestCase(TestCase):
+    def setUp(self):
+        # Disable general logging for test case
+        logging.disable(logging.CRITICAL)
+
+    def test_delete_old_media(self):
+        src1 = Source.objects.create(key='aaa', name='aaa', directory='/tmp/a', delete_old_media=False, days_to_keep=14)
+        src2 = Source.objects.create(key='bbb', name='bbb', directory='/tmp/b', delete_old_media=True, days_to_keep=14)
+
+        now = timezone.now()
+
+        m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5))
+        m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25))
+        m13 = Media.objects.create(source=src1, downloaded=False, key='a13')
+
+        m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5))
+        m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25))
+        m23 = Media.objects.create(source=src2, downloaded=False, key='a23')
+
+        self.assertEquals(src1.media_source.all().count(), 3)
+        self.assertEquals(src2.media_source.all().count(), 3)
+
+        cleanup_old_media()
+
+        self.assertEquals(src1.media_source.all().count(), 3)
+        self.assertEquals(src2.media_source.all().count(), 2)
+        self.assertEquals(Media.objects.filter(pk=m22.pk).exists(), False)

tubesync/sync/urls.py

@@ -2,7 +2,7 @@ from django.urls import path
 from .views import (DashboardView, SourcesView, ValidateSourceView, AddSourceView,
                     SourceView, UpdateSourceView, DeleteSourceView, MediaView,
                     MediaThumbView, MediaItemView, MediaRedownloadView, MediaSkipView,
-                    MediaEnableView, TasksView, CompletedTasksView, ResetTasks,
+                    MediaEnableView, MediaContent, TasksView, CompletedTasksView, ResetTasks,
                     MediaServersView, AddMediaServerView, MediaServerView,
                     DeleteMediaServerView, UpdateMediaServerView)
 
@@ -28,6 +28,10 @@ urlpatterns = [
          ValidateSourceView.as_view(),
          name='validate-source'),
 
+    path('source-sync-now/<uuid:pk>',
+         SourcesView.as_view(),
+         name='source-sync-now'),
+
     path('source-add',
          AddSourceView.as_view(),
          name='add-source'),
@@ -70,6 +74,10 @@ urlpatterns = [
          MediaEnableView.as_view(),
          name='enable-media'),
 
+    path('media-content/<uuid:pk>',
+         MediaContent.as_view(),
+         name='media-content'),
+
     # Task URLs
 
     path('tasks',

tubesync/sync/utils.py

@@ -14,9 +14,9 @@ def validate_url(url, validator):
         Validate a URL against a dict of validation requirements. Returns an extracted
         part of the URL if the URL is valid, if invalid raises a ValidationError.
     '''
-    valid_scheme, valid_netloc, valid_path, invalid_paths, valid_query, \
+    valid_scheme, valid_netlocs, valid_path, invalid_paths, valid_query, \
         extract_parts = (
-            validator['scheme'], validator['domain'], validator['path_regex'],
+            validator['scheme'], validator['domains'], validator['path_regex'],
             validator['path_must_not_match'], validator['qs_args'],
             validator['extract_key']
     )
@@ -25,8 +25,8 @@ def validate_url(url, validator):
     if url_scheme != valid_scheme:
         raise ValidationError(f'invalid scheme "{url_scheme}" must be "{valid_scheme}"')
     url_netloc = str(url_parts.netloc).strip().lower()
-    if url_netloc != valid_netloc:
+    if url_netloc not in valid_netlocs:
-        raise ValidationError(f'invalid domain "{url_netloc}" must be "{valid_netloc}"')
+        raise ValidationError(f'invalid domain "{url_netloc}" must be one of "{valid_netlocs}"')
     url_path = str(url_parts.path).strip()
     matches = re.findall(valid_path, url_path)
     if not matches:
@@ -78,7 +78,7 @@ def resize_image_to_height(image, width, height):
     if scaled_width < width:
         # Width too small, stretch it
         scaled_width = width
-    image = image.resize((scaled_width, height), Image.ANTIALIAS)
+    image = image.resize((scaled_width, height), Image.LANCZOS)
     if scaled_width > width:
         # Width too large, crop it
         delta = scaled_width - width

tubesync/sync/views.py

@@ -1,18 +1,22 @@
 import os
 import json
 from base64 import b64decode
+import pathlib
+import sys
 from django.conf import settings
-from django.http import Http404
+from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect
 from django.views.generic import TemplateView, ListView, DetailView
 from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateView,
                                        DeleteView)
 from django.views.generic.detail import SingleObjectMixin
+from django.core.exceptions import SuspiciousFileOperation
 from django.http import HttpResponse
 from django.urls import reverse_lazy
 from django.db import IntegrityError
 from django.db.models import Q, Count, Sum, When, Case
-from django.forms import ValidationError
+from django.forms import Form, ValidationError
 from django.utils.text import slugify
+from django.utils._os import safe_join
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from common.utils import append_uri_params
@@ -66,7 +70,7 @@ class DashboardView(TemplateView):
             data['average_bytes_per_media'] = 0
         # Latest downloads
         data['latest_downloads'] = Media.objects.filter(
-            downloaded=True
+            downloaded=True, downloaded_filesize__isnull=False
         ).order_by('-download_date')[:10]
         # Largest downloads
         data['largest_downloads'] = Media.objects.filter(
@@ -92,8 +96,27 @@ class SourcesView(ListView):
     paginate_by = settings.SOURCES_PER_PAGE
     messages = {
         'source-deleted': _('Your selected source has been deleted.'),
+        'source-refreshed': _('The source has been scheduled to be synced now.')
     }
 
+    def get(self, *args, **kwargs):
+        if args[0].path.startswith("/source-sync-now/"):
+            sobj = Source.objects.get(pk=kwargs["pk"])
+            if sobj is None:
+                return HttpResponseNotFound()
+            
+            verbose_name = _('Index media from source "{}" once')
+            index_source_task(
+                str(sobj.pk),
+                queue=str(sobj.pk),
+                repeat=0,
+                verbose_name=verbose_name.format(sobj.name))
+            url = reverse_lazy('sync:sources')
+            url = append_uri_params(url, {'message': 'source-refreshed'})
+            return HttpResponseRedirect(url)
+        else:
+            return super().get(self, *args, **kwargs)    
+
     def __init__(self, *args, **kwargs):
         self.message = None
         super().__init__(*args, **kwargs)
@@ -171,7 +194,7 @@ class ValidateSourceView(FormView):
     validation_urls = {
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL: {
             'scheme': 'https',
-            'domain': 'www.youtube.com',
+            'domains': ('m.youtube.com', 'www.youtube.com'),
             'path_regex': '^\/(c\/)?([^\/]+)(\/videos)?$',
             'path_must_not_match': ('/playlist', '/c/playlist'),
             'qs_args': [],
@@ -180,7 +203,7 @@ class ValidateSourceView(FormView):
         },
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: {
             'scheme': 'https',
-            'domain': 'www.youtube.com',
+            'domains': ('m.youtube.com', 'www.youtube.com'),
             'path_regex': '^\/channel\/([^\/]+)(\/videos)?$',
             'path_must_not_match': ('/playlist', '/c/playlist'),
             'qs_args': [],
@@ -189,7 +212,7 @@ class ValidateSourceView(FormView):
         },
         Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: {
             'scheme': 'https',
-            'domain': 'www.youtube.com',
+            'domains': ('m.youtube.com', 'www.youtube.com'),
             'path_regex': '^\/(playlist|watch)$',
             'path_must_not_match': (),
             'qs_args': ('list',),
@@ -269,23 +292,57 @@ class ValidateSourceView(FormView):
         return append_uri_params(url, fields)
 
 
-class AddSourceView(CreateView):
+class EditSourceMixin:
+    model = Source
+    fields = ('source_type', 'key', 'name', 'directory', 'media_format',
+              'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
+              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
+              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo', 
+              'write_json', 'embed_metadata', 'embed_thumbnail', 'enable_sponsorblock',
+              'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs')
+    errors = {
+        'invalid_media_format': _('Invalid media format, the media format contains '
+                                  'errors or is empty. Check the table at the end of '
+                                  'this page for valid media name variables'),
+        'dir_outside_dlroot': _('You cannot specify a directory outside of the '
+                                'base directory (%BASEDIR%)')
+    }
+
+    def form_valid(self, form: Form):
+        # Perform extra validation to make sure the media_format is valid
+        obj = form.save(commit=False)
+        source_type = form.cleaned_data['media_format']
+        example_media_file = obj.get_example_media_format()
+        
+        if example_media_file == '':
+            form.add_error(
+                'media_format',
+                ValidationError(self.errors['invalid_media_format'])
+            )
+
+        # Check for suspicious file path(s)
+        try:
+            targetCheck = form.cleaned_data['directory']+"/.virt"
+            newdir = safe_join(settings.DOWNLOAD_ROOT,targetCheck)
+        except SuspiciousFileOperation:
+            form.add_error(
+                'directory',
+                ValidationError(self.errors['dir_outside_dlroot'].replace("%BASEDIR%",str(settings.DOWNLOAD_ROOT)))
+            )
+
+        if form.errors:
+            return super().form_invalid(form)
+
+        return super().form_valid(form)
+
+
+class AddSourceView(EditSourceMixin, CreateView):
     '''
         Adds a new source, optionally takes some initial data querystring values to
         prepopulate some of the more unclear values.
     '''
 
     template_name = 'sync/source-add.html'
-    model = Source
-    fields = ('source_type', 'key', 'name', 'directory', 'media_format',
-              'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
-              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
-              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
-    errors = {
-        'invalid_media_format': _('Invalid media format, the media format contains '
-                                  'errors or is empty. Check the table at the end of '
-                                  'this page for valid media name variables'),
-    }
 
     def __init__(self, *args, **kwargs):
         self.prepopulated_data = {}
@@ -312,20 +369,6 @@ class AddSourceView(CreateView):
             initial[k] = v
         return initial
 
-    def form_valid(self, form):
-        # Perform extra validation to make sure the media_format is valid
-        obj = form.save(commit=False)
-        source_type = form.cleaned_data['media_format']
-        example_media_file = obj.get_example_media_format()
-        if example_media_file == '':
-            form.add_error(
-                'media_format',
-                ValidationError(self.errors['invalid_media_format'])
-            )
-        if form.errors:
-            return super().form_invalid(form)
-        return super().form_valid(form)
-
     def get_success_url(self):
         url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
         return append_uri_params(url, {'message': 'source-created'})
@@ -364,33 +407,9 @@ class SourceView(DetailView):
         return data
 
 
-class UpdateSourceView(UpdateView):
+class UpdateSourceView(EditSourceMixin, UpdateView):
 
     template_name = 'sync/source-update.html'
-    model = Source
-    fields = ('source_type', 'key', 'name', 'directory', 'media_format',
-              'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
-              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
-              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
-    errors = {
-        'invalid_media_format': _('Invalid media format, the media format contains '
-                                  'errors or is empty. Check the table at the end of '
-                                  'this page for valid media name variables'),
-    }
-
-    def form_valid(self, form):
-        # Perform extra validation to make sure the media_format is valid
-        obj = form.save(commit=False)
-        source_type = form.cleaned_data['media_format']
-        example_media_file = obj.get_example_media_format()
-        if example_media_file == '':
-            form.add_error(
-                'media_format',
-                ValidationError(self.errors['invalid_media_format'])
-            )
-        if form.errors:
-            return super().form_invalid(form)
-        return super().form_valid(form)
 
     def get_success_url(self):
         url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
@@ -416,11 +435,13 @@ class DeleteSourceView(DeleteView, FormMixin):
             for media in Media.objects.filter(source=source):
                 if media.media_file:
                     # Delete the media file
-                    delete_file(media.media_file.name)
+                    delete_file(media.media_file.path)
                     # Delete thumbnail copy if it exists
                     delete_file(media.thumbpath)
                     # Delete NFO file if it exists
                     delete_file(media.nfopath)
+                    # Delete JSON file if it exists
+                    delete_file(media.jsonpath)
         return super().post(request, *args, **kwargs)
 
     def get_success_url(self):
@@ -443,6 +464,7 @@ class MediaView(ListView):
     def __init__(self, *args, **kwargs):
         self.filter_source = None
         self.show_skipped = False
+        self.only_skipped = False
         super().__init__(*args, **kwargs)
 
     def dispatch(self, request, *args, **kwargs):
@@ -455,19 +477,27 @@ class MediaView(ListView):
         show_skipped = request.GET.get('show_skipped', '').strip()
         if show_skipped == 'yes':
             self.show_skipped = True
+        if not self.show_skipped:
+            only_skipped = request.GET.get('only_skipped', '').strip()
+            if only_skipped == 'yes':
+                self.only_skipped = True
         return super().dispatch(request, *args, **kwargs)
 
     def get_queryset(self):
         if self.filter_source:
             if self.show_skipped:
                 q = Media.objects.filter(source=self.filter_source)
+            elif self.only_skipped:
+                q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=True) | Q(manual_skip=True)))
             else:
-                q = Media.objects.filter(source=self.filter_source, skip=False)
+                q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=False) & Q(manual_skip=False)))
         else:
             if self.show_skipped:
                 q = Media.objects.all()
+            elif self.only_skipped:
+                q = Media.objects.filter(Q(skip=True)|Q(manual_skip=True))
             else:
-                q = Media.objects.filter(skip=False)
+                q = Media.objects.filter(Q(skip=False)&Q(manual_skip=False))
         return q.order_by('-published', '-created')
 
     def get_context_data(self, *args, **kwargs):
@@ -479,6 +509,7 @@ class MediaView(ListView):
             data['message'] = message.format(name=self.filter_source.name)
             data['source'] = self.filter_source
         data['show_skipped'] = self.show_skipped
+        data['only_skipped'] = self.only_skipped
         return data
 
 
@@ -628,6 +659,7 @@ class MediaSkipView(FormView, SingleObjectMixin):
             # If the media has an associated NFO file with it, also delete it
             delete_file(self.object.nfopath)
         # Reset all download data
+        self.object.metadata = None
         self.object.downloaded = False
         self.object.downloaded_audio_codec = None
         self.object.downloaded_video_codec = None
@@ -637,6 +669,7 @@ class MediaSkipView(FormView, SingleObjectMixin):
         self.object.downloaded_filesize = None
         # Mark it to be skipped
         self.object.skip = True
+        self.object.manual_skip = True
         self.object.save()
         return super().form_valid(form)
 
@@ -665,6 +698,7 @@ class MediaEnableView(FormView, SingleObjectMixin):
     def form_valid(self, form):
         # Mark it as not skipped
         self.object.skip = False
+        self.object.manual_skip = False
         self.object.save()
         return super().form_valid(form)
 
@@ -673,6 +707,52 @@ class MediaEnableView(FormView, SingleObjectMixin):
         return append_uri_params(url, {'message': 'enabled'})
 
 
+class MediaContent(DetailView):
+    '''
+        Redirect to nginx to download the file
+    '''
+    model = Media
+
+    def __init__(self, *args, **kwargs):
+        self.object = None
+        super().__init__(*args, **kwargs)
+
+    def dispatch(self, request, *args, **kwargs):
+        self.object = self.get_object()
+        # development direct file stream - DO NOT USE PRODUCTIVLY
+        if settings.DEBUG and 'runserver' in sys.argv:
+            # get media URL
+            pth = self.object.media_file.url
+            # remove "/media-data/"
+            pth = pth.split("/media-data/",1)[1]
+            # remove "/" (incase of absolute path)
+            pth = pth.split(str(settings.DOWNLOAD_ROOT).lstrip("/"),1)
+
+            # if we do not have a "/" at the beginning, it is not a absolute path...
+            if len(pth) > 1:
+                pth = pth[1]
+            else:
+                pth = pth[0]
+            
+            
+            # build final path
+            filepth = pathlib.Path(str(settings.DOWNLOAD_ROOT) + pth)
+            
+            if filepth.exists():
+                # return file
+                response = FileResponse(open(filepth,'rb'))
+                return response
+            else:
+                return HttpResponseNotFound()
+            
+        else:
+            headers = {
+                'Content-Type': self.object.content_type,
+                'X-Accel-Redirect': self.object.media_file.url,
+            }
+            return HttpResponse(headers=headers)
+
+
 class TasksView(ListView):
     '''
         A list of tasks queued to be completed. This is, for example, scraping for new

tubesync/sync/youtube.py

@@ -26,13 +26,23 @@ class YouTubeError(yt_dlp.utils.DownloadError):
     pass
 
 
+def get_yt_opts():
+    opts = copy(_defaults)
+    cookie_file = settings.COOKIES_FILE
+    if cookie_file.is_file():
+        cookie_file_path = str(cookie_file.resolve())
+        log.info(f'[youtube-dl] using cookies.txt from: {cookie_file_path}')
+        opts.update({'cookiefile': cookie_file_path})
+    return opts
+
+
 def get_media_info(url):
     '''
         Extracts information from a YouTube URL and returns it as a dict. For a channel
         or playlist this returns a dict of all the videos on the channel or playlist
         as well as associated metadata.
     '''
-    opts = copy(_defaults)
+    opts = get_yt_opts()
     opts.update({
         'skip_download': True,
         'forcejson': True,
@@ -54,13 +64,20 @@ def get_media_info(url):
     return response
 
 
-def download_media(url, media_format, extension, output_file):
+def download_media(url, media_format, extension, output_file, info_json, 
+                   sponsor_categories="all", 
+                   embed_thumbnail=False, embed_metadata=False, skip_sponsors=True, 
+                   write_subtitles=False, auto_subtitles=False, sub_langs='en'):
     '''
         Downloads a YouTube URL to a file on disk.
     '''
 
     def hook(event):
         filename = os.path.basename(event['filename'])
+    
+        if event.get('downloaded_bytes') is None or event.get('total_bytes') is None:
+            return None
+
         if event['status'] == 'error':
             log.error(f'[youtube-dl] error occured downloading: {filename}')
         elif event['status'] == 'downloading':
@@ -91,14 +108,41 @@ def download_media(url, media_format, extension, output_file):
             log.warn(f'[youtube-dl] unknown event: {str(event)}')
     hook.download_progress = 0
 
-    opts = copy(_defaults)
+    ytopts = {
-    opts.update({
         'format': media_format,
         'merge_output_format': extension,
         'outtmpl': output_file,
         'quiet': True,
         'progress_hooks': [hook],
-    })
+        'writeinfojson': info_json,
+        'postprocessors': [],
+        'writesubtitles': write_subtitles,
+        'writeautomaticsub': auto_subtitles,
+        'subtitleslangs': sub_langs.split(','),
+    }
+    
+    sbopt = {
+        'key': 'SponsorBlock',
+        'categories': [sponsor_categories]
+    }
+    ffmdopt = {
+        'key': 'FFmpegMetadata',
+        'add_chapters': True,
+        'add_metadata': True
+    }
+
+    opts = get_yt_opts()
+    if embed_thumbnail:
+        ytopts['postprocessors'].append({'key': 'EmbedThumbnail'})
+    if embed_metadata:
+        ffmdopt["add_metadata"] = True
+    if skip_sponsors:
+        ytopts['postprocessors'].append(sbopt)
+    
+    ytopts['postprocessors'].append(ffmdopt)
+        
+    opts.update(ytopts)
+        
     with yt_dlp.YoutubeDL(opts) as y:
         try:
             return y.download([url])

tubesync/tubesync/celery.py

@@ -0,0 +1,19 @@
+import os
+from celery import Celery
+
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tubesync.settings')
+REDIS_CONNECTION = os.getenv('REDIS_CONNECTION', 'redis://localhost:6379/0')
+
+
+app = Celery('tubesync')
+app.config_from_object('django.conf:settings', namespace='CELERY')
+app.autodiscover_tasks()
+app.conf.broker_url = REDIS_CONNECTION
+app.conf.beat_schedule = {
+    '10-second-beat': {
+        'task': 'sync.tasks.housekeeping_task',
+        'schedule': 60.0,
+        'args': ()
+    },
+}

tubesync/tubesync/dbutils.py

@@ -5,6 +5,11 @@ from django.db.backends.utils import CursorWrapper
 
 def patch_ensure_connection():
     for name, config in settings.DATABASES.items():
+
+        # Don't patch for PostgreSQL, it doesn't need it and can cause issues
+        if config['ENGINE'] == 'django.db.backends.postgresql':
+            continue
+
         module = importlib.import_module(config['ENGINE'] + '.base')
 
         def ensure_connection(self):

tubesync/tubesync/local_settings.py.container

@@ -1,5 +1,6 @@
 import os
 from pathlib import Path
+from urllib.parse import urljoin
 from common.logger import log
 from common.utils import parse_database_connection_string
 
@@ -8,16 +9,20 @@ BASE_DIR = Path(__file__).resolve().parent.parent
 ROOT_DIR = Path('/')
 CONFIG_BASE_DIR = ROOT_DIR / 'config'
 DOWNLOADS_BASE_DIR = ROOT_DIR / 'downloads'
+DJANGO_URL_PREFIX = os.getenv('DJANGO_URL_PREFIX', None)
+STATIC_URL = str(os.getenv('DJANGO_STATIC_URL', '/static/'))
+if DJANGO_URL_PREFIX and STATIC_URL:
+    STATIC_URL = urljoin(DJANGO_URL_PREFIX, STATIC_URL[1:])
 
 
 # This is not ever meant to be a public web interface so this isn't too critical
 SECRET_KEY = str(os.getenv('DJANGO_SECRET_KEY', 'tubesync-django-secret'))
 
 
-ALLOWED_HOSTS_STR = str(os.getenv('TUBESYNC_HOSTS', '127.0.0.1,localhost'))
+ALLOWED_HOSTS_STR = str(os.getenv('TUBESYNC_HOSTS', '*'))
 ALLOWED_HOSTS = ALLOWED_HOSTS_STR.split(',')
 DEBUG = True if os.getenv('TUBESYNC_DEBUG', False) else False
-FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', None)
+FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', DJANGO_URL_PREFIX)
 
 
 TIME_ZONE = os.getenv('TZ', 'UTC')
@@ -59,6 +64,13 @@ if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS:
 MEDIA_ROOT = CONFIG_BASE_DIR / 'media'
 DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR
 YOUTUBE_DL_CACHEDIR = CONFIG_BASE_DIR / 'cache'
+COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt'
+
+
+HEALTHCHECK_FIREWALL_STR = str(os.getenv('TUBESYNC_HEALTHCHECK_FIREWAL', 'True')).strip().lower()
+HEALTHCHECK_FIREWALL = True if HEALTHCHECK_FIREWALL_STR == 'true' else False
+HEALTHCHECK_ALLOWED_IPS_STR = str(os.getenv('TUBESYNC_HEALTHCHECK_ALLOWED_IPS', '127.0.0.1'))
+HEALTHCHECK_ALLOWED_IPS = HEALTHCHECK_ALLOWED_IPS_STR.split(',')
 
 
 BASICAUTH_USERNAME = os.getenv('HTTP_USER', '').strip()

tubesync/tubesync/settings.py

@@ -6,7 +6,7 @@ CONFIG_BASE_DIR = BASE_DIR
 DOWNLOADS_BASE_DIR = BASE_DIR
 
 
-VERSION = '0.10.0'
+VERSION = '0.13.0'
 SECRET_KEY = ''
 DEBUG = False
 ALLOWED_HOSTS = []
@@ -138,7 +138,7 @@ BACKGROUND_TASK_ASYNC_THREADS = 1           # Number of async tasks to run at on
 MAX_BACKGROUND_TASK_ASYNC_THREADS = 8       # For sanity reasons
 BACKGROUND_TASK_PRIORITY_ORDERING = 'ASC'   # Use 'niceness' task priority ordering
 COMPLETED_TASKS_DAYS_TO_KEEP = 7            # Number of days to keep completed tasks
-
+MAX_ENTRIES_PROCESSING = 0                  # Number of videos to process on source refresh (0 for no limit)
 
 SOURCES_PER_PAGE = 100
 MEDIA_PER_PAGE = 144
@@ -149,7 +149,7 @@ MEDIA_THUMBNAIL_WIDTH = 430                 # Width in pixels to resize thumbnai
 MEDIA_THUMBNAIL_HEIGHT = 240                # Height in pixels to resize thumbnails to
 
 
-VIDEO_HEIGHT_CUTOFF = 360       # Smallest resolution in pixels permitted to download
+VIDEO_HEIGHT_CUTOFF = 240       # Smallest resolution in pixels permitted to download
 VIDEO_HEIGHT_IS_HD = 500        # Height in pixels to count as 'HD'
 
 
@@ -161,6 +161,7 @@ YOUTUBE_DEFAULTS = {
     'cachedir': False,      # Disable on-disk caching
     'addmetadata': True,    # Embed metadata during postprocessing where available
 }
+COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt'
 
 
 MEDIA_FORMATSTR_DEFAULT = '{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}'

tubesync/tubesync/wsgi.py

@@ -1,6 +1,25 @@
 import os
+from urllib.parse import urljoin
 from django.core.wsgi import get_wsgi_application
 
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tubesync.settings')
-application = get_wsgi_application()
+DJANGO_URL_PREFIX = os.getenv('DJANGO_URL_PREFIX', None)
+_application = get_wsgi_application()
+
+
+def application(environ, start_response):
+    script_name = None
+    if DJANGO_URL_PREFIX:
+        if DJANGO_URL_PREFIX.endswith('/'):
+            script_name = DJANGO_URL_PREFIX
+        else:
+            raise Exception(f'DJANGO_URL_PREFIX must end with a /, '
+                            f'got: {DJANGO_URL_PREFIX}')
+    if script_name:
+        static_url = urljoin(script_name, 'static/')
+        environ['SCRIPT_NAME'] = script_name
+        path_info = environ['PATH_INFO']
+        if path_info.startswith(script_name) and not path_info.startswith(static_url):
+            environ['PATH_INFO'] = path_info[len(script_name) - 1:]
+    return _application(environ, start_response)