.github/workflows/ci.yaml

@@ -4,10 +4,12 @@ env:
   IMAGE_NAME: tubesync
 
 on:
-  workflow_dispatch:
   push:
     branches:
       - main
+  pull_request:
+    branches:
+      - main
 
 jobs:
   test:
@@ -25,7 +27,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install pipenv
-          pipenv install --system --skip-lock
+          pipenv install --system
       - name: Set up Django environment
         run: cp tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py
       - name: Run Django tests
@@ -33,24 +35,13 @@ jobs:
   containerise:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up QEMU
+    - uses: actions/checkout@v2
-        uses: docker/setup-qemu-action@v1
+    - name: Build the container image
-      - name: Set up Docker Buildx
+      run: docker build . --tag $IMAGE_NAME
-        uses: docker/setup-buildx-action@v1
+    - name: Log into GitHub Container Registry
-      - name: Log into GitHub Container Registry
+      run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
-        run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
+    - name: Push image to GitHub Container Registry
-      - name: Lowercase github username for ghcr
+      run: |
-        id: string
+        LATEST_TAG=ghcr.io/meeb/$IMAGE_NAME:latest
-        uses: ASzc/change-string-case-action@v1
+        docker tag $IMAGE_NAME $LATEST_TAG
-        with:
+        docker push $LATEST_TAG
-          string: ${{ github.actor }}
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest
-          cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest
-          cache-to: type=inline
-          build-args: |
-            IMAGE_NAME=${{ env.IMAGE_NAME }}

.github/workflows/release.yaml

@@ -11,28 +11,18 @@ jobs:
   containerise:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up QEMU
+    - uses: actions/checkout@v2
-        uses: docker/setup-qemu-action@v1
+    - name: Get tag
-      - name: Get tag
+      id: vars
-        id: tag
+      run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
-        uses: dawidd6/action-get-tag@v1
+    - name: Build the container image
-      - uses: docker/build-push-action@v2
+      run: docker build . --tag $IMAGE_NAME
-      - name: Set up Docker Buildx
+    - name: Log into GitHub Container Registry
-        uses: docker/setup-buildx-action@v1
+      run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
-      - name: Log into GitHub Container Registry
+    - name: Push image to GitHub Container Registry
-        run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
+      env:
-      - name: Lowercase github username for ghcr
+        RELEASE_TAG: ${{ steps.vars.outputs.tag }}
-        id: string
+      run: |
-        uses: ASzc/change-string-case-action@v1
+        REF_TAG=ghcr.io/meeb/$IMAGE_NAME:$RELEASE_TAG
-        with:
+        docker tag $IMAGE_NAME $REF_TAG
-          string: ${{ github.actor }}
+        docker push $REF_TAG
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
-          cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
-          cache-to: type=inline
-          build-args: |
-            IMAGE_NAME=${{ env.IMAGE_NAME }}

.gitignore

@@ -1,4 +1,3 @@
-.DS_Store
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -131,6 +130,3 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
-
-Pipfile.lock
-.vscode/launch.json

Dockerfile

@@ -1,112 +1,72 @@
-FROM debian:bookworm-slim
+FROM debian:buster-slim
 
-ARG TARGETPLATFORM
+ARG ARCH="amd64"
-ARG S6_VERSION="3.1.5.0"
+ARG S6_VERSION="2.1.0.2"
-ARG FFMPEG_DATE="autobuild-2023-11-29-14-19"
+ARG FFMPEG_VERSION="4.3.1"
-ARG FFMPEG_VERSION="112875-g47e214245b"
 
 ENV DEBIAN_FRONTEND="noninteractive" \
-  HOME="/root" \
+    HOME="/root" \
-  LANGUAGE="en_US.UTF-8" \
+    LANGUAGE="en_US.UTF-8" \
-  LANG="en_US.UTF-8" \
+    LANG="en_US.UTF-8" \
-  LC_ALL="en_US.UTF-8" \
+    LC_ALL="en_US.UTF-8" \
-  TERM="xterm" \
+    TERM="xterm" \
-  S6_CMD_WAIT_FOR_SERVICES_MAXTIME="0"
+    S6_EXPECTED_SHA256="52460473413601ff7a84ae690b161a074217ddc734990c2cdee9847166cf669e" \
+    S6_DOWNLOAD="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${ARCH}.tar.gz" \
+    FFMPEG_EXPECTED_SHA256="47d95c0129fba27d051748a442a44a73ce1bd38d1e3f9fe1e9dd7258c7581fa5" \
+    FFMPEG_DOWNLOAD="https://tubesync.sfo2.digitaloceanspaces.com/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static.tar.xz"
+
 
 # Install third party software
-RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
+RUN set -x && \
-  "linux/amd64")   echo "amd64"  ;; \
+    apt-get update && \
-  "linux/arm64")   echo "aarch64" ;; \
+    apt-get -y --no-install-recommends install locales && \
-  *)               echo ""        ;; esac) && \
+    echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
-  export S6_ARCH_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
+    locale-gen en_US.UTF-8 && \
-  "linux/amd64")   echo "65d0d0f353d2ff9d0af202b268b4bf53a9948a5007650854855c729289085739" ;; \
+    # Install required distro packages
-  "linux/arm64")   echo "3fbd14201473710a592b2189e81f00f3c8998e96d34f16bd2429c35d1bc36d00" ;; \
+    apt-get -y --no-install-recommends install curl xz-utils ca-certificates binutils && \
-  *)               echo ""        ;; esac) && \
+    # Install s6
-  export S6_DOWNLOAD_ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
+    curl -L ${S6_DOWNLOAD} --output /tmp/s6-overlay-${ARCH}.tar.gz && \
-  "linux/amd64")   echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-x86_64.tar.xz"   ;; \
+    sha256sum /tmp/s6-overlay-${ARCH}.tar.gz && \
-  "linux/arm64")   echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-aarch64.tar.xz" ;; \
+    echo "${S6_EXPECTED_SHA256}  /tmp/s6-overlay-${ARCH}.tar.gz" | sha256sum -c - && \
-  *)               echo ""        ;; esac) && \
+    tar xzf /tmp/s6-overlay-${ARCH}.tar.gz -C / && \
-  export FFMPEG_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
+    # Install ffmpeg
-  "linux/amd64")   echo "36bac8c527bf390603416f749ab0dd860142b0a66f0865b67366062a9c286c8b" ;; \
+    curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg-${ARCH}-static.tar.xz && \
-  "linux/arm64")   echo "8f36e45d99d2367a5c0c220ee3164fa48f4f0cec35f78204ccced8dc303bfbdc" ;; \
+    echo "${FFMPEG_EXPECTED_SHA256}  /tmp/ffmpeg-${ARCH}-static.tar.xz" | sha256sum -c - && \
-  *)               echo ""        ;; esac) && \
+    xz --decompress /tmp/ffmpeg-${ARCH}-static.tar.xz && \
-  export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \
+    tar -xvf /tmp/ffmpeg-${ARCH}-static.tar -C /tmp && \
-  "linux/amd64")   echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linux64-gpl.tar.xz"   ;; \
+    install -v -s -g root -o root -m 0755 -s /tmp/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static/ffmpeg -t /usr/local/bin && \
-  "linux/arm64")   echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linuxarm64-gpl.tar.xz" ;; \
+    # Clean up
-  *)               echo ""        ;; esac) && \
+    rm -rf /tmp/s6-overlay-${ARCH}.tar.gz && \
-  export S6_NOARCH_EXPECTED_SHA256="fd80c231e8ae1a0667b7ae2078b9ad0e1269c4d117bf447a4506815a700dbff3" && \
+    rm -rf /tmp/ffmpeg-${ARCH}-static.tar && \
-  export S6_DOWNLOAD_NOARCH="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-noarch.tar.xz" && \
+    rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static && \
-  echo "Building for arch: ${ARCH}|${ARCH44}, downloading S6 from: ${S6_DOWNLOAD}}, expecting S6 SHA256: ${S6_EXPECTED_SHA256}" && \
+    apt-get -y autoremove --purge curl xz-utils binutils
-  set -x && \
-  apt-get update && \
-  apt-get -y --no-install-recommends install locales && \
-  echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
-  locale-gen en_US.UTF-8 && \
-  # Install required distro packages
-  apt-get -y --no-install-recommends install curl ca-certificates binutils xz-utils && \
-  # Install s6
-  curl -L ${S6_DOWNLOAD_NOARCH} --output /tmp/s6-overlay-noarch.tar.xz && \
-  echo "${S6_NOARCH_EXPECTED_SHA256}  /tmp/s6-overlay-noarch.tar.xz" | sha256sum -c - && \
-  tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && \
-  curl -L ${S6_DOWNLOAD_ARCH} --output /tmp/s6-overlay-${ARCH}.tar.xz && \
-  echo "${S6_ARCH_EXPECTED_SHA256}  /tmp/s6-overlay-${ARCH}.tar.xz" | sha256sum -c - && \
-  tar -C / -Jxpf /tmp/s6-overlay-${ARCH}.tar.xz && \
-  # Install ffmpeg
-  echo "Building for arch: ${ARCH}|${ARCH44}, downloading FFMPEG from: ${FFMPEG_DOWNLOAD}, expecting FFMPEG SHA256: ${FFMPEG_EXPECTED_SHA256}" && \
-  curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg-${ARCH}.tar.xz && \
-  sha256sum /tmp/ffmpeg-${ARCH}.tar.xz && \
-  echo "${FFMPEG_EXPECTED_SHA256}  /tmp/ffmpeg-${ARCH}.tar.xz" | sha256sum -c - && \
-  tar -xf /tmp/ffmpeg-${ARCH}.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffmpeg" && \
-  tar -xf /tmp/ffmpeg-${ARCH}.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffprobe" && \
-  # Clean up
-  rm -rf /tmp/s6-overlay-${ARCH}.tar.gz && \
-  rm -rf /tmp/ffmpeg-${ARCH}.tar.xz && \
-  apt-get -y autoremove --purge curl binutils xz-utils
 
 # Copy app
 COPY tubesync /app
 COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings.py
 
-# Copy over pip.conf to use piwheels
+# Append container bundled software versions
-COPY pip.conf /etc/pip.conf
+RUN echo "ffmpeg_version = '${FFMPEG_VERSION}-static'" >> /app/common/third_party_versions.py
 
 # Add Pipfile
 COPY Pipfile /app/Pipfile
+COPY Pipfile.lock /app/Pipfile.lock
 
 # Switch workdir to the the app
 WORKDIR /app
 
 # Set up the app
 RUN set -x && \
-  apt-get update && \
   # Install required distro packages
   apt-get -y install nginx-light && \
-  apt-get -y --no-install-recommends install \
+  apt-get -y --no-install-recommends install python3 python3-setuptools python3-pip python3-dev gcc make && \
-  python3 \
+  # Install pipenv
-  python3-dev \
+  pip3 --disable-pip-version-check install pipenv && \
-  python3-pip \
-  python3-wheel \
-  pipenv \
-  gcc \
-  g++ \
-  make \
-  pkgconf \
-  default-libmysqlclient-dev \
-  libmariadb3 \
-  postgresql-common \
-  libpq-dev \
-  libpq5 \
-  libjpeg62-turbo \
-  libwebp7 \
-  libjpeg-dev \
-  zlib1g-dev \
-  libwebp-dev \
-  redis-server && \
   # Create a 'app' user which the application will run as
   groupadd app && \
   useradd -M -d /app -s /bin/false -g app app && \
   # Install non-distro packages
-  PIPENV_VERBOSITY=64 pipenv install --system --skip-lock && \
+  pipenv install --system && \
   # Make absolutely sure we didn't accidentally bundle a SQLite dev database
   rm -rf /app/db.sqlite3 && \
   # Run any required app commands
@@ -119,19 +79,10 @@ RUN set -x && \
   mkdir -p /downloads/video && \
   # Clean up
   rm /app/Pipfile && \
+  rm /app/Pipfile.lock && \
   pipenv --clear && \
-  apt-get -y autoremove --purge \
+  pip3 --disable-pip-version-check uninstall -y pipenv wheel virtualenv && \
-  python3-pip \
+  apt-get -y autoremove --purge python3-pip python3-dev gcc make && \
-  python3-dev \
-  gcc \
-  g++ \
-  make \
-  default-libmysqlclient-dev \
-  postgresql-common \
-  libpq-dev \
-  libjpeg-dev \
-  zlib1g-dev \
-  libwebp-dev && \
   apt-get -y autoremove && \
   apt-get -y autoclean && \
   rm -rf /var/lib/apt/lists/* && \
@@ -141,12 +92,7 @@ RUN set -x && \
   rm -rf /root && \
   mkdir -p /root && \
   chown root:root /root && \
-  chmod 0755 /root
+  chmod 0700 /root
-
-# Append software versions
-RUN set -x && \
-  FFMPEG_VERSION=$(/usr/local/bin/ffmpeg -version | head -n 1 | awk '{ print $3 }') && \
-  echo "ffmpeg_version = '${FFMPEG_VERSION}'" >> /app/common/third_party_versions.py
 
 # Copy root
 COPY config/root /
@@ -156,7 +102,7 @@ HEALTHCHECK --interval=1m --timeout=10s CMD /app/healthcheck.py http://127.0.0.1
 
 # ENVS and ports
 ENV PYTHONPATH "/app:${PYTHONPATH}"
-EXPOSE 4848
+EXPOSE 8080
 
 # Volumes
 VOLUME ["/config", "/downloads"]

Makefile

@@ -8,17 +8,17 @@ all: clean build
 
 
 dev:
-	$(python) tubesync/manage.py runserver
+	$(python) app/manage.py runserver
 
 
 build:
-	mkdir -p tubesync/media
+	mkdir -p app/media
-	mkdir -p tubesync/static
+	mkdir -p app/static
-	$(python) tubesync/manage.py collectstatic --noinput
+	$(python) app/manage.py collectstatic --noinput
 
 
 clean:
-	rm -rf tubesync/static
+	rm -rf app/static
 
 
 container: clean
@@ -29,13 +29,5 @@ runcontainer:
 	$(docker) run --rm --name $(name) --env-file dev.env --log-opt max-size=50m -ti -p 4848:4848 $(image)
 
 
-stopcontainer:
+test:
-	$(docker) stop $(name)
+	$(python) app/manage.py test --verbosity=2
-
-
-test: build
-	cd tubesync && $(python) manage.py test --verbosity=2 && cd ..
-
-
-shell:
-	cd tubesync && $(python) manage.py shell

Pipfile

@@ -4,10 +4,9 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
-autopep8 = "*"
 
 [packages]
-django = "~=3.2"
+django = "*"
 django-sass-processor = "*"
 libsass = "*"
 pillow = "*"
@@ -15,11 +14,9 @@ whitenoise = "*"
 gunicorn = "*"
 django-compressor = "*"
 httptools = "*"
+youtube-dl = "*"
 django-background-tasks = "*"
-django-basicauth = "*"
+requests = "*"
-psycopg2-binary = "*"
+
-mysqlclient = "*"
+[requires]
-yt-dlp = "*"
+python_version = "3"
-redis = "*"
-hiredis = "*"
-requests = {extras = ["socks"], version = "*"}

Pipfile.lock

@@ -0,0 +1,251 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "a4bb556fc61ee4583f9588980450b071814298ee4d1a1023fad149c14d14aaba"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "asgiref": {
+            "hashes": [
+                "sha256:5ee950735509d04eb673bd7f7120f8fa1c9e2df495394992c73234d526907e17",
+                "sha256:7162a3cb30ab0609f1a4c95938fd73e8604f63bdba516a7f7d64b83ff09478f0"
+            ],
+            "version": "==3.3.1"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
+                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
+            ],
+            "version": "==2020.12.5"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
+                "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
+            ],
+            "version": "==4.0.0"
+        },
+        "django": {
+            "hashes": [
+                "sha256:169e2e7b4839a7910b393eec127fd7cbae62e80fa55f89c6510426abf673fe5f",
+                "sha256:c6c0462b8b361f8691171af1fb87eceb4442da28477e12200c40420176206ba7"
+            ],
+            "index": "pypi",
+            "version": "==3.1.6"
+        },
+        "django-appconf": {
+            "hashes": [
+                "sha256:1b1d0e1069c843ebe8ae5aa48ec52403b1440402b320c3e3a206a0907e97bb06",
+                "sha256:be58deb54a43d77d2e1621fe59f787681376d3cd0b8bd8e4758ef6c3a6453380"
+            ],
+            "version": "==1.0.4"
+        },
+        "django-background-tasks": {
+            "hashes": [
+                "sha256:e1b19e8d495a276c9d64c5a1ff8b41132f75d2f58e45be71b78650dad59af9de"
+            ],
+            "index": "pypi",
+            "version": "==1.2.5"
+        },
+        "django-compat": {
+            "hashes": [
+                "sha256:3ac9a3bedc56b9365d9eb241bc5157d0c193769bf995f9a78dc1bc24e7c2331b"
+            ],
+            "version": "==1.0.15"
+        },
+        "django-compressor": {
+            "hashes": [
+                "sha256:57ac0a696d061e5fc6fbc55381d2050f353b973fb97eee5593f39247bc0f30af",
+                "sha256:d2ed1c6137ddaac5536233ec0a819e14009553fee0a869bea65d03e5285ba74f"
+            ],
+            "index": "pypi",
+            "version": "==2.4"
+        },
+        "django-sass-processor": {
+            "hashes": [
+                "sha256:9b46a12ca8bdcb397d46fbcc49e6a926ff9f76a93c5efeb23b495419fd01fc7a"
+            ],
+            "index": "pypi",
+            "version": "==0.8.2"
+        },
+        "gunicorn": {
+            "hashes": [
+                "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626",
+                "sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"
+            ],
+            "index": "pypi",
+            "version": "==20.0.4"
+        },
+        "httptools": {
+            "hashes": [
+                "sha256:0a4b1b2012b28e68306575ad14ad5e9120b34fccd02a81eb08838d7e3bbb48be",
+                "sha256:3592e854424ec94bd17dc3e0c96a64e459ec4147e6d53c0a42d0ebcef9cb9c5d",
+                "sha256:41b573cf33f64a8f8f3400d0a7faf48e1888582b6f6e02b82b9bd4f0bf7497ce",
+                "sha256:56b6393c6ac7abe632f2294da53f30d279130a92e8ae39d8d14ee2e1b05ad1f2",
+                "sha256:86c6acd66765a934e8730bf0e9dfaac6fdcf2a4334212bd4a0a1c78f16475ca6",
+                "sha256:96da81e1992be8ac2fd5597bf0283d832287e20cb3cfde8996d2b00356d4e17f",
+                "sha256:96eb359252aeed57ea5c7b3d79839aaa0382c9d3149f7d24dd7172b1bcecb009",
+                "sha256:a2719e1d7a84bb131c4f1e0cb79705034b48de6ae486eb5297a139d6a3296dce",
+                "sha256:ac0aa11e99454b6a66989aa2d44bca41d4e0f968e395a0a8f164b401fefe359a",
+                "sha256:bc3114b9edbca5a1eb7ae7db698c669eb53eb8afbbebdde116c174925260849c",
+                "sha256:fa3cd71e31436911a44620473e873a256851e1f53dee56669dae403ba41756a4",
+                "sha256:fea04e126014169384dee76a153d4573d90d0cbd1d12185da089f73c78390437"
+            ],
+            "index": "pypi",
+            "version": "==0.1.1"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+            ],
+            "version": "==2.10"
+        },
+        "libsass": {
+            "hashes": [
+                "sha256:1521d2a8d4b397c6ec90640a1f6b5529077035efc48ef1c2e53095544e713d1b",
+                "sha256:1b2d415bbf6fa7da33ef46e549db1418498267b459978eff8357e5e823962d35",
+                "sha256:25ebc2085f5eee574761ccc8d9cd29a9b436fc970546d5ef08c6fa41eb57dff1",
+                "sha256:2ae806427b28bc1bb7cb0258666d854fcf92ba52a04656b0b17ba5e190fb48a9",
+                "sha256:4a246e4b88fd279abef8b669206228c92534d96ddcd0770d7012088c408dff23",
+                "sha256:553e5096414a8d4fb48d0a48f5a038d3411abe254d79deac5e008516c019e63a",
+                "sha256:697f0f9fa8a1367ca9ec6869437cb235b1c537fc8519983d1d890178614a8903",
+                "sha256:a8fd4af9f853e8bf42b1425c5e48dd90b504fa2e70d7dac5ac80b8c0a5a5fe85",
+                "sha256:c9411fec76f480ffbacc97d8188322e02a5abca6fc78e70b86a2a2b421eae8a2",
+                "sha256:daa98a51086d92aa7e9c8871cf1a8258124b90e2abf4697852a3dca619838618",
+                "sha256:e0e60836eccbf2d9e24ec978a805cd6642fa92515fbd95e3493fee276af76f8a",
+                "sha256:e64ae2587f1a683e831409aad03ba547c245ef997e1329fffadf7a866d2510b8",
+                "sha256:f6852828e9e104d2ce0358b73c550d26dd86cc3a69439438c3b618811b9584f5"
+            ],
+            "index": "pypi",
+            "version": "==0.20.1"
+        },
+        "pillow": {
+            "hashes": [
+                "sha256:165c88bc9d8dba670110c689e3cc5c71dbe4bfb984ffa7cbebf1fac9554071d6",
+                "sha256:1d208e670abfeb41b6143537a681299ef86e92d2a3dac299d3cd6830d5c7bded",
+                "sha256:22d070ca2e60c99929ef274cfced04294d2368193e935c5d6febfd8b601bf865",
+                "sha256:2353834b2c49b95e1313fb34edf18fca4d57446675d05298bb694bca4b194174",
+                "sha256:39725acf2d2e9c17356e6835dccebe7a697db55f25a09207e38b835d5e1bc032",
+                "sha256:3de6b2ee4f78c6b3d89d184ade5d8fa68af0848f9b6b6da2b9ab7943ec46971a",
+                "sha256:47c0d93ee9c8b181f353dbead6530b26980fe4f5485aa18be8f1fd3c3cbc685e",
+                "sha256:5e2fe3bb2363b862671eba632537cd3a823847db4d98be95690b7e382f3d6378",
+                "sha256:604815c55fd92e735f9738f65dabf4edc3e79f88541c221d292faec1904a4b17",
+                "sha256:6c5275bd82711cd3dcd0af8ce0bb99113ae8911fc2952805f1d012de7d600a4c",
+                "sha256:731ca5aabe9085160cf68b2dbef95fc1991015bc0a3a6ea46a371ab88f3d0913",
+                "sha256:7612520e5e1a371d77e1d1ca3a3ee6227eef00d0a9cddb4ef7ecb0b7396eddf7",
+                "sha256:7916cbc94f1c6b1301ac04510d0881b9e9feb20ae34094d3615a8a7c3db0dcc0",
+                "sha256:81c3fa9a75d9f1afafdb916d5995633f319db09bd773cb56b8e39f1e98d90820",
+                "sha256:887668e792b7edbfb1d3c9d8b5d8c859269a0f0eba4dda562adb95500f60dbba",
+                "sha256:93a473b53cc6e0b3ce6bf51b1b95b7b1e7e6084be3a07e40f79b42e83503fbf2",
+                "sha256:96d4dc103d1a0fa6d47c6c55a47de5f5dafd5ef0114fa10c85a1fd8e0216284b",
+                "sha256:a3d3e086474ef12ef13d42e5f9b7bbf09d39cf6bd4940f982263d6954b13f6a9",
+                "sha256:b02a0b9f332086657852b1f7cb380f6a42403a6d9c42a4c34a561aa4530d5234",
+                "sha256:b09e10ec453de97f9a23a5aa5e30b334195e8d2ddd1ce76cc32e52ba63c8b31d",
+                "sha256:b6f00ad5ebe846cc91763b1d0c6d30a8042e02b2316e27b05de04fa6ec831ec5",
+                "sha256:bba80df38cfc17f490ec651c73bb37cd896bc2400cfba27d078c2135223c1206",
+                "sha256:c3d911614b008e8a576b8e5303e3db29224b455d3d66d1b2848ba6ca83f9ece9",
+                "sha256:ca20739e303254287138234485579b28cb0d524401f83d5129b5ff9d606cb0a8",
+                "sha256:cb192176b477d49b0a327b2a5a4979552b7a58cd42037034316b8018ac3ebb59",
+                "sha256:cdbbe7dff4a677fb555a54f9bc0450f2a21a93c5ba2b44e09e54fcb72d2bd13d",
+                "sha256:cf6e33d92b1526190a1de904df21663c46a456758c0424e4f947ae9aa6088bf7",
+                "sha256:d355502dce85ade85a2511b40b4c61a128902f246504f7de29bbeec1ae27933a",
+                "sha256:d673c4990acd016229a5c1c4ee8a9e6d8f481b27ade5fc3d95938697fa443ce0",
+                "sha256:dc577f4cfdda354db3ae37a572428a90ffdbe4e51eda7849bf442fb803f09c9b",
+                "sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d",
+                "sha256:f50e7a98b0453f39000619d845be8b06e611e56ee6e8186f7f60c3b1e2f0feae"
+            ],
+            "index": "pypi",
+            "version": "==8.1.0"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
+                "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
+            ],
+            "version": "==2021.1"
+        },
+        "rcssmin": {
+            "hashes": [
+                "sha256:ca87b695d3d7864157773a61263e5abb96006e9ff0e021eff90cbe0e1ba18270"
+            ],
+            "version": "==1.0.6"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
+                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
+            ],
+            "index": "pypi",
+            "version": "==2.25.1"
+        },
+        "rjsmin": {
+            "hashes": [
+                "sha256:0ab825839125eaca57cc59581d72e596e58a7a56fbc0839996b7528f0343a0a8",
+                "sha256:211c2fe8298951663bbc02acdffbf714f6793df54bfc50e1c6c9e71b3f2559a3",
+                "sha256:466fe70cc5647c7c51b3260c7e2e323a98b2b173564247f9c89e977720a0645f",
+                "sha256:585e75a84d9199b68056fd4a083d9a61e2a92dfd10ff6d4ce5bdb04bc3bdbfaf",
+                "sha256:6044ca86e917cd5bb2f95e6679a4192cef812122f28ee08c677513de019629b3",
+                "sha256:714329db774a90947e0e2086cdddb80d5e8c4ac1c70c9f92436378dedb8ae345",
+                "sha256:799890bd07a048892d8d3deb9042dbc20b7f5d0eb7da91e9483c561033b23ce2",
+                "sha256:975b69754d6a76be47c0bead12367a1ca9220d08e5393f80bab0230d4625d1f4",
+                "sha256:b15dc75c71f65d9493a8c7fa233fdcec823e3f1b88ad84a843ffef49b338ac32",
+                "sha256:dd0f4819df4243ffe4c964995794c79ca43943b5b756de84be92b445a652fb86",
+                "sha256:e3908b21ebb584ce74a6ac233bdb5f29485752c9d3be5e50c5484ed74169232c",
+                "sha256:e487a7783ac4339e79ec610b98228eb9ac72178973e3dee16eba0e3feef25924",
+                "sha256:ecd29f1b3e66a4c0753105baec262b331bcbceefc22fbe6f7e8bcd2067bcb4d7"
+            ],
+            "version": "==1.1.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
+                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
+            ],
+            "version": "==1.15.0"
+        },
+        "sqlparse": {
+            "hashes": [
+                "sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0",
+                "sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8"
+            ],
+            "version": "==0.4.1"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80",
+                "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73"
+            ],
+            "version": "==1.26.3"
+        },
+        "whitenoise": {
+            "hashes": [
+                "sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7",
+                "sha256:05d00198c777028d72d8b0bbd234db605ef6d60e9410125124002518a48e515d"
+            ],
+            "index": "pypi",
+            "version": "==5.2.0"
+        },
+        "youtube-dl": {
+            "hashes": [
+                "sha256:b390cddbd4d605bd887d0d4063988cef0fa13f916d2e1e3564badbb22504d754",
+                "sha256:e7d48cd42f3081e1e0064e69f31f2856508ef31c0fc80eeebd8e70c6a031a24d"
+            ],
+            "index": "pypi",
+            "version": "==2021.2.10"
+        }
+    },
+    "develop": {}
+}

README.md

@@ -9,10 +9,10 @@ downloaded.
 
 If you want to watch YouTube videos in particular quality or settings from your local
 media server, then TubeSync is for you. Internally, TubeSync is a web interface wrapper
-on `yt-dlp` and `ffmpeg` with a task scheduler.
+on `youtube-dl` and `ffmpeg` with a task scheduler.
 
-There are several other web interfaces to YouTube and `yt-dlp` all with varying
+There are several other web interfaces to YouTube and `youtube-dl` all with varying
-features and implementations. TubeSync's largest difference is full PVR experience of
+features and implemenations. TubeSync's largest difference is full PVR experience of
 updating media servers and better selection of media formats. Additionally, to be as
 hands-free as possible, TubeSync has gradual retrying of failures with back-off timers
 so media which fails to download will be retried for an extended period making it,
@@ -22,9 +22,12 @@ hopefully, quite reliable.
 # Latest container image
 
 ```yaml
-ghcr.io/meeb/tubesync:latest
+ghcr.io/meeb/tubesync:v0.9
 ```
 
+**NOTE: the `:latest` tag does exist, but will contain in-development commits and may
+be broken. Use at your own risk.**
+
 # Screenshots
 
 ### Dashboard
@@ -69,12 +72,11 @@ currently just Plex, to complete the PVR experience.
 # Installation
 
 TubeSync is designed to be run in a container, such as via Docker or Podman. It also
-works in a Docker Compose stack. `amd64` (most desktop PCs and servers) and `arm64`
+works in a Docker Compose stack. Only `amd64` is initially supported.
-(modern ARM computers, such as the Rasperry Pi 3 or later) are supported.
 
 Example (with Docker on *nix):
 
-First find the user ID and group ID you want to run TubeSync as, if you're not
+First find your the user ID and group ID you want to run TubeSync as, if you're not
 sure what this is it's probably your current user ID and group ID:
 
 ```bash
@@ -99,8 +101,8 @@ $ mkdir /some/directory/tubesync-downloads
 Finally, download and run the container:
 
 ```bash
-# Pull image
+# Pull a versioned image
-$ docker pull ghcr.io/meeb/tubesync:latest
+$ docker pull ghcr.io/meeb/tubesync:v0.9
 # Start the container using your user ID and group ID
 $ docker run \
   -d \
@@ -111,21 +113,19 @@ $ docker run \
   -v /some/directory/tubesync-config:/config \
   -v /some/directory/tubesync-downloads:/downloads \
   -p 4848:4848 \
-  ghcr.io/meeb/tubesync:latest
+  ghcr.io/meeb/tubesync:v0.9
 ```
 
 Once running, open `http://localhost:4848` in your browser and you should see the
 TubeSync dashboard. If you do, you can proceed to adding some sources (YouTube channels
 and playlists). If not, check `docker logs tubesync` to see what errors might be
-occurring, typical ones are file permission issues.
+occuring, typical ones are file permission issues.
 
 Alternatively, for Docker Compose, you can use something like:
 
-```yml
+```yaml
-version: '3.7'
-services:
   tubesync:
-    image: ghcr.io/meeb/tubesync:latest
+    image: ghcr.io/meeb/tubesync:v0.9
     container_name: tubesync
     restart: unless-stopped
     ports:
@@ -139,41 +139,6 @@ services:
       - PGID=1000
 ```
 
-
-## Optional authentication
-
-Available in `v1.0` (or `:latest`)and later. If you want to enable a basic username and
-password to be required to access the TubeSync dashboard you can set them with the
-following environment variables:
-
-```bash
-HTTP_USER
-HTTP_PASS
-```
-
-For example, in the `docker run ...` line add in:
-
-```bash
-...
--e HTTP_USER=some-username \
--e HTTP_PASS=some-secure-password \
-...
-```
-
-Or in your Docker Compose file you would add in:
-
-```yaml
-...
-    environment:
-      - HTTP_USER=some-username
-      - HTTP_PASS=some-secure-password
-...
-```
-
-When BOTH `HTTP_USER` and `HTTP_PASS` are set then basic HTTP authentication will be
-enabled.
-
-
 # Updating
 
 To update, you can just pull a new version of the container image as they are released.
@@ -236,12 +201,9 @@ $ docker logs --follow tubesync
 Once you're happy using TubeSync there are some advanced usage guides for more complex
 and less common features:
 
- * [Import existing media into TubeSync](https://github.com/meeb/tubesync/blob/main/docs/import-existing-media.md)
+![Import existing media into TubeSync](https://github.com/meeb/tubesync/blob/main/docs/import-existing-media.md)
- * [Sync or create missing metadata files](https://github.com/meeb/tubesync/blob/main/docs/create-missing-metadata.md)
+
- * [Reset tasks from the command line](https://github.com/meeb/tubesync/blob/main/docs/reset-tasks.md)
+![Sync or create missing metadata files](https://github.com/meeb/tubesync/blob/main/docs/create-missing-metadata.md)
- * [Using PostgreSQL, MySQL or MariaDB as database backends](https://github.com/meeb/tubesync/blob/main/docs/other-database-backends.md)
- * [Using cookies](https://github.com/meeb/tubesync/blob/main/docs/using-cookies.md)
- * [Reset metadata](https://github.com/meeb/tubesync/blob/main/docs/reset-metadata.md)
 
 
 # Warnings
@@ -283,7 +245,7 @@ automatically.
 ### Does TubeSync support any other video platforms?
 
 At the moment, no. This is a pre-release. The library TubeSync uses that does most
-of the downloading work, `yt-dlp`, supports many hundreds of video sources so it's
+of the downloading work, `youtube-dl`, supports many hundreds of video sources so it's
 likely more will be added to TubeSync if there is demand for it.
 
 ### Is there a progress bar?
@@ -295,27 +257,27 @@ your install is doing check the container logs.
 
 ### Are there alerts when a download is complete?
 
-No, this feature is best served by existing services such as the excellent 
+No, this feature is best served by existing services such as the execelent
 [Tautulli](https://tautulli.com/) which can monitor your Plex server and send alerts
 that way.
 
-### There are errors in my "tasks" tab!
+### There's errors in my "tasks" tab!
 
 You only really need to worry about these if there is a permanent failure. Some errors
-are temporary and will be retried for you automatically, such as a download got
+are temproary and will be retried for you automatically, such as a download got
-interrupted and will be tried again later. Sources with permanent errors (such as no
+interrupted and will be tried again later. Sources with permanet errors (such as no
 media available because you got a channel name wrong) will be shown as errors on the
 "sources" tab.
 
 ### What is TubeSync written in?
 
-Python3 using Django, embedding yt-dlp. It's pretty much glue between other much
+Python3 using Django, embedding youtube-dl. It's pretty much glue between other much
 larger libraries.
 
 Notable libraries and software used:
 
  * [Django](https://www.djangoproject.com/)
- * [yt-dlp](https://github.com/yt-dlp/yt-dlp)
+ * [youtube-dl](https://yt-dl.org/)
  * [ffmpeg](https://ffmpeg.org/)
  * [Django Background Tasks](https://github.com/arteria/django-background-tasks/)
  * [django-sass](https://github.com/coderedcorp/django-sass/)
@@ -325,7 +287,7 @@ See the [Pipefile](https://github.com/meeb/tubesync/blob/main/Pipfile) for a ful
 
 ### Can I get access to the full Django admin?
 
-Yes, although pretty much all operations are available through the front-end interface
+Yes, although pretty much all operations are available through the front end interface
 and you can probably break things by playing in the admin. If you still want to access
 it you can run:
 
@@ -338,9 +300,7 @@ can log in at http://localhost:4848/admin
 
 ### Are there user accounts or multi-user support?
 
-There is support for basic HTTP authentication by setting the `HTTP_USER` and
+No not at the moment. This could be added later if there is demand for it.
-`HTTP_PASS` environment variables. There is not support for multi-user or user
-management.
 
 ### Does TubeSync support HTTPS?
 
@@ -351,10 +311,6 @@ etc.). Configuration of this is beyond the scope of this README.
 
 Just `amd64` for the moment. Others may be made available if there is demand.
 
-### The pipenv install fails with "Locking failed"!
-
-Make sure that you have `mysql_config` or `mariadb_config` available, as required by the python module `mysqlclient`. On Debian-based systems this is usually found in the package `libmysqlclient-dev`
-
 
 # Advanced configuration
 
@@ -362,26 +318,22 @@ There are a number of other environment variables you can set. These are, mostly
 **NOT** required to be set in the default container installation, they are really only
 useful if you are manually installing TubeSync in some other environment. These are:
 
-| Name                        | What                                                         | Example                              |
+| Name                     | What                                                         | Example                            |
-| --------------------------- | ------------------------------------------------------------ | ------------------------------------ |
+| ------------------------ | ------------------------------------------------------------ | ---------------------------------- |
-| DJANGO_SECRET_KEY           | Django's SECRET_KEY                                          | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l     |
+| DJANGO_SECRET_KEY        | Django's SECRET_KEY                                          | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l   |
-| DJANGO_URL_PREFIX           | Run TubeSync in a sub-URL on the web server                  | /somepath/                           |
+| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME                                   | /somepath                          |
-| TUBESYNC_DEBUG              | Enable debugging                                             | True                                 |
+| TUBESYNC_DEBUG           | Enable debugging                                             | True                               |
-| TUBESYNC_WORKERS            | Number of background workers, default is 2, max allowed is 8 | 2                                    |
+| TUBESYNC_WORKERS         | Number of background workers, default is 2, max allowed is 8 | 2                                  |
-| TUBESYNC_HOSTS              | Django's ALLOWED_HOSTS, defaults to `*`                      | tubesync.example.com,otherhost.com   |
+| TUBESYNC_HOSTS           | Django's ALLOWED_HOSTS                                       | tubesync.example.com,otherhost.com |
-| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True  | True
+| GUNICORN_WORKERS         | Number of gunicorn workers to spawn                          | 3                                  |
-| GUNICORN_WORKERS            | Number of gunicorn workers to spawn                          | 3                                    |
+| LISTEN_HOST              | IP address for gunicorn to listen on                         | 127.0.0.1                          |
-| LISTEN_HOST                 | IP address for gunicorn to listen on                         | 127.0.0.1                            |
+| LISTEN_PORT              | Port number for gunicorn to listen on                        | 8080                               |
-| LISTEN_PORT                 | Port number for gunicorn to listen on                        | 8080                                 |
-| HTTP_USER                   | Sets the username for HTTP basic authentication              | some-username                        |
-| HTTP_PASS                   | Sets the password for HTTP basic authentication              | some-secure-password                 |
-| DATABASE_CONNECTION         | Optional external database connection details                | mysql://user:pass@host:port/database |
 
 
 # Manual, non-containerised, installation
 
 As a relatively normal Django app you can run TubeSync without the container. Beyond
-following this rough guide, you are on your own and should be knowledgeable about
+following this rough guide you are on your own and should be knowledgeable about
 installing and running WSGI-based Python web applications before attempting this.
 
 1. Clone or download this repo
@@ -392,7 +344,7 @@ installing and running WSGI-based Python web applications before attempting this
    `tubesync/tubesync/local_settings.py` and edit it as appropriate
 5. Run migrations with `./manage.py migrate`
 6. Collect static files with `./manage.py collectstatic`
-6. Set up your prefered WSGI server, such as `gunicorn` pointing it to the application
+6. Set up your prefered WSGI server, such as `gunicorn` poiting it to the application
    in `tubesync/tubesync/wsgi.py`
 7. Set up your proxy server such as `nginx` and forward it to the WSGI server
 8. Check the web interface is working
@@ -404,7 +356,7 @@ installing and running WSGI-based Python web applications before attempting this
 
 # Tests
 
-There is a moderately comprehensive test suite focusing on the custom media format
+There is a moderately comprehensive test suite focussing on the custom media format
 matching logic and that the front-end interface works. You can run it via Django:
 
 ```bash

config/root/etc/cont-init.d/10-tubesync

@@ -0,0 +1,27 @@
+#!/usr/bin/with-contenv bash
+
+# Change runtime user UID and GID
+PUID=${PUID:-911}
+PGID=${PGID:-911}
+groupmod -o -g "$PGID" app
+usermod -o -u "$PUID" app
+
+# Reset permissions
+chown -R app:app /run/app && \
+chmod -R 0700 /run/app && \
+chown -R app:app /config && \
+chmod -R 0755 /config && \
+chown -R app:app /downloads && \
+chmod -R 0755 /downloads && \
+chown -R root:app /app && \
+chmod -R 0750 /app && \
+chown -R app:app /app/common/static && \
+chmod -R 0750 /app/common/static && \
+chown -R app:app /app/static && \
+chmod -R 0750 /app/static && \
+find /app -type f ! -iname healthcheck.py -exec chmod 640 {} \; && \
+chmod 0755 /app/healthcheck.py
+
+# Run migrations
+exec s6-setuidgid app \
+    /usr/bin/python3 /app/manage.py migrate

config/root/etc/nginx/nginx.conf

@@ -79,11 +79,6 @@ http {
             proxy_connect_timeout 10;
         }
 
-        # File dwnload and streaming
-        location /media-data/ {
-            internal;
-            alias /downloads/;
-        }
     }
 
 }

config/root/etc/redis/redis.conf

@@ -1,46 +0,0 @@
-bind 127.0.0.1
-protected-mode yes
-port 6379
-tcp-backlog 511
-timeout 0
-tcp-keepalive 300
-daemonize no
-supervised no
-loglevel notice
-logfile ""
-databases 1
-always-show-logo no
-save ""
-dir /var/lib/redis
-maxmemory 64mb
-maxmemory-policy noeviction
-lazyfree-lazy-eviction no
-lazyfree-lazy-expire no
-lazyfree-lazy-server-del no
-replica-lazy-flush no
-lazyfree-lazy-user-del no
-oom-score-adj no
-oom-score-adj-values 0 200 800
-appendonly no
-appendfsync no
-lua-time-limit 5000
-slowlog-log-slower-than 10000
-slowlog-max-len 128
-latency-monitor-threshold 0
-notify-keyspace-events ""
-hash-max-ziplist-entries 512
-hash-max-ziplist-value 64
-list-max-ziplist-size -2
-list-compress-depth 0
-set-max-intset-entries 512
-zset-max-ziplist-entries 128
-zset-max-ziplist-value 64
-hll-sparse-max-bytes 3000
-stream-node-max-bytes 4096
-stream-node-max-entries 100
-activerehashing yes
-client-output-buffer-limit normal 0 0 0
-client-output-buffer-limit replica 256mb 64mb 60
-client-output-buffer-limit pubsub 32mb 8mb 60
-hz 10
-dynamic-hz yes

config/root/etc/s6-overlay/s6-rc.d/celery-beat/dependencies

@@ -1 +0,0 @@
-gunicorn

config/root/etc/s6-overlay/s6-rc.d/celery-beat/run

@@ -1,25 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-UMASK_SET=${UMASK_SET:-022}
-umask "$UMASK_SET"
-
-cd /app || exit
-
-PIDFILE=/run/app/celery-beat.pid
-SCHEDULE=/tmp/tubesync-celerybeat-schedule
-
-if [ -f "${PIDFILE}" ]
-then
-    PID=$(cat $PIDFILE)
-    echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
-    if kill -0 $PID
-    then
-        echo "Killing old gunicorn process with PID: ${PID}"
-        kill -9 $PID
-    fi
-    echo "Removing stale PID file: ${PIDFILE}"
-    rm ${PIDFILE}
-fi
-
-#exec s6-setuidgid app \
-#    /usr/local/bin/celery --workdir /app -A tubesync beat --pidfile ${PIDFILE} -s ${SCHEDULE}

config/root/etc/s6-overlay/s6-rc.d/celery-beat/type

@@ -1 +0,0 @@
-longrun

config/root/etc/s6-overlay/s6-rc.d/celery-worker/dependencies

@@ -1 +0,0 @@
-gunicorn

config/root/etc/s6-overlay/s6-rc.d/celery-worker/run

@@ -1,24 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-UMASK_SET=${UMASK_SET:-022}
-umask "$UMASK_SET"
-
-cd /app || exit
-
-PIDFILE=/run/app/celery-worker.pid
-
-if [ -f "${PIDFILE}" ]
-then
-    PID=$(cat $PIDFILE)
-    echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
-    if kill -0 $PID
-    then
-        echo "Killing old gunicorn process with PID: ${PID}"
-        kill -9 $PID
-    fi
-    echo "Removing stale PID file: ${PIDFILE}"
-    rm ${PIDFILE}
-fi
-
-#exec s6-setuidgid app \
-#    /usr/local/bin/celery --workdir /app -A tubesync worker --pidfile ${PIDFILE} -l INFO

config/root/etc/s6-overlay/s6-rc.d/celery-worker/type

@@ -1 +0,0 @@
-longrun

config/root/etc/s6-overlay/s6-rc.d/gunicorn/dependencies

@@ -1 +0,0 @@
-tubesync-init

config/root/etc/s6-overlay/s6-rc.d/gunicorn/run

@@ -1,24 +0,0 @@
-#!/command/with-contenv bash
-
-UMASK_SET=${UMASK_SET:-022}
-umask "$UMASK_SET"
-
-cd /app || exit
-
-PIDFILE=/run/app/gunicorn.pid
-
-if [ -f "${PIDFILE}" ]
-then
-    PID=$(cat $PIDFILE)
-    echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
-    if kill -0 $PID
-    then
-        echo "Killing old gunicorn process with PID: ${PID}"
-        kill -9 $PID
-    fi
-    echo "Removing stale PID file: ${PIDFILE}"
-    rm ${PIDFILE}
-fi
-
-exec s6-setuidgid app \
-    /usr/local/bin/gunicorn -c /app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application

config/root/etc/s6-overlay/s6-rc.d/gunicorn/type

@@ -1 +0,0 @@
-longrun

config/root/etc/s6-overlay/s6-rc.d/nginx/dependencies

@@ -1 +0,0 @@
-gunicorn

config/root/etc/s6-overlay/s6-rc.d/nginx/run

@@ -1,5 +0,0 @@
-#!/command/with-contenv bash
-
-cd /
-
-/usr/sbin/nginx

config/root/etc/s6-overlay/s6-rc.d/nginx/type

@@ -1 +0,0 @@
-longrun

config/root/etc/s6-overlay/s6-rc.d/redis/run

@@ -1,4 +0,0 @@
-#!/command/with-contenv bash
-
-exec s6-setuidgid redis \
-    /usr/bin/redis-server /etc/redis/redis.conf

config/root/etc/s6-overlay/s6-rc.d/redis/type

@@ -1 +0,0 @@
-longrun

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run

@@ -1,34 +0,0 @@
-#!/command/with-contenv bash
-
-# Change runtime user UID and GID
-PUID="${PUID:-911}"
-PUID="${PUID:-911}"
-groupmod -o -g "$PGID" app
-usermod -o -u "$PUID" app
-
-# Reset permissions
-chown -R app:app /run/app
-chmod -R 0700 /run/app
-chown -R app:app /config
-chmod -R 0755 /config
-chown -R root:app /app
-chmod -R 0750 /app
-chown -R app:app /app/common/static
-chmod -R 0750 /app/common/static
-chown -R app:app /app/static
-chmod -R 0750 /app/static
-find /app -type f ! -iname healthcheck.py -exec chmod 640 {} \;
-chmod 0755 /app/healthcheck.py
-
-# Optionally reset the download dir permissions
-TUBESYNC_RESET_DOWNLOAD_DIR="${TUBESYNC_RESET_DOWNLOAD_DIR:-True}"
-if [ "$TUBESYNC_RESET_DOWNLOAD_DIR" == "True" ]
-then
-    echo "TUBESYNC_RESET_DOWNLOAD_DIR=True, Resetting /downloads directory permissions"
-    chown -R app:app /downloads
-    chmod -R 0755 /downloads
-fi
-
-# Run migrations
-exec s6-setuidgid app \
-    /usr/bin/python3 /app/manage.py migrate

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/type

@@ -1 +0,0 @@
-oneshot

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/up

@@ -1,3 +0,0 @@
-#!/command/execlineb -P
-
-/etc/s6-overlay/s6-rc.d/tubesync-init/run

config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/dependencies

@@ -1 +0,0 @@
-gunicorn

config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/type

@@ -1 +0,0 @@
-longrun

config/root/etc/services.d/gunicorn/run

@@ -0,0 +1,9 @@
+#!/usr/bin/with-contenv bash
+
+UMASK_SET=${UMASK_SET:-022}
+umask "$UMASK_SET"
+
+cd /app || exit
+
+exec s6-setuidgid app \
+    /usr/local/bin/gunicorn -c /app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application

config/root/etc/services.d/nginx/run

@@ -0,0 +1,5 @@
+#!/usr/bin/with-contenv bash
+
+cd /
+
+/usr/sbin/nginx

config/root/etc/services.d/tubesync-worker/run

@@ -1,4 +1,4 @@
-#!/command/with-contenv bash
+#!/usr/bin/with-contenv bash
 
 exec s6-setuidgid app \
     /usr/bin/python3 /app/manage.py process_tasks

docs/other-database-backends.md

@@ -1,132 +0,0 @@
-# TubeSync
-
-## Advanced usage guide - using other database backends
-
-This is a new feature in v1.0 of TubeSync and later. It allows you to use a custom
-existing external database server instead of the default SQLite database. You may want
-to use this if you encounter performance issues with adding very large or a large
-number of channels and database write contention (as shown by errors in the log)
-become an issue.
-
-## Requirements
-
-TubeSync supports SQLite (the automatic default) as well as PostgreSQL, MySQL and
-MariaDB. For MariaDB just follow the MySQL instructions as the driver is the same.
-
-You should start with a blank install of TubeSync. Migrating to a new database will
-reset your database. If you are comfortable with Django you can export and re-import
-existing database data with:
-
-```bash
-$ docker exec -i tubesync python3 /app/manage.py dumpdata > some-file.json
-```
-
-Then change you database backend over, then use
-
-```bash
-$ cat some-file.json | docker exec -i tubesync python3 /app/manage.py loaddata - --format=json
-```
-
-As detailed in the Django documentation:
-
-https://docs.djangoproject.com/en/3.1/ref/django-admin/#dumpdata
-
-and:
-
-https://docs.djangoproject.com/en/3.1/ref/django-admin/#loaddata
-
-Further instructions are beyond the scope of TubeSync documenation and you should refer
-to Django documentation for more details.
-
-If you are not comfortable with the above, then skip the `dumpdata` steps, however
-remember you will start again with a completely new database.
-
-## Steps
-
-### 1. Create a database in your external database server
-
-You need to create a database and a user with permissions to access the database in
-your chosen external database server. Steps vary between PostgreSQL, MySQL and MariaDB
-so this is up to you to work out.
-
-### 2. Set the database connection string environment variable
-
-You need to provide the database connection details to TubeSync via an environment
-variable. The environment variable name is `DATABASE_CONNECTION` and the format is the
-standard URL-style string. Examples are:
-
-`postgresql://tubesync:password@localhost:5432/tubesync`
-
-and
-
-`mysql://tubesync:password@localhost:3306/tubesync`
-
-*Important note:* For MySQL databases make SURE you create the tubesync database with
-`utf8mb4` encoding, like:
-
-`CREATE DATABASE tubesync CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;`
-
-Without `utf8mb4` encoding things like emojis in video titles (or any extended UTF8
-characters) can cause issues.
-
-### 3. Start TubeSync and check the logs
-
-Once you start TubeSync with the new database connection you should see the folling log
-entry in the container or stdout logs:
-
-`2021-04-04 22:42:17,912 [tubesync/INFO] Using database connection: django.db.backends.postgresql://tubesync:[hidden]@localhost:5432/tubesync`
-
-If you see a line similar to the above and the web interface loads, congratulations,
-you are now using an external database server for your TubeSync data!
-
-## Database Compression (For MariaDB)
-With a lot of media files the `sync_media` table grows in size quickly.
-You can save space using column compression using the following steps while using MariaDB:
-
- 1. Stop tubesync 
- 2. Execute `ALTER TABLE sync_media MODIFY metadata LONGTEXT COMPRESSED;` on database tubesync
- 3. Start tunesync and confirm the connection still works.
-
-## Docker Compose
-
-If you're using Docker Compose and simply want to connect to another container with
-the DB for the performance benefits, a configuration like this would be enough:
-
-```
- tubesync-db:
-  image: postgres:15.2
-  container_name: tubesync-db
-  restart: unless-stopped
-  volumes:
-   - /<path/to>/init.sql:/docker-entrypoint-initdb.d/init.sql
-   - /<path/to>/tubesync-db:/var/lib/postgresql/data
-  environment:
-   - POSTGRES_USER=postgres
-   - POSTGRES_PASSWORD=testpassword
-
- tubesync:
-  image: ghcr.io/meeb/tubesync:latest
-  container_name: tubesync
-  restart: unless-stopped
-  ports:
-   - 4848:4848
-  volumes:
-   - /<path/to>/tubesync/config:/config
-   - /<path/to>/YouTube:/downloads
-  environment:
-   - DATABASE_CONNECTION=postgresql://postgres:testpassword@tubesync-db:5432/tubesync
-  depends_on:
-   - tubesync-db
-```
-
-Note that an `init.sql` file is needed to initialize the `tubesync`
-database before it can be written to. This file should contain:
-
-```
-CREATE DATABASE tubesync;
-```
-
-
-Then it must be mapped to `/docker-entrypoint-initdb.d/init.sql` for it
-to be executed on first startup of the container. See the `tubesync-db`
-volume mapping above for how to do this.

docs/reset-metadata.md

@@ -1,30 +0,0 @@
-# TubeSync
-
-## Advanced usage guide - reset media metadata from the command line
-
-This command allows you to reset all media item metadata. You might want to use
-this if you have a lot of media items with invalid metadata and you want to
-wipe it which triggers the metadata to be redownloaded.
-
-
-## Requirements
-
-You have added some sources and media
-
-## Steps
-
-### 1. Run the reset tasks command
-
-Execute the following Django command:
-
-`./manage.py reset-metadata`
-
-When deploying TubeSync inside a container, you can execute this with:
-
-`docker exec -ti tubesync python3 /app/manage.py reset-metadata`
-
-This command will log what its doing to the terminal when you run it.
-
-When this is run, new tasks will be immediately created so all your media
-items will start downloading updated metadata straight away, any missing information
-such as thumbnails will be redownloaded, etc.

docs/reset-tasks.md

@@ -1,33 +0,0 @@
-# TubeSync
-
-## Advanced usage guide - reset tasks from the command line
-
-This is a new feature in v1.0 of TubeSync and later. It allows you to reset all
-scheduled tasks from the command line as well as the "reset tasks" button in the
-"tasks" tab of the dashboard.
-
-This is useful for TubeSync installations where you may have a lot of media and
-sources added and the "reset tasks" button may take too long to the extent where
-the page times out (with a 502 error or similar issue).
-
-## Requirements
-
-You have added some sources and media
-
-## Steps
-
-### 1. Run the reset tasks command
-
-Execute the following Django command:
-
-`./manage.py reset-tasks`
-
-When deploying TubeSync inside a container, you can execute this with:
-
-`docker exec -ti tubesync python3 /app/manage.py reset-tasks`
-
-This command will log what its doing to the terminal when you run it.
-
-When this is run, new tasks will be immediately created so all your sources will be
-indexed again straight away, any missing information such as thumbnails will be
-redownloaded, etc.

docs/using-cookies.md

@@ -1,50 +0,0 @@
-# TubeSync
-
-## Advanced usage guide - using exported cookies
-
-This is a new feature in v0.10 of TubeSync and later. It allows you to use the cookies
-file exported from your browser in "Netscape" format with TubeSync to authenticate
-to YouTube. This can bypass some throttling, age restrictions and other blocks at
-YouTube.
-
-**IMPORTANT NOTE**: Using cookies exported from your browser that is authenticated
-to YouTube identifes your Google account as using TubeSync. This may result in
-potential account impacts and is entirely at your own risk. Do not use this
-feature unless you really know what you're doing.
-
-## Requirements
-
-Have a browser that supports exporting your cookies and be logged into YouTube.
-
-## Steps
-
-### 1. Export your cookies
-
-You need to export cookies for youtube.com from your browser, you can either do
-this manually or there are plug-ins to automate this for you. This file must be
-in the "Netscape" cookie export format.
-
-Save your cookies as a `cookies.txt` file.
-
-### 2. Import into TubeSync
-
-Drop the `cookies.txt` file into your TubeSync `config` directory.
-
-If detected correctly, you will see something like this in the worker or container
-logs:
-
-```
-YYYY-MM-DD HH:MM:SS,mmm [tubesync/INFO] [youtube-dl] using cookies.txt from: /config/cookies.txt
-```
-
-If you see that line it's working correctly.
-
-If you see errors in your logs like this:
-
-```
-http.cookiejar.LoadError: '/config/cookies.txt' does not look like a Netscape format cookies file
-```
-
-Then your `cookies.txt` file was not generated or created correctly as it's not
-in the required "Netscape" format. You can fix this by exporting your `cookies.txt`
-in the correct "Netscape" format.

pip.conf

@@ -1,2 +0,0 @@
-[global]
-extra-index-url=https://www.piwheels.org/simple

tubesync/common/context_processors.py

@@ -1,10 +1,10 @@
 from django.conf import settings
-from .third_party_versions import yt_dlp_version, ffmpeg_version
+from .third_party_versions import youtube_dl_version, ffmpeg_version
 
 
 def app_details(request):
     return {
         'app_version': str(settings.VERSION),
-        'yt_dlp_version': yt_dlp_version,
+        'youtube_dl_version': youtube_dl_version,
         'ffmpeg_version': ffmpeg_version,
     }

tubesync/common/errors.py

@@ -20,10 +20,3 @@ class DownloadFailedException(Exception):
         exist.
     '''
     pass
-
-
-class DatabaseConnectionError(Exception):
-    '''
-        Raised when parsing or initially connecting to a database.
-    '''
-    pass

tubesync/common/logger.py

@@ -1,14 +1,10 @@
 import logging
-from django.conf import settings
-
-
-logging_level = logging.DEBUG if settings.DEBUG else logging.INFO
 
 
 log = logging.getLogger('tubesync')
-log.setLevel(logging_level)
+log.setLevel(logging.DEBUG)
 ch = logging.StreamHandler()
-ch.setLevel(logging_level)
+ch.setLevel(logging.DEBUG)
 formatter = logging.Formatter('%(asctime)s [%(name)s/%(levelname)s] %(message)s')
 ch.setFormatter(formatter)
 log.addHandler(ch)

tubesync/common/middleware.py

@@ -1,6 +1,4 @@
-from django.conf import settings
 from django.forms import BaseForm
-from basicauth.middleware import BasicAuthMiddleware as BaseBasicAuthMiddleware
 
 
 class MaterializeDefaultFieldsMiddleware:
@@ -21,12 +19,3 @@ class MaterializeDefaultFieldsMiddleware:
                 for _, field in v.fields.items():
                     field.widget.attrs.update({'class':'browser-default'})
         return response
-
-
-class BasicAuthMiddleware(BaseBasicAuthMiddleware):
-
-    def process_request(self, request):
-        bypass_uris = getattr(settings, 'BASICAUTH_ALWAYS_ALLOW_URIS', [])
-        if request.path in bypass_uris:
-            return None
-        return super().process_request(request)

tubesync/common/static/styles/_fonts.scss

@@ -1,20 +1,20 @@
 @font-face {
-    font-family: 'roboto';
+    font-family: 'roboto-light';
-    src: url('../fonts/roboto/roboto-light.woff') format('woff');
+    src: url('/static/fonts/roboto/roboto-light.woff') format('woff');
-    font-weight: lighter;
-    font-style: normal;
-}
-
-@font-face {
-    font-family: 'roboto';
-    src: url('../fonts/roboto/roboto-regular.woff') format('woff');
     font-weight: normal;
     font-style: normal;
 }
 
 @font-face {
-    font-family: 'roboto';
+    font-family: 'roboto-regular';
-    src: url('../fonts/roboto/roboto-bold.woff') format('woff');
+    src: url('/static/fonts/roboto/roboto-regular.woff') format('woff');
+    font-weight: normal;
+    font-style: normal;
+}
+
+@font-face {
+    font-family: 'roboto-bold';
+    src: url('/static/fonts/roboto/roboto-bold.woff') format('woff');
     font-weight: bold;
     font-style: normal;
 }

tubesync/common/static/styles/_template.scss

@@ -181,10 +181,8 @@ main {
             display: inline-block;
             font-weight: bold;
             text-decoration: none;
-            padding: 5px 8px 4px 8px;
+            padding: 5px 10px 5px 10px;
-            margin: 0 3px 6px 3px;
+            margin: 0 3px 0 3px;
-            min-width: 40px;
-            min-height: 40px;
             background-color: $pagination-background-colour;
             color: $pagination-text-colour;
             border: 2px $pagination-border-colour solid;

tubesync/common/static/styles/_variables.scss

@@ -1,2 +1,2 @@
-$font-family: 'roboto', Arial, Helvetica, sans-serif;
+$font-family: 'roboto-regular', Arial, Helvetica, sans-serif;
 $font-size: 1.05rem;

tubesync/common/static/styles/fontawesome/_icons.scss

@@ -65,7 +65,6 @@ readers do not read off random characters that represent icons */
 .#{$fa-css-prefix}-arrows-alt-h:before { content: fa-content($fa-var-arrows-alt-h); }
 .#{$fa-css-prefix}-arrows-alt-v:before { content: fa-content($fa-var-arrows-alt-v); }
 .#{$fa-css-prefix}-artstation:before { content: fa-content($fa-var-artstation); }
-.#{$fa-css-prefix}-arrow-rotate-right:before { content: fa-content($fa-var-arrow-rotate-right); }
 .#{$fa-css-prefix}-assistive-listening-systems:before { content: fa-content($fa-var-assistive-listening-systems); }
 .#{$fa-css-prefix}-asterisk:before { content: fa-content($fa-var-asterisk); }
 .#{$fa-css-prefix}-asymmetrik:before { content: fa-content($fa-var-asymmetrik); }

tubesync/common/static/styles/fontawesome/_variables.scss

@@ -80,7 +80,6 @@ $fa-var-arrow-right: \f061;
 $fa-var-arrow-up: \f062;
 $fa-var-arrows-alt: \f0b2;
 $fa-var-arrows-alt-h: \f337;
-$fa-var-arrow-rotate-right: \f01e;
 $fa-var-arrows-alt-v: \f338;
 $fa-var-artstation: \f77a;
 $fa-var-assistive-listening-systems: \f2a2;

tubesync/common/static/styles/materializecss/components/forms/_checkboxes.scss

@@ -14,7 +14,7 @@
   // Text Label Style
   + span:not(.lever) {
     position: relative;
-    padding-left: 27px;
+    padding-left: 35px;
     cursor: pointer;
     display: inline-block;
     height: 25px;

tubesync/common/static/styles/tubesync.scss

@@ -17,16 +17,3 @@ html {
     visibility: visible;
     opacity: 1;
 }
-
-.flex-collection-container {
-    display: flex !important;
-    align-items: center;
-}
-
-.flex-grow {
-    flex-grow: 1;
-}
-
-.help-text > i {
-    padding-right: 6px;
-}

tubesync/common/templates/base.html

@@ -57,7 +57,7 @@
         </p>
         <p>
           <a href="https://github.com/meeb/tubesync" class="nowrap" target="_blank"><i class="fab fa-github"></i> TubeSync</a> version <strong>{{ app_version }}</strong> with
-          <a href="https://github.com/yt-dlp/yt-dlp" class="nowrap" target="_blank"><i class="fas fa-link"></i> yt-dlp</a> version <strong>{{ yt_dlp_version }}</strong> and 
+          <a href="https://yt-dl.org/" class="nowrap" target="_blank"><i class="fas fa-link"></i> youtube-dl</a> version <strong>{{ youtube_dl_version }}</strong> and 
           <a href="https://ffmpeg.org/" class="nowrap" target="_blank"><i class="fas fa-link"></i> FFmpeg</a> version <strong>{{ ffmpeg_version }}</strong>. 
         </p>
       </div>

tubesync/common/templates/pagination.html

@@ -3,7 +3,7 @@
   <div class="col s12">
     <div class="pagination">
       {% for i in paginator.page_range %}
-        <a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}{% if show_skipped %}&show_skipped=yes{% endif %}{% if only_skipped %}&only_skipped=yes{% endif %}">{{ i }}</a>
+        <a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}">{{ i }}</a>
       {% endfor %}
     </div>
   </div>

tubesync/common/tests.py

@@ -2,8 +2,6 @@ import os.path
 from django.conf import settings
 from django.test import TestCase, Client
 from .testutils import prevent_request_warnings
-from .utils import parse_database_connection_string, clean_filename
-from .errors import DatabaseConnectionError
 
 
 class ErrorPageTestCase(TestCase):
@@ -63,75 +61,3 @@ class CommonStaticTestCase(TestCase):
         favicon_real_path = os.path.join(os.sep.join(root_parts),
                                          os.sep.join(url_parts))
         self.assertTrue(os.path.exists(favicon_real_path))
-
-
-class UtilsTestCase(TestCase):
-
-    def test_parse_database_connection_string(self):
-        database_dict = parse_database_connection_string(
-            'postgresql://tubesync:password@localhost:5432/tubesync')
-        self.assertEqual(database_dict,
-            {
-                'DRIVER': 'postgresql',
-                'ENGINE': 'django.db.backends.postgresql',
-                'USER': 'tubesync',
-                'PASSWORD': 'password',
-                'HOST': 'localhost',
-                'PORT': 5432,
-                'NAME': 'tubesync',
-                'CONN_MAX_AGE': 300,
-                'OPTIONS': {},
-            }
-        )
-        database_dict = parse_database_connection_string(
-            'mysql://tubesync:password@localhost:3306/tubesync')
-        self.assertEqual(database_dict,
-            {
-                'DRIVER': 'mysql',
-                'ENGINE': 'django.db.backends.mysql',
-                'USER': 'tubesync',
-                'PASSWORD': 'password',
-                'HOST': 'localhost',
-                'PORT': 3306,
-                'NAME': 'tubesync',
-                'CONN_MAX_AGE': 300,
-                'OPTIONS': {'charset': 'utf8mb4'}
-            }
-        )
-        # Invalid driver
-        with self.assertRaises(DatabaseConnectionError):
-            parse_database_connection_string(
-                'test://tubesync:password@localhost:5432/tubesync')
-        # No username
-        with self.assertRaises(DatabaseConnectionError):
-            parse_database_connection_string(
-                'postgresql://password@localhost:5432/tubesync')
-        # No database name
-        with self.assertRaises(DatabaseConnectionError):
-            parse_database_connection_string(
-                'postgresql://tubesync:password@5432')
-        # Invalid port
-        with self.assertRaises(DatabaseConnectionError):
-            parse_database_connection_string(
-                'postgresql://tubesync:password@localhost:test/tubesync')
-        # Invalid port
-        with self.assertRaises(DatabaseConnectionError):
-            parse_database_connection_string(
-                'postgresql://tubesync:password@localhost:65537/tubesync')
-        # Invalid username or password
-        with self.assertRaises(DatabaseConnectionError):
-            parse_database_connection_string(
-                'postgresql://tubesync:password:test@localhost:5432/tubesync')
-        # Invalid database name
-        with self.assertRaises(DatabaseConnectionError):
-            parse_database_connection_string(
-                'postgresql://tubesync:password@localhost:5432/tubesync/test')
-
-    def test_clean_filename(self):
-        self.assertEqual(clean_filename('a'), 'a')
-        self.assertEqual(clean_filename('a\t'), 'a')
-        self.assertEqual(clean_filename('a\n'), 'a')
-        self.assertEqual(clean_filename('a a'), 'a a')
-        self.assertEqual(clean_filename('a  a'), 'a  a')
-        self.assertEqual(clean_filename('a\t\t\ta'), 'a   a')
-        self.assertEqual(clean_filename('a\t\t\ta\t\t\t'), 'a   a')

tubesync/common/third_party_versions.py

@@ -1,7 +1,7 @@
-from yt_dlp import version as yt_dlp_version
+from youtube_dl import version as yt_version
 
 
-yt_dlp_version = str(yt_dlp_version.__version__)
+youtube_dl_version = str(yt_version.__version__)
 ffmpeg_version = '(shared install)'
 
 

tubesync/common/utils.py

@@ -1,96 +1,4 @@
-import string
+from urllib.parse import urlunsplit, urlencode
-from datetime import datetime
-from urllib.parse import urlunsplit, urlencode, urlparse
-from yt_dlp.utils import LazyList
-from .errors import DatabaseConnectionError
-
-
-def parse_database_connection_string(database_connection_string):
-    '''
-        Parses a connection string in a URL style format, such as:
-            postgresql://tubesync:password@localhost:5432/tubesync
-            mysql://someuser:somepassword@localhost:3306/tubesync
-        into a Django-compatible settings.DATABASES dict format. 
-    '''
-    valid_drivers = ('postgresql', 'mysql')
-    default_ports = {
-        'postgresql': 5432,
-        'mysql': 3306,
-    }
-    django_backends = {
-        'postgresql': 'django.db.backends.postgresql',
-        'mysql': 'django.db.backends.mysql',
-    }
-    backend_options = {
-        'postgresql': {},
-        'mysql': {
-            'charset': 'utf8mb4',
-        }
-    }
-    try:
-        parts = urlparse(str(database_connection_string))
-    except Exception as e:
-        raise DatabaseConnectionError(f'Failed to parse "{database_connection_string}" '
-                                      f'as a database connection string: {e}') from e
-    driver = parts.scheme
-    user_pass_host_port = parts.netloc
-    database = parts.path
-    if driver not in valid_drivers:
-        raise DatabaseConnectionError(f'Database connection string '
-                                      f'"{database_connection_string}" specified an '
-                                      f'invalid driver, must be one of {valid_drivers}')
-    django_driver = django_backends.get(driver)
-    host_parts = user_pass_host_port.split('@')
-    if len(host_parts) != 2:
-        raise DatabaseConnectionError(f'Database connection string netloc must be in '
-                                      f'the format of user:pass@host')
-    user_pass, host_port = host_parts
-    user_pass_parts = user_pass.split(':')
-    if len(user_pass_parts) != 2:
-        raise DatabaseConnectionError(f'Database connection string netloc must be in '
-                                      f'the format of user:pass@host')
-    username, password = user_pass_parts
-    host_port_parts = host_port.split(':')
-    if len(host_port_parts) == 1:
-        # No port number, assign a default port
-        hostname = host_port_parts[0]
-        port = default_ports.get(driver)
-    elif len(host_port_parts) == 2:
-        # Host name and port number
-        hostname, port = host_port_parts
-        try:
-            port = int(port)
-        except (ValueError, TypeError) as e:
-            raise DatabaseConnectionError(f'Database connection string contained an '
-                                          f'invalid port, ports must be integers: '
-                                          f'{e}') from e
-        if not 0 < port < 63336:
-            raise DatabaseConnectionError(f'Database connection string contained an '
-                                          f'invalid port, ports must be between 1 and '
-                                          f'65535, got {port}')
-    else:
-        # Malformed
-        raise DatabaseConnectionError(f'Database connection host must be a hostname or '
-                                      f'a hostname:port combination')
-    if database.startswith('/'):
-        database = database[1:]
-    if not database:
-        raise DatabaseConnectionError(f'Database connection string path must be a '
-                                      f'string in the format of /databasename')    
-    if '/' in database:
-        raise DatabaseConnectionError(f'Database connection string path can only '
-                                      f'contain a single string name, got: {database}')
-    return {
-        'DRIVER': driver,
-        'ENGINE': django_driver,
-        'NAME': database,
-        'USER': username,
-        'PASSWORD': password,
-        'HOST': hostname,
-        'PORT': port,
-        'CONN_MAX_AGE': 300,
-        'OPTIONS': backend_options.get(driver),
-    }
 
 
 def get_client_ip(request):
@@ -114,18 +22,5 @@ def clean_filename(filename):
     to_scrub = '<>\/:*?"|%'
     for char in to_scrub:
         filename = filename.replace(char, '')
-    clean_filename = ''
+    filename = ''.join([c for c in filename if ord(c) > 30])
-    for c in filename:
+    return ' '.join(filename.split())
-        if c in string.whitespace:
-            c = ' '
-        if ord(c) > 30:
-            clean_filename += c
-    return clean_filename.strip()
-
-
-def json_serial(obj):
-    if isinstance(obj, datetime):
-        return obj.isoformat()
-    if isinstance(obj, LazyList):
-        return list(obj)
-    raise TypeError(f'Type {type(obj)} is not json_serial()-able')

tubesync/sync/fields.py

@@ -1,109 +0,0 @@
-from django.forms import MultipleChoiceField,  CheckboxSelectMultiple, Field, TypedMultipleChoiceField
-from django.db import models
-from typing import Any, Optional, Dict
-from django.utils.translation import gettext_lazy as _
-
-# this is a form field!
-class CustomCheckboxSelectMultiple(CheckboxSelectMultiple):
-    template_name = 'widgets/checkbox_select.html'
-    option_template_name = 'widgets/checkbox_option.html'
-
-    def get_context(self, name: str, value: Any, attrs) -> Dict[str, Any]:
-        ctx = super().get_context(name, value, attrs)['widget']
-        ctx["multipleChoiceProperties"] = []
-        for _group, options, _index in ctx["optgroups"]:
-            for option in options:
-                if not isinstance(value,str) and not isinstance(value,list) and ( option["value"] in value.selected_choices or ( value.allow_all and value.all_choice in value.selected_choices ) ):
-                    checked = True
-                else:
-                    checked = False
-
-                ctx["multipleChoiceProperties"].append({
-                    "template_name": option["template_name"],
-                    "type": option["type"],
-                    "value": option["value"],
-                    "label": option["label"],
-                    "name": option["name"],
-                    "checked": checked})
-
-        return { 'widget': ctx }
-
-# this is a database field!
-class CommaSepChoiceField(models.Field):
-    "Implements comma-separated storage of lists"
-
-    def __init__(self, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, *args, **kwargs):
-        self.separator = separator
-        self.possible_choices = possible_choices
-        self.selected_choices = []
-        self.allow_all = allow_all
-        self.all_label = all_label
-        self.all_choice = all_choice
-        super().__init__(*args, **kwargs)
-
-    def deconstruct(self):
-        name, path, args, kwargs = super().deconstruct()
-        if self.separator != ",":
-            kwargs['separator'] = self.separator
-        kwargs['possible_choices'] = self.possible_choices
-        return name, path, args, kwargs
-
-    def db_type(self, connection):
-        return 'text'
-
-    def get_my_choices(self):
-        choiceArray = []
-        if self.possible_choices is None:
-            return choiceArray
-        if self.allow_all:
-            choiceArray.append((self.all_choice, _(self.all_label)))
-
-        for t in self.possible_choices:
-            choiceArray.append(t)
-        
-        return choiceArray
-
-    def formfield(self, **kwargs):
-        # This is a fairly standard way to set up some defaults
-        # while letting the caller override them.
-        defaults = {'form_class': MultipleChoiceField, 
-                    'choices': self.get_my_choices,
-                    'widget': CustomCheckboxSelectMultiple,
-                    'label': '',
-                    'required': False}
-        defaults.update(kwargs)
-        #del defaults.required
-        return super().formfield(**defaults)
-
-    def deconstruct(self):
-        name, path, args, kwargs = super().deconstruct()
-        # Only include kwarg if it's not the default
-        if self.separator != ",":
-            kwargs['separator'] = self.separator
-        return name, path, args, kwargs
-
-    def from_db_value(self, value, expr, conn):
-        if value is None:
-            self.selected_choices = []
-        else:
-            self.selected_choices = value.split(",")
-
-        return self
-
-    def get_prep_value(self, value):
-        if value is None:
-            return ""
-        if not isinstance(value,list):
-            return ""
-
-        if self.all_choice not in value:
-            return ",".join(value)
-        else:
-            return self.all_choice
-
-    def get_text_for_value(self, val):
-        fval = [i for i in self.possible_choices if i[0] == val]
-        if len(fval) <= 0:
-            return []
-        else:
-            return fval[0][1]

tubesync/sync/management/commands/delete-source.py

@@ -1,51 +0,0 @@
-import os
-import uuid
-from django.utils.translation import gettext_lazy as _
-from django.core.management.base import BaseCommand, CommandError
-from django.db.models import signals
-from common.logger import log
-from sync.models import Source, Media, MediaServer
-from sync.signals import media_post_delete
-from sync.tasks import rescan_media_server
-
-
-class Command(BaseCommand):
-
-    help = ('Deletes a source by UUID')
-
-    def add_arguments(self, parser):
-        parser.add_argument('--source', action='store', required=True, help='Source UUID')
-
-    def handle(self, *args, **options):
-        source_uuid_str = options.get('source', '')
-        try:
-            source_uuid = uuid.UUID(source_uuid_str)
-        except Exception as e:
-            raise CommandError(f'Failed to parse source UUID: {e}')
-        log.info(f'Deleting source with UUID: {source_uuid}')
-        # Fetch the source by UUID
-        try:
-            source = Source.objects.get(uuid=source_uuid)
-        except Source.DoesNotExist:
-            raise CommandError(f'Source does not exist with '
-                               f'UUID: {source_uuid}')
-        # Detach post-delete signal for Media so we don't spam media servers
-        signals.post_delete.disconnect(media_post_delete, sender=Media)
-        # Delete the source, triggering pre-delete signals for each media item
-        log.info(f'Found source with UUID "{source.uuid}" with name '
-                 f'"{source.name}" and deleting it, this may take some time!')
-        source.delete()
-        # Update any media servers
-        for mediaserver in MediaServer.objects.all():
-            log.info(f'Scheduling media server updates')
-            verbose_name = _('Request media server rescan for "{}"')
-            rescan_media_server(
-                str(mediaserver.pk),
-                priority=0,
-                verbose_name=verbose_name.format(mediaserver),
-                remove_existing_tasks=True
-            )
-        # Re-attach signals
-        signals.post_delete.connect(media_post_delete, sender=Media)
-        # All done
-        log.info('Done')

tubesync/sync/management/commands/list-sources.py

@@ -1,15 +0,0 @@
-import os
-from django.core.management.base import BaseCommand, CommandError
-from common.logger import log
-from sync.models import Source, Media, MediaServer
-
-
-class Command(BaseCommand):
-
-    help = ('Lists sources')
-
-    def handle(self, *args, **options):
-        log.info('Listing sources...')
-        for source in Source.objects.all():
-            log.info(f' - {source.uuid}: {source.name}')
-        log.info('Done')

tubesync/sync/management/commands/reset-metadata.py

@@ -1,19 +0,0 @@
-from django.core.management.base import BaseCommand
-from sync.models import Media
-
-
-from common.logger import log
-
-
-class Command(BaseCommand):
-
-    help = 'Resets all media item metadata'
-
-    def handle(self, *args, **options):
-        log.info('Resettings all media metadata...')
-        # Delete all metadata
-        Media.objects.update(metadata=None)
-        # Trigger the save signal on each media item
-        for item in Media.objects.all():
-            item.save()
-        log.info('Done')

tubesync/sync/management/commands/reset-tasks.py

@@ -1,33 +0,0 @@
-from django.core.management.base import BaseCommand, CommandError
-from django.utils.translation import gettext_lazy as _
-from background_task.models import Task
-from sync.models import Source
-from sync.tasks import index_source_task
-
-
-from common.logger import log
-
-
-class Command(BaseCommand):
-
-    help = 'Resets all tasks'
-
-    def handle(self, *args, **options):
-        log.info('Resettings all tasks...')
-        # Delete all tasks
-        Task.objects.all().delete()
-        # Iter all tasks
-        for source in Source.objects.all():
-            # Recreate the initial indexing task
-            log.info(f'Resetting tasks for source: {source}')
-            verbose_name = _('Index media from source "{}"')
-            index_source_task(
-                str(source.pk),
-                repeat=source.index_schedule,
-                queue=str(source.pk),
-                priority=5,
-                verbose_name=verbose_name.format(source.name)
-            )
-            # This also chains down to call each Media objects .save() as well
-            source.save()
-        log.info('Done')

tubesync/sync/management/commands/youtube-dl-info.py

@@ -1,7 +1,6 @@
 import json
 from django.core.management.base import BaseCommand, CommandError
 from sync.youtube import get_media_info
-from common.utils import json_serial
 
 
 class Command(BaseCommand):
@@ -15,6 +14,5 @@ class Command(BaseCommand):
         url = options['url']
         self.stdout.write(f'Showing information for URL: {url}')
         info = get_media_info(url)
-        d = json.dumps(info, indent=4, sort_keys=True, default=json_serial)
+        self.stdout.write(json.dumps(info, indent=4, sort_keys=True))
-        self.stdout.write(d)
         self.stdout.write('Done')

tubesync/sync/matching.py

@@ -53,8 +53,6 @@ def get_best_audio_format(media):
         # If the format has a video stream, skip it
         if fmt['vcodec'] is not None:
             continue
-        if not fmt['acodec']:
-            continue
         audio_formats.append(fmt)
     audio_formats = list(reversed(sorted(audio_formats, key=lambda k: k['abr'])))
     if not audio_formats:
@@ -90,8 +88,6 @@ def get_best_video_format(media):
         # If the format has an audio stream, skip it
         if fmt['acodec'] is not None:
             continue
-        if not fmt['vcodec']:
-            continue
         if media.source.source_resolution.strip().upper() == fmt['format']:
             video_formats.append(fmt)
     # Check we matched some streams

tubesync/sync/mediaservers.py

@@ -44,9 +44,7 @@ class PlexMediaServer(MediaServer):
              '<p>The <strong>libraries</strong> is a comma-separated list of Plex '
              'library or section IDs, you can find out how to get your library or '
              'section IDs <a href="https://support.plex.tv/articles/201242707-plex-'
-             'media-scanner-via-command-line/#toc-1" target="_blank">here</a> or '
+             'media-scanner-via-command-line/#toc-1" target="_blank">here</a>.</p>')
-             '<a href="https://www.plexopedia.com/plex-media-server/api/server/libraries/" '
-             'target="_blank">here</a></p>.')
 
     def make_request(self, uri='/', params={}):
         headers = {'User-Agent': 'TubeSync'}
@@ -126,7 +124,7 @@ class PlexMediaServer(MediaServer):
         # Seems we have a valid library sections page, get the library IDs
         remote_libraries = {}
         try:
-            for parent in parsed_response.iter('MediaContainer'):
+            for parent in parsed_response.getiterator('MediaContainer'):
                 for d in parent:
                     library_id = d.attrib['key']
                     library_name = d.attrib['title']

tubesync/sync/migrations/0010_auto_20210924_0554.py

@@ -1,30 +0,0 @@
-# Generated by Django 3.2.7 on 2021-09-24 05:54
-
-import django.core.files.storage
-from django.db import migrations, models
-import sync.models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('sync', '0009_auto_20210218_0442'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='media',
-            name='media_file',
-            field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'),
-        ),
-        migrations.AlterField(
-            model_name='source',
-            name='index_schedule',
-            field=models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours'), (259200, 'Every 3 days'), (604800, 'Every 7 days'), (0, 'Never')], db_index=True, default=86400, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule'),
-        ),
-        migrations.AlterField(
-            model_name='source',
-            name='media_format',
-            field=models.CharField(default='{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'),
-        ),
-    ]

tubesync/sync/migrations/0011_auto_20220201_1654.py

@@ -1,21 +0,0 @@
-# Generated by Django 3.2.11 on 2022-02-01 16:54
-
-import django.core.files.storage
-from django.db import migrations, models
-import sync.models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('sync', '0010_auto_20210924_0554'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='source',
-            name='write_json',
-            field=models.BooleanField(
-                default=False, help_text='Write a JSON file with the media info, these may be detected and used by some media servers', verbose_name='write json'),
-        ),
-    ]

tubesync/sync/migrations/0012_alter_media_downloaded_format.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.2.12 on 2022-04-06 06:19
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('sync', '0011_auto_20220201_1654'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='media',
-            name='downloaded_format',
-            field=models.CharField(blank=True, help_text='Video format (resolution) of the downloaded media', max_length=30, null=True, verbose_name='downloaded format'),
-        ),
-    ]

tubesync/sync/migrations/0013_fix_elative_media_file.py

@@ -1,25 +0,0 @@
-# Generated by Django 3.2.12 on 2022-04-06 06:19
-
-from django.conf import settings
-from django.db import migrations, models
-
-
-def fix_media_file(apps, schema_editor):
-    Media = apps.get_model('sync', 'Media')
-    for media in Media.objects.filter(downloaded=True):
-        download_dir = str(settings.DOWNLOAD_ROOT)
-
-        if media.media_file.name.startswith(download_dir):
-            media.media_file.name = media.media_file.name[len(download_dir) + 1:]
-            media.save()
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('sync', '0012_alter_media_downloaded_format'),
-    ]
-
-    operations = [
-        migrations.RunPython(fix_media_file)
-    ]

tubesync/sync/migrations/0014_alter_media_media_file.py

@@ -1,21 +0,0 @@
-# Generated by Django 3.2.15 on 2022-12-28 20:33
-
-import django.core.files.storage
-from django.conf import settings
-from django.db import migrations, models
-import sync.models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('sync', '0013_fix_elative_media_file'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='media',
-            name='media_file',
-            field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=str(settings.DOWNLOAD_ROOT)), upload_to=sync.models.get_media_file_path, verbose_name='media file'),
-        ),
-    ]

tubesync/sync/migrations/0015_auto_20230213_0603.py

@@ -1,23 +0,0 @@
-# Generated by Django 3.2.17 on 2023-02-13 06:03
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('sync', '0014_alter_media_media_file'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='media',
-            name='manual_skip',
-            field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\' be downloaded', verbose_name='manual_skip'),
-        ),
-        migrations.AlterField(
-            model_name='media',
-            name='skip',
-            field=models.BooleanField(db_index=True, default=False, help_text='INTERNAL FLAG - Media will be skipped and not downloaded', verbose_name='skip'),
-        ),
-    ]

tubesync/sync/migrations/0016_auto_20230214_2052.py

@@ -1,34 +0,0 @@
-# Generated by Django 3.2.18 on 2023-02-14 20:52
-
-from django.db import migrations, models
-import sync.models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('sync', '0015_auto_20230213_0603'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='source',
-            name='embed_metadata',
-            field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'),
-        ),
-        migrations.AddField(
-            model_name='source',
-            name='embed_thumbnail',
-            field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'),
-        ),
-        migrations.AddField(
-            model_name='source',
-            name='enable_sponsorblock',
-            field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'),
-        ),
-        migrations.AddField(
-            model_name='source',
-            name='sponsorblock_categories',
-            field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))),
-        ),
-    ]

tubesync/sync/migrations/0017_alter_source_sponsorblock_categories.py

@@ -1,19 +0,0 @@
-# Generated by Django 3.2.18 on 2023-02-20 02:23
-
-from django.db import migrations
-import sync.fields
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('sync', '0016_auto_20230214_2052'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='source',
-            name='sponsorblock_categories',
-            field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', separator=''),
-        ),
-    ]

tubesync/sync/migrations/0018_source_subtitles.py

@@ -1,27 +0,0 @@
-# Generated by pac
-
-from django.db import migrations, models
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('sync', '0017_alter_source_sponsorblock_categories'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='source',
-            name='write_subtitles',
-            field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'),
-        ),
-        migrations.AddField(
-            model_name='source',
-            name='auto_subtitles',
-            field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto subtitles'),
-        ),
-        migrations.AddField(
-            model_name='source',
-            name='sub_langs',
-            field=models.CharField(default='en', help_text='List of subtitles langs to download comma-separated. Example: en,fr',max_length=30),
-        ),
-    ]

tubesync/sync/migrations/0019_add_delete_removed_media.py

@@ -1,17 +0,0 @@
-# Generated by pac
-
-from django.db import migrations, models
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('sync', '0018_source_subtitles'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='source',
-            name='delete_removed_media',
-            field=models.BooleanField(default=False, help_text='Delete media that is no longer on this playlist', verbose_name='delete removed media'),
-        ),
-    ]

tubesync/sync/migrations/0020_auto_20231024_1825.py

@@ -1,29 +0,0 @@
-# Generated by Django 3.2.22 on 2023-10-24 17:25
-
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('sync', '0019_add_delete_removed_media'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='source',
-            name='filter_text',
-            field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=100, verbose_name='filter string'),
-        ),
-        migrations.AlterField(
-            model_name='source',
-            name='auto_subtitles',
-            field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto-generated subs'),
-        ),
-        migrations.AlterField(
-            model_name='source',
-            name='sub_langs',
-            field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z]+,)*(\\-?[\\_\\.a-zA-Z]+){1}$')], verbose_name='subs langs'),
-        ),
-    ]

tubesync/sync/migrations/0021_add_delete_files_on_disk.py

@@ -1,17 +0,0 @@
-# Generated by pac
-
-from django.db import migrations, models
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('sync', '0020_auto_20231024_1825'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='source',
-            name='delete_files_on_disk',
-            field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'),
-        ),
-    ]

tubesync/sync/models.py

@@ -1,7 +1,6 @@
 import os
 import uuid
 import json
-import re
 from xml.etree import ElementTree
 from collections import OrderedDict
 from datetime import datetime, timedelta
@@ -9,7 +8,6 @@ from pathlib import Path
 from django.conf import settings
 from django.db import models
 from django.core.files.storage import FileSystemStorage
-from django.core.validators import RegexValidator
 from django.utils.text import slugify
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
@@ -18,12 +16,13 @@ from common.utils import clean_filename
 from .youtube import (get_media_info as get_youtube_media_info,
                       download_media as download_youtube_media)
 from .utils import seconds_to_timestr, parse_media_format
-from .matching import (get_best_combined_format, get_best_audio_format,
+from .matching import (get_best_combined_format, get_best_audio_format, 
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
-from .fields import CommaSepChoiceField
 
-media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/')
+
+media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT))
+
 
 class Source(models.Model):
     '''
@@ -107,43 +106,6 @@ class Source(models.Model):
     EXTENSION_MKV = 'mkv'
     EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
 
-    # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
-    SPONSORBLOCK_CATEGORIES_CHOICES = (
-        ('sponsor', 'Sponsor'),
-        ('intro', 'Intermission/Intro Animation'),
-        ('outro', 'Endcards/Credits'),
-        ('selfpromo', 'Unpaid/Self Promotion'),
-        ('preview', 'Preview/Recap'),
-        ('filler', 'Filler Tangent'),
-        ('interaction', 'Interaction Reminder'),
-        ('music_offtopic', 'Non-Music Section'),
-    )
-
-    sponsorblock_categories = CommaSepChoiceField(
-        _(''),
-        possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
-        all_choice='all',
-        allow_all=True,
-        all_label='(all options)',
-        default='all',
-        help_text=_('Select the sponsorblocks you want to enforce')
-    )
-    embed_metadata = models.BooleanField(
-        _('embed metadata'),
-        default=False,
-        help_text=_('Embed metadata from source into file')
-    )
-    embed_thumbnail = models.BooleanField(
-        _('embed thumbnail'),
-        default=False,
-        help_text=_('Embed thumbnail into the file')
-    )
-    enable_sponsorblock = models.BooleanField(
-        _('enable sponsorblock'),
-        default=True,
-        help_text=_('Use SponsorBlock?')
-    )
-
     # Fontawesome icons used for the source on the front end
     ICONS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
@@ -196,9 +158,6 @@ class Source(models.Model):
         EVERY_6_HOURS = 21600, _('Every 6 hours')
         EVERY_12_HOURS = 43200, _('Every 12 hours')
         EVERY_24_HOURS = 86400, _('Every 24 hours')
-        EVERY_3_DAYS = 259200, _('Every 3 days')
-        EVERY_7_DAYS = 604800, _('Every 7 days')
-        NEVER = 0, _('Never')
 
     uuid = models.UUIDField(
         _('uuid'),
@@ -259,7 +218,7 @@ class Source(models.Model):
         _('index schedule'),
         choices=IndexSchedule.choices,
         db_index=True,
-        default=IndexSchedule.EVERY_24_HOURS,
+        default=IndexSchedule.EVERY_6_HOURS,
         help_text=_('Schedule of how often to index the source for new media')
     )
     download_media = models.BooleanField(
@@ -284,23 +243,6 @@ class Source(models.Model):
         help_text=_('If "delete old media" is ticked, the number of days after which '
                     'to automatically delete media')
     )
-    filter_text = models.CharField(
-        _('filter string'),
-        max_length=100,
-        default='',
-        blank=True,
-        help_text=_('Regex compatible filter string for video titles')
-    )
-    delete_removed_media = models.BooleanField(
-        _('delete removed media'),
-        default=False,
-        help_text=_('Delete media that is no longer on this playlist')
-    )
-    delete_files_on_disk = models.BooleanField(
-        _('delete files on disk'),
-        default=False,
-        help_text=_('Delete files on disk when they are removed from TubeSync')
-    )
     source_resolution = models.CharField(
         _('source resolution'),
         max_length=8,
@@ -353,41 +295,12 @@ class Source(models.Model):
         default=False,
         help_text=_('Write an NFO file in XML with the media info, these may be detected and used by some media servers')
     )
-    write_json = models.BooleanField(
-        _('write json'),
-        default=False,
-        help_text=_('Write a JSON file with the media info, these may be detected and used by some media servers')
-    )
     has_failed = models.BooleanField(
         _('has failed'),
         default=False,
         help_text=_('Source has failed to index media')
     )
 
-    write_subtitles = models.BooleanField(
-        _('write subtitles'),
-        default=False,
-        help_text=_('Download video subtitles')
-    )
-
-    auto_subtitles = models.BooleanField(
-        _('accept auto-generated subs'),
-        default=False,
-        help_text=_('Accept auto-generated subtitles')
-    )
-    sub_langs = models.CharField(
-        _('subs langs'),
-        max_length=30,
-        default='en',
-        help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'),
-        validators=[
-            RegexValidator(
-                regex=r"^(\-?[\_\.a-zA-Z]+,)*(\-?[\_\.a-zA-Z]+){1}$",
-                message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat')
-            )
-        ]
-    )
-
     def __str__(self):
         return self.name
 
@@ -471,14 +384,10 @@ class Source(models.Model):
     @property
     def directory_path(self):
         download_dir = Path(media_file_storage.location)
-        return download_dir / self.type_directory_path
-
-    @property
-    def type_directory_path(self):
         if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO:
-            return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory
+            return download_dir / settings.DOWNLOAD_AUDIO_DIR / self.directory
         else:
-            return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory
+            return download_dir / settings.DOWNLOAD_VIDEO_DIR / self.directory
 
     def make_directory(self):
         return os.makedirs(self.directory_path, exist_ok=True)
@@ -516,20 +425,19 @@ class Source(models.Model):
             fmt.append('60fps')
         if self.prefer_hdr:
             fmt.append('hdr')
-        now = timezone.now()
         return {
-            'yyyymmdd': now.strftime('%Y%m%d'),
+            'yyyymmdd': timezone.now().strftime('%Y%m%d'),
-            'yyyy_mm_dd': now.strftime('%Y-%m-%d'),
+            'yyyy_mm_dd': timezone.now().strftime('%Y-%m-%d'),
-            'yyyy': now.strftime('%Y'),
+            'yyyy': timezone.now().strftime('%Y'),
-            'mm': now.strftime('%m'),
+            'mm': timezone.now().strftime('%m'),
-            'dd': now.strftime('%d'),
+            'dd': timezone.now().strftime('%d'),
             'source': self.slugname,
             'source_full': self.name,
-            'uploader': 'Some Channel Name',
             'title': 'some-media-title-name',
             'title_full': 'Some Media Title Name',
             'key': 'SoMeUnIqUiD',
             'format': '-'.join(fmt),
+            'playlist_index': 1,
             'playlist_title': 'Some Playlist Title',
             'ext': self.extension,
             'resolution': self.source_resolution if self.source_resolution else '',
@@ -547,11 +455,6 @@ class Source(models.Model):
         except Exception as e:
             return ''
 
-    def is_regex_match(self, media_item_title):
-        if not self.filter_text:
-            return True
-        return bool(re.search(self.filter_text, media_item_title))
-    
     def index_media(self):
         '''
             Index the media source returning a list of media metadata as dicts.
@@ -562,11 +465,7 @@ class Source(models.Model):
         response = indexer(self.index_url)
         if not isinstance(response, dict):
             return []
-        entries = response.get('entries', [])
+        return response.get('entries', [])
-
-        if settings.MAX_ENTRIES_PROCESSING:
-            entries = entries[:settings.MAX_ENTRIES_PROCESSING]
-        return entries
 
 
 def get_media_thumb_path(instance, filename):
@@ -660,6 +559,11 @@ class Media(models.Model):
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count',
         },
+        'playlist_index': {
+            Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_index',
+            Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_index',
+            Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'playlist_index',
+        },
         'playlist_title': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title',
@@ -754,7 +658,7 @@ class Media(models.Model):
     media_file = models.FileField(
         _('media file'),
         upload_to=get_media_file_path,
-        max_length=255,
+        max_length=200,
         blank=True,
         null=True,
         storage=media_file_storage,
@@ -764,13 +668,7 @@ class Media(models.Model):
         _('skip'),
         db_index=True,
         default=False,
-        help_text=_('INTERNAL FLAG - Media will be skipped and not downloaded')
+        help_text=_('Media will be skipped and not downloaded')
-    )
-    manual_skip = models.BooleanField(
-        _('manual_skip'),
-        db_index=True,
-        default=False,
-        help_text=_('Media marked as "skipped", won\' be downloaded')
     )
     downloaded = models.BooleanField(
         _('downloaded'),
@@ -790,7 +688,7 @@ class Media(models.Model):
         max_length=30,
         blank=True,
         null=True,
-        help_text=_('Video format (resolution) of the downloaded media')
+        help_text=_('Audio codec of the downloaded media')
     )
     downloaded_height = models.PositiveIntegerField(
         _('downloaded height'),
@@ -917,24 +815,7 @@ class Media(models.Model):
         hdr = ''
         # If the download has completed use existing values
         if self.downloaded:
-            # Check if there's any stored meta data at all
+            resolution = f'{self.downloaded_height}p'
-            if (not self.downloaded_video_codec and \
-                not self.downloaded_audio_codec):
-                # Marked as downloaded but no metadata, imported?
-                return {
-                    'resolution': resolution,
-                    'height': height,
-                    'width': width,
-                    'vcodec': vcodec,
-                    'acodec': acodec,
-                    'fps': fps,
-                    'hdr': hdr,
-                    'format': tuple(fmt),
-                }
-            if self.downloaded_format:
-                resolution = self.downloaded_format.lower()
-            elif self.downloaded_height:
-                resolution = f'{self.downloaded_height}p'
             if self.downloaded_format != 'audio':
                 vcodec = self.downloaded_video_codec.lower()
                 fmt.append(vcodec)
@@ -961,7 +842,7 @@ class Media(models.Model):
         # Otherwise, calculate from matched format codes
         vformat = None
         aformat = None
-        if format_str and '+' in format_str:
+        if '+' in format_str:
             # Seperate audio and video streams
             vformat_code, aformat_code = format_str.split('+')
             vformat = self.get_format_by_code(vformat_code)
@@ -970,7 +851,7 @@ class Media(models.Model):
             # Combined stream or audio only
             cformat = self.get_format_by_code(format_str)
             aformat = cformat
-            if cformat and cformat['vcodec']:
+            if cformat['vcodec']:
                 # Combined
                 vformat = cformat
         if vformat:
@@ -1031,6 +912,7 @@ class Media(models.Model):
             'title_full': clean_filename(self.title),
             'key': self.key,
             'format': '-'.join(display_format['format']),
+            'playlist_index': self.playlist_index,
             'playlist_title': self.playlist_title,
             'ext': self.source.extension,
             'resolution': display_format['resolution'],
@@ -1040,7 +922,6 @@ class Media(models.Model):
             'acodec': display_format['acodec'],
             'fps': display_format['fps'],
             'hdr': display_format['hdr'],
-            'uploader': self.uploader,
         }
 
     @property
@@ -1050,10 +931,7 @@ class Media(models.Model):
     @property
     def loaded_metadata(self):
         try:
-            data = json.loads(self.metadata)
+            return json.loads(self.metadata)
-            if not isinstance(data, dict):
-                return {}
-            return data
         except Exception as e:
             return {}
 
@@ -1090,10 +968,7 @@ class Media(models.Model):
     @property
     def upload_date(self):
         field = self.get_metadata_field('upload_date')
-        try:
+        upload_date_str = self.loaded_metadata.get(field, '').strip()
-            upload_date_str = self.loaded_metadata.get(field, '').strip()
-        except (AttributeError, ValueError) as e:
-            return None
         try:
             return datetime.strptime(upload_date_str, '%Y%m%d')
         except (AttributeError, ValueError) as e:
@@ -1102,12 +977,7 @@ class Media(models.Model):
     @property
     def duration(self):
         field = self.get_metadata_field('duration')
-        duration = self.loaded_metadata.get(field, 0)
+        return int(self.loaded_metadata.get(field, 0))
-        try:
-            duration = int(duration)
-        except (TypeError, ValueError):
-            duration = 0
-        return duration
 
     @property
     def duration_formatted(self):
@@ -1153,6 +1023,11 @@ class Media(models.Model):
         field = self.get_metadata_field('formats')
         return self.loaded_metadata.get(field, [])
 
+    @property
+    def playlist_index(self):
+        field = self.get_metadata_field('playlist_index')
+        return self.loaded_metadata.get(field, 0)
+
     @property
     def playlist_title(self):
         field = self.get_metadata_field('playlist_title')
@@ -1167,10 +1042,7 @@ class Media(models.Model):
 
     @property
     def thumbname(self):
-        if self.downloaded and self.media_file:
+        filename = self.filename
-            filename = os.path.basename(self.media_file.path)
-        else:
-            filename = self.filename
         prefix, ext = os.path.splitext(filename)
         return f'{prefix}.jpg'
 
@@ -1180,10 +1052,7 @@ class Media(models.Model):
 
     @property
     def nfoname(self):
-        if self.downloaded and self.media_file:
+        filename = self.filename
-            filename = os.path.basename(self.media_file.path)
-        else:
-            filename = self.filename
         prefix, ext = os.path.splitext(filename)
         return f'{prefix}.nfo'
     
@@ -1191,19 +1060,6 @@ class Media(models.Model):
     def nfopath(self):
         return self.source.directory_path / self.nfoname
 
-    @property
-    def jsonname(self):
-        if self.downloaded and self.media_file:
-            filename = os.path.basename(self.media_file.path)
-        else:
-            filename = self.filename
-        prefix, ext = os.path.splitext(filename)
-        return f'{prefix}.info.json'
-    
-    @property
-    def jsonpath(self):
-        return self.source.directory_path / self.jsonname
-
     @property
     def directory_path(self):
         # Otherwise, create a suitable filename from the source media_format
@@ -1228,29 +1084,6 @@ class Media(models.Model):
             return False
         return os.path.exists(self.media_file.path)
 
-    @property
-    def content_type(self):
-        if not self.downloaded:
-            return 'video/mp4'
-        vcodec = self.downloaded_video_codec
-        if vcodec is None:
-            acodec = self.downloaded_audio_codec
-            if acodec is None:
-                raise TypeError() # nothing here.
-            acodec = acodec.lower()
-            if acodec == "mp4a":
-                return "audio/mp4"
-            elif acodec == "opus":
-                return "audio/opus"
-            else:
-                # fall-fall-back.
-                return 'audio/ogg'
-        vcodec = vcodec.lower()
-        if vcodec == 'vp9':
-            return 'video/webm'
-        else:
-            return 'video/mp4'
-
     @property
     def nfoxml(self):
         '''
@@ -1268,22 +1101,6 @@ class Media(models.Model):
         showtitle.text = str(self.source.name).strip()
         showtitle.tail = '\n  '
         nfo.append(showtitle)
-        # season = upload date year
-        season = nfo.makeelement('season', {})
-        if self.source.source_type == Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
-            # If it's a playlist, set season to 1
-            season.text = '1'
-        else:
-            # If it's not a playlist, set season to upload date year
-            season.text = str(self.upload_date.year) if self.upload_date else ''
-        season.tail = '\n  '
-        nfo.append(season)
-        # episode = number of video in the year
-        episode = nfo.makeelement('episode', {})
-        episode_number = self.calculate_episode_number()
-        episode.text = str(episode_number) if episode_number else ''
-        episode.tail = '\n  '
-        nfo.append(episode)
         # ratings = media metadata youtube rating
         value = nfo.makeelement('value', {})
         value.text = str(self.rating)
@@ -1391,10 +1208,7 @@ class Media(models.Model):
                                     f'no valid format available')
         # Download the media with youtube-dl
         download_youtube_media(self.url, format_str, self.source.extension,
-                               str(self.filepath), self.source.write_json,
+                               str(self.filepath))
-                               self.source.sponsorblock_categories.selected_choices, self.source.embed_thumbnail,
-                               self.source.embed_metadata, self.source.enable_sponsorblock,
-                              self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs )
         # Return the download paramaters
         return format_str, self.source.extension
 
@@ -1404,23 +1218,10 @@ class Media(models.Model):
         '''
         indexer = self.INDEXERS.get(self.source.source_type, None)
         if not callable(indexer):
-            raise Exception(f'Media with source type f"{self.source.source_type}" '
+            raise Exception(f'Meida with source type f"{self.source.source_type}" '
                             f'has no indexer')
         return indexer(self.url)
 
-    def calculate_episode_number(self):
-        if self.source.source_type == Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
-            sorted_media = Media.objects.filter(source=self.source)
-        else:
-            self_year = self.upload_date.year if self.upload_date else self.created.year
-            filtered_media = Media.objects.filter(source=self.source, published__year=self_year)
-            sorted_media = sorted(filtered_media, key=lambda x: (x.upload_date, x.key))
-        position_counter = 1
-        for media in sorted_media:
-            if media == self:
-                return position_counter
-            position_counter += 1
-
 
 class MediaServer(models.Model):
     '''

tubesync/sync/signals.py

@@ -1,5 +1,4 @@
 import os
-import glob
 from django.conf import settings
 from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
 from django.dispatch import receiver
@@ -48,18 +47,17 @@ def source_post_save(sender, instance, created, **kwargs):
             priority=0,
             verbose_name=verbose_name.format(instance.name)
         )
-        if instance.index_schedule > 0:
+        delete_task_by_source('sync.tasks.index_source_task', instance.pk)
-            delete_task_by_source('sync.tasks.index_source_task', instance.pk)
+        log.info(f'Scheduling media indexing for source: {instance.name}')
-            log.info(f'Scheduling media indexing for source: {instance.name}')
+        verbose_name = _('Index media from source "{}"')
-            verbose_name = _('Index media from source "{}"')
+        index_source_task(
-            index_source_task(
+            str(instance.pk),
-                str(instance.pk),
+            repeat=instance.index_schedule,
-                repeat=instance.index_schedule,
+            queue=str(instance.pk),
-                queue=str(instance.pk),
+            priority=5,
-                priority=5,
+            verbose_name=verbose_name.format(instance.name),
-                verbose_name=verbose_name.format(instance.name),
+            remove_existing_tasks=True
-                remove_existing_tasks=True
+        )
-            )
     # Trigger the post_save signal for each media item linked to this source as various
     # flags may need to be recalculated
     for media in Media.objects.filter(source=instance):
@@ -75,7 +73,6 @@ def source_pre_delete(sender, instance, **kwargs):
         media.delete()
 
 
-
 @receiver(post_delete, sender=Source)
 def source_post_delete(sender, instance, **kwargs):
     # Triggered after a source is deleted
@@ -95,81 +92,18 @@ def task_task_failed(sender, task_id, completed_task, **kwargs):
 
 @receiver(post_save, sender=Media)
 def media_post_save(sender, instance, created, **kwargs):
-    # If the media is skipped manually, bail.
+    # Triggered after media is saved, Recalculate the "can_download" flag, this may
-    if instance.manual_skip:
-        return
-    # Triggered after media is saved
-    cap_changed = False
-    can_download_changed = False
-    # Reset the skip flag if the download cap has changed if the media has not
-    # already been downloaded
-    if not instance.downloaded and instance.metadata:
-        max_cap_age = instance.source.download_cap_date
-        filter_text = instance.source.filter_text.strip()
-        published = instance.published
-        if not published:
-            if not instance.skip:
-                log.warn(f'Media: {instance.source} / {instance} has no published date '
-                         f'set, marking to be skipped')
-                instance.skip = True
-                cap_changed = True
-            else:
-                log.debug(f'Media: {instance.source} / {instance} has no published date '
-                          f'set but is already marked to be skipped')
-        else:
-            if max_cap_age:
-                if published > max_cap_age and instance.skip:
-                    if filter_text:
-                        if instance.source.is_regex_match(instance.title):
-                            log.info(f'Media: {instance.source} / {instance} has a valid '
-                                    f'publishing date and title filter, marking to be unskipped')
-                            instance.skip = False
-                            cap_changed = True
-                        else:
-                            log.debug(f'Media: {instance.source} / {instance} has a valid publishing date '
-                                      f'but failed the title filter match, already marked skipped')
-                    else:
-                        log.info(f'Media: {instance.source} / {instance} has a valid '
-                                 f'publishing date, marking to be unskipped')
-                        instance.skip = False
-                        cap_changed = True
-                elif published <= max_cap_age and not instance.skip:
-                    log.info(f'Media: {instance.source} / {instance} is too old for '
-                            f'the download cap date, marking to be skipped')
-                    instance.skip = True
-                    cap_changed = True
-            else:
-                if instance.skip:
-                    # Media marked to be skipped but source download cap removed
-                    if filter_text:
-                        if instance.source.is_regex_match(instance.title):
-                            log.info(f'Media: {instance.source} / {instance} has a valid '
-                                     f'publishing date and title filter, marking to be unskipped')
-                            instance.skip = False
-                            cap_changed = True
-                        else:
-                            log.info(f'Media: {instance.source} / {instance} has a valid publishing date '
-                                     f'but failed the title filter match, already marked skipped')
-                else:
-                    log.debug(f'Media: {instance.source} / {instance} has a valid publishing date and '
-                              f'is already marked as not to be skipped')
-
-                    cap_changed = False
-    # Recalculate the "can_download" flag, this may
     # need to change if the source specifications have been changed
     if instance.metadata:
+        post_save.disconnect(media_post_save, sender=Media)
         if instance.get_format_str():
             if not instance.can_download:
                 instance.can_download = True
-                can_download_changed = True
+                instance.save()
         else:
             if instance.can_download:
                 instance.can_download = False
-                can_download_changed = True
+                instance.save()
-    # Save the instance if any changes were required
-    if cap_changed or can_download_changed:
-        post_save.disconnect(media_post_save, sender=Media)
-        instance.save()
         post_save.connect(media_post_save, sender=Media)
     # If the media is missing metadata schedule it to be downloaded
     if not instance.metadata:
@@ -177,7 +111,7 @@ def media_post_save(sender, instance, created, **kwargs):
         verbose_name = _('Downloading metadata for "{}"')
         download_media_metadata(
             str(instance.pk),
-            priority=5,
+            priority=10,
             verbose_name=verbose_name.format(instance.pk),
             remove_existing_tasks=True
         )
@@ -224,16 +158,6 @@ def media_pre_delete(sender, instance, **kwargs):
     if thumbnail_url:
         delete_task_by_media('sync.tasks.download_media_thumbnail',
                              (str(instance.pk), thumbnail_url))
-    if instance.source.delete_files_on_disk and (instance.media_file or instance.thumb):
-        # Delete all media files if it contains filename
-        filepath = instance.media_file.path if instance.media_file else instance.thumb.path
-        barefilepath, fileext = os.path.splitext(filepath)
-        # Get all files that start with the bare file path
-        all_related_files = glob.glob(f'{barefilepath}.*')
-        for file in all_related_files:
-            log.info(f'Deleting file for: {instance} path: {file}')
-            delete_file(file)
-
 
 
 @receiver(post_delete, sender=Media)

tubesync/sync/tasks.py

@@ -10,7 +10,7 @@ import math
 import uuid
 from io import BytesIO
 from hashlib import sha1
-from datetime import timedelta, datetime
+from datetime import timedelta
 from shutil import copyfile
 from PIL import Image
 from django.conf import settings
@@ -22,7 +22,6 @@ from background_task import background
 from background_task.models import Task, CompletedTask
 from common.logger import log
 from common.errors import NoMediaException, DownloadFailedException
-from common.utils import json_serial
 from .models import Source, Media, MediaServer
 from .utils import (get_remote_image, resize_image_to_height, delete_file,
                     write_text_file)
@@ -52,7 +51,7 @@ def map_task_to_instance(task):
         Source: 'sync:source',
         Media: 'sync:media-item',
     }
-    # Unpack
+    # Unpack 
     task_func, task_args_str = task.task_name, task.task_params
     model = TASK_MAP.get(task_func, None)
     if not model:
@@ -132,23 +131,16 @@ def cleanup_completed_tasks():
 
 
 def cleanup_old_media():
-    for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0):
+    for media in Media.objects.filter(download_date__isnull=False):
-        delta = timezone.now() - timedelta(days=source.days_to_keep)
+        if media.source.delete_old_media and media.source.days_to_keep > 0:
-        for media in source.media_source.filter(downloaded=True, download_date__lt=delta):
+            delta = timezone.now() - timedelta(days=media.source.days_to_keep)
-            log.info(f'Deleting expired media: {source} / {media} '
+            if media.downloaded and media.download_date < delta:
-                     f'(now older than {source.days_to_keep} days / '
+                # Media was downloaded after the cutoff date, delete it
-                     f'download_date before {delta})')
+                log.info(f'Deleting expired media: {media.source} / {media} '
-            # .delete() also triggers a pre_delete signal that removes the files
+                         f'(now older than {media.source.days_to_keep} days / '
-            media.delete()
+                         f'download_date before {delta})')
-
+                # .delete() also triggers a pre_delete signal that removes the files
-
+                media.delete()
-def cleanup_removed_media(source, videos):
-    media_objects = Media.objects.filter(source=source, downloaded=True)
-    for item in media_objects:
-        matching_source_item = [video['id'] for video in videos if video['id'] == item.key]
-        if not matching_source_item:
-            log.info(f'{item.title} is no longer in source, removing')
-            item.delete()
 
 
 @background(schedule=0)
@@ -160,6 +152,7 @@ def index_source_task(source_id):
         source = Source.objects.get(pk=source_id)
     except Source.DoesNotExist:
         # Task triggered but the Source has been deleted, delete the task
+        delete_index_source_task(source_id)
         return
     # Reset any errors
     source.has_failed = False
@@ -182,7 +175,7 @@ def index_source_task(source_id):
             # Video has no unique key (ID), it can't be indexed
             continue
         try:
-            media = Media.objects.get(key=key, source=source)
+            media = Media.objects.get(key=key)
         except Media.DoesNotExist:
             media = Media(key=key)
         media.source = source
@@ -195,9 +188,6 @@ def index_source_task(source_id):
     cleanup_completed_tasks()
     # Tack on a cleanup of old media
     cleanup_old_media()
-    if source.delete_removed_media:
-        log.info(f'Cleaning up media no longer in source {source}')
-        cleanup_removed_media(source, videos)
 
 
 @background(schedule=0)
@@ -211,6 +201,7 @@ def check_source_directory_exists(source_id):
         source = Source.objects.get(pk=source_id)
     except Source.DoesNotExist:
         # Task triggered but the Source has been deleted, delete the task
+        delete_index_source_task(source_id)
         return
     # Check the source output directory exists
     if not source.directory_exists():
@@ -231,12 +222,9 @@ def download_media_metadata(media_id):
         log.error(f'Task download_media_metadata(pk={media_id}) called but no '
                   f'media exists with ID: {media_id}')
         return
-    if media.manual_skip:
-        log.info(f'Task for ID: {media_id} skipped, due to task being manually skipped.')
-        return
     source = media.source
     metadata = media.index_metadata()
-    media.metadata = json.dumps(metadata, default=json_serial)
+    media.metadata = json.dumps(metadata)
     upload_date = media.upload_date
     # Media must have a valid upload date
     if upload_date:
@@ -246,30 +234,20 @@ def download_media_metadata(media_id):
         media.skip = True
     # If the source has a download cap date check the upload date is allowed
     max_cap_age = source.download_cap_date
-    if media.published and max_cap_age:
+    if max_cap_age:
         if media.published < max_cap_age:
             # Media was published after the cap date, skip it
             log.warn(f'Media: {source} / {media} is older than cap age '
                      f'{max_cap_age}, skipping')
             media.skip = True
-    # If the source has a search filter, check the video title matches the filter
-    if source.filter_text and not source.is_regex_match(media.title):
-        # Filter text not found in the media title. Accepts regex string, blank search filter results in this returning false
-        log.warn(f'Media: {source} / {media} does not match {source.filter_text}, skipping')
-        media.skip = True
     # If the source has a cut-off check the upload date is within the allowed delta
     if source.delete_old_media and source.days_to_keep > 0:
-        if not isinstance(media.published, datetime):
+        delta = timezone.now() - timedelta(days=source.days_to_keep)
-            # Media has no known published date or incomplete metadata
+        if media.published < delta:
-            log.warn(f'Media: {source} / {media} has no published date, skipping')
+            # Media was published after the cutoff date, skip it
+            log.warn(f'Media: {source} / {media} is older than '
+                     f'{source.days_to_keep} days, skipping')
             media.skip = True
-        else:
-            delta = timezone.now() - timedelta(days=source.days_to_keep)
-            if media.published < delta:
-                # Media was published after the cutoff date, skip it
-                log.warn(f'Media: {source} / {media} is older than '
-                         f'{source.days_to_keep} days, skipping')
-                media.skip = True
     # Check we can download the media item
     if not media.skip:
         if media.get_format_str():
@@ -327,28 +305,20 @@ def download_media(media_id):
         return
     if media.skip:
         # Media was toggled to be skipped after the task was scheduled
-        log.warn(f'Download task triggered for  media: {media} (UUID: {media.pk}) but '
+        log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but it '
-                 f'it is now marked to be skipped, not downloading')
+                 f'is now marked to be skipped, not downloading')
         return
     if media.downloaded and media.media_file:
         # Media has been marked as downloaded before the download_media task was fired,
         # skip it
-        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
+        log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but it '
-                 f'it has already been marked as downloaded, not downloading again')
+                 f'has already been marked as downloaded, not downloading again')
         return
     if not media.source.download_media:
-        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
+        log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but the '
-                 f'the source {media.source} has since been marked to not download, '
+                 f'source {media.source} has since been marked to not download media, '
                  f'not downloading')
         return
-    max_cap_age = media.source.download_cap_date
-    published = media.published
-    if max_cap_age and published:
-        if published <= max_cap_age:
-            log.warn(f'Download task triggered media: {media} (UUID: {media.pk}) but '
-                     f'the source has a download cap and the media is now too old, '
-                     f'not downloading')
-            return
     filepath = media.filepath
     log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"')
     format_str, container = media.download_media()
@@ -357,7 +327,7 @@ def download_media(media_id):
         log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: '
                  f'"{filepath}"')
         # Link the media file to the object and update info about the download
-        media.media_file.name = str(media.source.type_directory_path / media.filename)
+        media.media_file.name = str(filepath)
         media.downloaded = True
         media.download_date = timezone.now()
         media.downloaded_filesize = os.path.getsize(filepath)

tubesync/sync/templates/sync/_mediaformatvars.html

@@ -25,12 +25,12 @@
     </tr>
     <tr>
       <td>{mm}</td>
-      <td>Media publish month in MM</td>
+      <td>Media publish year in MM</td>
       <td>01</td>
     </tr>
     <tr>
       <td>{dd}</td>
-      <td>Media publish day in DD</td>
+      <td>Media publish year in DD</td>
       <td>31</td>
     </tr>
     <tr>
@@ -43,11 +43,6 @@
       <td>Full source name</td>
       <td>My Source</td>
     </tr>
-    <tr>
-      <td>{uploader}</td>
-      <td>Uploader name</td>
-      <td>Some Channel Name</td>
-    </tr>
     <tr>
       <td>{title}</td>
       <td>Lower case media title, max 80 chars</td>
@@ -68,6 +63,11 @@
       <td>Media format string</td>
       <td>720p-avc1-mp4a</td>
     </tr>
+    <tr>
+      <td>{playlist_index}</td>
+      <td>Playlist index of media, if it's in a playlist</td>
+      <td>12</td>
+    </tr>
     <tr>
       <td>{playlist_title}</td>
       <td>Playlist title of media, if it's in a playlist</td>

tubesync/sync/templates/sync/dashboard.html

@@ -101,7 +101,7 @@
 </div>
 <div class="row">
   <div class="col s12">
-    <h2 class="truncate">Runtime information</h2>
+    <h2 class="truncate">Runtime infomation</h2>
   </div>
 </div>
 <div class="row">
@@ -123,10 +123,6 @@
         <td class="hide-on-small-only">Downloads directory</td>
         <td><span class="hide-on-med-and-up">Downloads directory<br></span><strong>{{ downloads_dir }}</strong></td>
       </tr>
-      <tr title="Database connection used by TubeSync">
-        <td class="hide-on-small-only">Database</td>
-        <td><span class="hide-on-med-and-up">Database<br></span><strong>{{ database_connection }}</strong></td>
-      </tr>
     </table>
   </div>
 </div>

tubesync/sync/templates/sync/media-item.html

@@ -9,24 +9,10 @@
     {% if media.title %}<h2 class="truncate"><strong>{{ media.title }}</strong></h2>{% endif %}
     <p class="truncate"><strong><a href="{{ media.url }}" target="_blank"><i class="fas fa-link"></i> {{ media.url }}</a></strong></p>
     <p class="truncate">Downloading to: <strong>{{ media.source.directory_path }}</strong></p>
-    {% if download_state == 'downloaded' %}
-    {% if media.source.is_audio %}
-    <audio controls src="{% url 'sync:media-content' pk=media.pk %}"></audio>
-    {% else %}
-    <video controls style="width: 100%">
-      <source src="{% url 'sync:media-content' pk=media.pk %}">
-    </video>
-    {% endif %}
-
-    <p class="truncate"><a href="{% url 'sync:media-content' pk=media.pk %}" download="{{ media.filename }}"><strong><i class="fas fa-download"></i> Download</strong></a></p>
-    {% endif %}
   </div>
 </div>
-{% if media.manual_skip %}{% include 'errorbox.html' with message='Media is marked to be skipped and will not be downloaded.' %}
+{% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
-{% else %}
+{% if media.skip %}{% include 'errorbox.html' with message='Media is marked to be skipped and will not be downloaded.' %}{% endif %}
-  {% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
-  {% if media.skip %}{% include 'errorbox.html' with message='This media may be skipped due to error(s).' %}{% endif %}
-{% endif %}
 {% include 'infobox.html' with message=message %}
 <div class="row">
   <div class="col s12 m7">
@@ -170,10 +156,10 @@
 {% else %}
 <div class="row">
   <div class="col s12">
-    {% if media.manual_skip %}
+    {% if media.skip %}
-      <a href="{% url 'sync:enable-media' pk=media.pk %}" class="btn">Unskip media (manually) <i class="fas fa-cloud-download-alt"></i></a>
+      <a href="{% url 'sync:enable-media' pk=media.pk %}" class="btn">Enable (unskip) media <i class="fas fa-cloud-download-alt"></i></a>
     {% else %}
-      <a href="{% url 'sync:skip-media' pk=media.pk %}" class="btn delete-button">Manually mark media to be skipped <i class="fas fa-times-circle"></i></a>
+      <a href="{% url 'sync:skip-media' pk=media.pk %}" class="btn delete-button">Skip media <i class="fas fa-times-circle"></i></a>
     {% endif %}
   </div>
 </div>

tubesync/sync/templates/sync/media.html

@@ -4,23 +4,9 @@
 
 {% block content %}
 <div class="row">
-  <div class="col s12 m6">
+  <div class="col s12">
     <h1 class="truncate">Media</h1>
   </div>
-  <div class="col s12 m3">
-    {% if show_skipped %}
-    <a href="{% url 'sync:media' %}{% if source %}?filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye-slash"></i> Hide skipped media</a>
-    {% else %}
-    <a href="{% url 'sync:media' %}?show_skipped=yes{% if source %}&filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye"></i> Show skipped media</a>
-    {% endif %}
-  </div>
-  <div class="col s12 m3">
-    {% if only_skipped %}
-    <a href="{% url 'sync:media' %}{% if source %}?filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye-slash"></i> Only skipped media</a>
-    {% else %}
-    <a href="{% url 'sync:media' %}?only_skipped=yes{% if source %}&filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye"></i> Only skipped media</a>
-    {% endif %}
-  </div>
 </div>
 {% include 'infobox.html' with message=message %}
 <div class="row no-margin-bottom">
@@ -36,10 +22,8 @@
             {% if m.downloaded %}
               <i class="fas fa-check-circle" title="Downloaded"></i> {{ m.download_date|date:'Y-m-d' }}
             {% else %}
-              {% if m.manual_skip %}
+              {% if m.skip %}
-              <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Manually skipped</span>
+              <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped</span>
-              {% elif m.skip %}
-              <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped by system</span>
               {% elif not m.source.download_media %}
               <span class="error-text"><i class="fas fa-times" title="Not downloading media for this source"></i> Disabled at source</span>
               {% elif not m.has_metadata %}
@@ -64,5 +48,5 @@
   </div>
   {% endfor %}
 </div>
-{% include 'pagination.html' with pagination=sources.paginator filter=source.pk show_skipped=show_skipped only_skipped=only_skipped%}
+{% include 'pagination.html' with pagination=sources.paginator filter=source.pk %}
 {% endblock %}

tubesync/sync/templates/sync/source-delete.html

@@ -9,8 +9,8 @@
     <p>
       Are you sure you want to delete this source? Deleting a source is permanent.
       By default, deleting a source does not delete any saved media files. You can
-      <strong>tick the &quot;also delete downloaded media&quot; checkbox to also remove directory {{ source.directory_path }}
+      tick the &quot;also delete downloaded media&quot; checkbox to also remove save
-	  </strong>when you delete the source. Deleting a source cannot be undone.
+      media when you delete the source. Deleting a source cannot be undone.
     </p>
   </div>
 </div>

tubesync/sync/templates/sync/source.html

@@ -43,10 +43,6 @@
         <td class="hide-on-small-only">Directory</td>
         <td><span class="hide-on-med-and-up">Directory<br></span><strong>{{ source.directory }}</strong></td>
       </tr>
-      <tr title="Filter text">
-        <td class="hide-on-small-only">Filter text</td>
-        <td><span class="hide-on-med-and-up">Filter text<br></span><strong>{{ source.filter_text }}</strong></td>
-      </tr>
       <tr title="Media file name format to use for saving files">
         <td class="hide-on-small-only">Media format</td>
         <td><span class="hide-on-med-and-up">Media format<br></span><strong>{{ source.media_format }}</strong></td>
@@ -115,18 +111,6 @@
         <td class="hide-on-small-only">Write NFO?</td>
         <td><span class="hide-on-med-and-up">Write NFO?<br></span><strong>{% if source.write_nfo %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
-      <tr title="Should a JSON file be written with the media?">
-        <td class="hide-on-small-only">Write JSON?</td>
-        <td><span class="hide-on-med-and-up">Write JSON?<br></span><strong>{% if source.write_json %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
-      </tr>
-      <tr title="Delete media that is no longer on this playlist?">
-        <td class="hide-on-small-only">Delete removed media</td>
-        <td><span class="hide-on-med-and-up">Delete removed media<br></span><strong>{% if source.delete_removed_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
-      </tr>
-        <tr title="Delete files on disk when they are removed from TubeSync?">
-        <td class="hide-on-small-only">Delete files on disk</td>
-        <td><span class="hide-on-med-and-up">Delete files on disk<br></span><strong>{% if source.delete_files_on_disk %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
-      </tr>
       {% if source.delete_old_media and source.days_to_keep > 0 %}
       <tr title="Days after which your media from this source will be locally deleted">
         <td class="hide-on-small-only">Delete old media</td>
@@ -142,55 +126,6 @@
         <td class="hide-on-small-only">UUID</td>
         <td><span class="hide-on-med-and-up">UUID<br></span><strong>{{ source.uuid }}</strong></td>
       </tr>
-      <tr title="{{ _('Embedding thumbnail?') }}">
-        <td class="hide-on-small-only">{{ _("Embed thumbnail?") }}:</td>
-        <td><span class="hide-on-med-and-up">{{ _("Embed thumbnail?") }}<br></span><strong><i class="fas {% if source.embed_thumbnail %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
-      </tr>
-      <tr title="{{ _('Embedding metadata?') }}">
-        <td class="hide-on-small-only">{{ _("Embed metadata?") }}:</td>
-        <td><span class="hide-on-med-and-up">{{ _("Embed metadata?") }}<br></span><strong><i class="fas {% if source.embed_metadata %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
-      </tr>
-
-      <tr title="{{ _('Is sponsorblock enabled?') }}">
-        <td class="hide-on-small-only">{{ _("SponsorBlock?") }}:</td>
-        <td><span class="hide-on-med-and-up">{{ _("Sponsorblock enabled?") }}<br></span><strong><i class="fas {% if source.enable_sponsorblock %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
-      </tr>
-
-      {% if source.enable_sponsorblock %}
-      <tr title="{{ _('SponsorBlock: What to block?') }}">
-        <td class="hide-on-small-only">{{ _("What blocked?") }}:</td>
-        <td><span class="hide-on-med-and-up">{{ _("What blocked?") }}<br></span><strong>
-          {% if source.sponsorblock_categories.all_choice in source.sponsorblock_categories.selected_choices %}
-            {% for k,v in source.sponsorblock_categories.possible_choices %}
-              {{ v }}: <i class="fas fa-check"></i><BR>
-            {% endfor %}
-          {% else %}
-            {% for c in source.sponsorblock_categories.selected_choices %} 
-              {% for k,v in source.sponsorblock_categories.possible_choices %}
-                {% if k == c %} {{ v }}: <i class="fas fa-check"></i><BR>{% endif %}
-              {% endfor %}
-            {% endfor %}
-          {% endif %}
-        </strong></td>
-      </tr>
-      {% endif %}
-
-      <tr title="{{ _('Are Subtitles downloaded?') }}">
-        <td class="hide-on-small-only">{{ _("Download subtitles?") }}:</td>
-        <td><span class="hide-on-med-and-up">{{ _("Download subtitles?") }}:</span><strong><i class="fas {% if source.write_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
-      </tr>
-
-      {% if source.write_subtitles %}
-      <tr title="{{ _('Are auto subs accepted?') }}">
-        <td class="hide-on-small-only">{{ _("Auto-generated subtitles?") }}:</td>
-        <td><span class="hide-on-med-and-up">{{ _("Auto-generated subtitles?") }}:</span><strong><i class="fas {% if source.auto_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
-      </tr>
-      <tr title="{{ _('Subs langs?') }}">
-        <td class="hide-on-small-only">{{ _("Subs langs?") }}:</td>
-        <td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
-      </tr>
-      {% endif %}
-      
     </table>
   </div>
 </div>

tubesync/sync/templates/sync/sources.html

@@ -24,18 +24,15 @@
   <div class="col s12">
     <div class="collection">
     {% for source in sources %}
-      <span class="collection-item flex-collection-container">
+      <a href="{% url 'sync:source' pk=source.pk %}" class="collection-item">
-        <a href="{% url 'sync:source' pk=source.pk %}" class="flex-grow">
+        {{ source.icon|safe }} <strong>{{ source.name }}</strong> ({{ source.get_source_type_display }} &quot;{{ source.key }}&quot;)<br>
-          {{ source.icon|safe }} <strong>{{ source.name }}</strong> ({{ source.get_source_type_display }} &quot;{{ source.key }}&quot;)<br>
+        {{ source.format_summary }}<br>
-          {{ source.format_summary }}<br>
+        {% if source.has_failed %}
-          {% if source.has_failed %}
+        <span class="error-text"><i class="fas fa-exclamation-triangle"></i> <strong>Source has permanent failures</strong></span>
-          <span class="error-text"><i class="fas fa-exclamation-triangle"></i> <strong>Source has permanent failures</strong></span>
+        {% else %}
-          {% else %}
+        <strong>{{ source.media_count }}</strong> media items{% if source.delete_old_media and source.days_to_keep > 0 %}, keep {{ source.days_to_keep }} days of media{% endif %}
-          <strong>{{ source.media_count }}</strong> media items, <strong>{{ source.downloaded_count }}</strong> downloaded{% if source.delete_old_media and source.days_to_keep > 0 %}, keeping {{ source.days_to_keep }} days of media{% endif %}
+        {% endif %}
-          {% endif %}
+      </a>
-        </a>
-        <a href="{% url 'sync:source-sync-now' pk=source.pk %}" class="collection-item"><i class="fas fa-arrow-rotate-right"></i></a>
-      </span>
     {% empty %}
       <span class="collection-item no-items"><i class="fas fa-info-circle"></i> You haven't added any sources.</span>
     {% endfor %}

tubesync/sync/templates/sync/tasks.html

@@ -66,7 +66,7 @@
       {% for task in scheduled %}
         <a href="{% url task.url pk=task.instance.pk %}" class="collection-item">
           <i class="fas fa-stopwatch"></i> <strong>{{ task }}</strong><br>
-          {% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
+          {% if task.instance.index_schedule %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
           <i class="fas fa-redo"></i> Task will run {% if task.run_now %}<strong>immediately</strong>{% else %}at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>{% endif %}
         </a>
       {% empty %}

tubesync/sync/templates/widgets/checkbox_option.html

@@ -1,7 +0,0 @@
-<!--<input type="{{ option.type }}" name="{{ option.name }}" value="{{ option.value }}" id="{{ option.value }}"><BR>
-    <label for="{{ option.value }}">{{option.label}}</label>-->
-
-<label>
-    <input type="{{ option.type }}" name="{{ option.name }}" value="{{ option.value }}" id="{{ option.value }}" {% if option.checked %}checked{% endif %}>
-    <span>{{option.label}}</span>
-</label>

tubesync/sync/templates/widgets/checkbox_select.html

@@ -1,5 +0,0 @@
-</label>
-{% for option in widget.multipleChoiceProperties %}
-    {% include option.template_name with option=option %}
-{% endfor %}
-<label>

tubesync/sync/testdata/metadata.json

@@ -9,6 +9,7 @@
     "average_rating": 1.2345,
     "dislike_count": 123,
     "like_count": 456,
+    "playlist_index": 789,
     "playlist_title": "test playlist",
     "uploader": "test uploader",
     "categories":[

tubesync/sync/tests.py

@@ -6,7 +6,7 @@
 
 
 import logging
-from datetime import datetime, timedelta
+from datetime import datetime
 from urllib.parse import urlsplit
 from xml.etree import ElementTree
 from django.conf import settings
@@ -14,7 +14,6 @@ from django.test import TestCase, Client
 from django.utils import timezone
 from background_task.models import Task
 from .models import Source, Media
-from .tasks import cleanup_old_media
 
 
 class FrontEndTestCase(TestCase):
@@ -22,7 +21,7 @@ class FrontEndTestCase(TestCase):
     def setUp(self):
         # Disable general logging for test case
         logging.disable(logging.CRITICAL)
-
+    
     def test_dashboard(self):
         c = Client()
         response = c.get('/')
@@ -37,9 +36,6 @@ class FrontEndTestCase(TestCase):
         test_sources = {
             'youtube-channel': {
                 'valid': (
-                    'https://m.youtube.com/testchannel',
-                    'https://m.youtube.com/c/testchannel',
-                    'https://m.youtube.com/c/testchannel/videos',
                     'https://www.youtube.com/testchannel',
                     'https://www.youtube.com/c/testchannel',
                     'https://www.youtube.com/c/testchannel/videos',
@@ -51,7 +47,6 @@ class FrontEndTestCase(TestCase):
                 'invalid_domain': (
                     'https://www.test.com/c/testchannel',
                     'https://www.example.com/c/testchannel',
-                    'https://n.youtube.com/c/testchannel',
                 ),
                 'invalid_path': (
                     'https://www.youtube.com/test/invalid',
@@ -67,8 +62,6 @@ class FrontEndTestCase(TestCase):
             },
             'youtube-channel-id': {
                 'valid': (
-                    'https://m.youtube.com/channel/channelid',
-                    'https://m.youtube.com/channel/channelid/videos',
                     'https://www.youtube.com/channel/channelid',
                     'https://www.youtube.com/channel/channelid/videos',
                 ),
@@ -79,7 +72,6 @@ class FrontEndTestCase(TestCase):
                 'invalid_domain': (
                     'https://www.test.com/channel/channelid',
                     'https://www.example.com/channel/channelid',
-                    'https://n.youtube.com/channel/channelid',
                 ),
                 'invalid_path': (
                     'https://www.youtube.com/test/invalid',
@@ -91,8 +83,6 @@ class FrontEndTestCase(TestCase):
             },
             'youtube-playlist': {
                 'valid': (
-                    'https://m.youtube.com/playlist?list=testplaylist',
-                    'https://m.youtube.com/watch?v=testvideo&list=testplaylist',
                     'https://www.youtube.com/playlist?list=testplaylist',
                     'https://www.youtube.com/watch?v=testvideo&list=testplaylist',
                 ),
@@ -103,7 +93,6 @@ class FrontEndTestCase(TestCase):
                 'invalid_domain': (
                     'https://www.test.com/playlist?list=testplaylist',
                     'https://www.example.com/playlist?list=testplaylist',
-                    'https://n.youtube.com/playlist?list=testplaylist',
                 ),
                 'invalid_path': (
                     'https://www.youtube.com/notplaylist?list=testplaylist',
@@ -175,7 +164,6 @@ class FrontEndTestCase(TestCase):
             'directory': 'testdirectory',
             'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
             'download_cap': 0,
-            'filter_text':'.*',
             'index_schedule': 3600,
             'delete_old_media': False,
             'days_to_keep': 14,
@@ -184,8 +172,7 @@ class FrontEndTestCase(TestCase):
             'source_acodec': 'OPUS',
             'prefer_60fps': False,
             'prefer_hdr': False,
-            'fallback': 'f',
+            'fallback': 'f'
-            'sub_langs': 'en',
         }
         response = c.post('/source-add', data)
         self.assertEqual(response.status_code, 302)
@@ -218,7 +205,6 @@ class FrontEndTestCase(TestCase):
             'directory': 'testdirectory',
             'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
             'download_cap': 0,
-            'filter_text':'.*',
             'index_schedule': Source.IndexSchedule.EVERY_HOUR,
             'delete_old_media': False,
             'days_to_keep': 14,
@@ -227,8 +213,7 @@ class FrontEndTestCase(TestCase):
             'source_acodec': Source.SOURCE_ACODEC_OPUS,
             'prefer_60fps': False,
             'prefer_hdr': False,
-            'fallback': Source.FALLBACK_FAIL,
+            'fallback': Source.FALLBACK_FAIL
-            'sub_langs': 'en',
         }
         response = c.post(f'/source-update/{source_uuid}', data)
         self.assertEqual(response.status_code, 302)
@@ -249,7 +234,6 @@ class FrontEndTestCase(TestCase):
             'directory': 'testdirectory',
             'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
             'download_cap': 0,
-            'filter_text':'.*',
             'index_schedule': Source.IndexSchedule.EVERY_2_HOURS,  # changed
             'delete_old_media': False,
             'days_to_keep': 14,
@@ -258,8 +242,7 @@ class FrontEndTestCase(TestCase):
             'source_acodec': Source.SOURCE_ACODEC_OPUS,
             'prefer_60fps': False,
             'prefer_hdr': False,
-            'fallback': Source.FALLBACK_FAIL,
+            'fallback': Source.FALLBACK_FAIL
-            'sub_langs': 'en',
         }
         response = c.post(f'/source-update/{source_uuid}', data)
         self.assertEqual(response.status_code, 302)
@@ -361,25 +344,21 @@ class FrontEndTestCase(TestCase):
                 }]
             } 
         '''
-        past_date = timezone.make_aware(datetime(year=2000, month=1, day=1))
         test_media1 = Media.objects.create(
             key='mediakey1',
             source=test_source,
-            published=past_date,
             metadata=test_minimal_metadata
         )
         test_media1_pk = str(test_media1.pk)
         test_media2 = Media.objects.create(
             key='mediakey2',
             source=test_source,
-            published=past_date,
             metadata=test_minimal_metadata
         )
         test_media2_pk = str(test_media2.pk)
         test_media3 = Media.objects.create(
             key='mediakey3',
             source=test_source,
-            published=past_date,
             metadata=test_minimal_metadata
         )
         test_media3_pk = str(test_media3.pk)
@@ -471,14 +450,11 @@ metadata_60fps_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_60
 metadata_60fps = open(metadata_60fps_filepath, 'rt').read()
 metadata_60fps_hdr_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_60fps_hdr.json'
 metadata_60fps_hdr = open(metadata_60fps_hdr_filepath, 'rt').read()
-metadata_20230629_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_2023-06-29.json'
-metadata_20230629 = open(metadata_20230629_filepath, 'rt').read()
 all_test_metadata = {
     'boring': metadata,
     'hdr': metadata_hdr,
     '60fps': metadata_60fps,
     '60fps+hdr': metadata_60fps_hdr,
-    '20230629': metadata_20230629,
 }
 
 
@@ -511,7 +487,7 @@ class FilepathTestCase(TestCase):
             metadata=metadata,
         )
 
-    def test_source_media_format(self):
+    def test_source_dirname(self):
         # Check media format validation is working
         # Empty
         self.source.media_format = ''
@@ -559,6 +535,9 @@ class FilepathTestCase(TestCase):
         self.source.media_format = 'test-{format}'
         self.assertEqual(self.source.get_example_media_format(),
                          'test-1080p-vp9-opus')
+        self.source.media_format = 'test-{playlist_index}'
+        self.assertEqual(self.source.get_example_media_format(),
+                         'test-1')
         self.source.media_format = 'test-{playlist_title}'
         self.assertEqual(self.source.get_example_media_format(),
                          'test-Some Playlist Title')
@@ -661,8 +640,6 @@ class MediaTestCase(TestCase):
             '<episodedetails>',
             '  <title>no fancy stuff title</title>',
             '  <showtitle>testname</showtitle>',
-            '  <season>2017</season>',
-            '  <episode></episode>',
             '  <ratings>',
             '    <rating default="True" max="5" name="youtube">',
             '      <value>1.2345</value>',
@@ -1184,14 +1161,14 @@ class FormatMatchingTestCase(TestCase):
             ('2160p', 'VP9', False, True): (True, '337'),              # Exact match, hdr
             ('2160p', 'VP9', True, False): (True, '315'),              # Exact match, 60fps
             ('2160p', 'VP9', True, True): (True, '337'),               # Exact match, 60fps+hdr
-            ('4320p', 'AVC1', False, False): (False, False),
+            ('4320P', 'AVC1', False, False): (False, False),
-            ('4320p', 'AVC1', False, True): (False, False),
+            ('4320P', 'AVC1', False, True): (False, False),
-            ('4320p', 'AVC1', True, False): (False, False),
+            ('4320P', 'AVC1', True, False): (False, False),
-            ('4320p', 'AVC1', True, True): (False, False),
+            ('4320P', 'AVC1', True, True): (False, False),
-            ('4320p', 'VP9', False, False): (False, False),
+            ('4320P', 'VP9', False, False): (False, False),
-            ('4320p', 'VP9', False, True): (False, False),
+            ('4320P', 'VP9', False, True): (False, False),
-            ('4320p', 'VP9', True, False): (True, '272'),              # Exact match, 60fps
+            ('4320P', 'VP9', True, False): (True, '272'),              # Exact match, 60fps
-            ('4320p', 'VP9', True, True): (False, False),
+            ('4320P', 'VP9', True, True): (False, False),
         }
         for params, expected in expected_matches.items():
             resolution, vcodec, prefer_60fps, prefer_hdr = params
@@ -1390,14 +1367,14 @@ class FormatMatchingTestCase(TestCase):
             ('2160p', 'VP9', False, True): (True, '337'),              # Exact match, hdr
             ('2160p', 'VP9', True, False): (True, '315'),              # Exact match, 60fps
             ('2160p', 'VP9', True, True): (True, '337'),               # Exact match, 60fps+hdr
-            ('4320p', 'AVC1', False, False): (False, '272'),           # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320P', 'AVC1', False, False): (False, '272'),           # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
-            ('4320p', 'AVC1', False, True): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320P', 'AVC1', False, True): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
-            ('4320p', 'AVC1', True, False): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320P', 'AVC1', True, False): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
-            ('4320p', 'AVC1', True, True): (False, '272'),             # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
+            ('4320P', 'AVC1', True, True): (False, '272'),             # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
-            ('4320p', 'VP9', False, False): (False, '272'),            # Fallback, 60fps (no other 8k streams)
+            ('4320P', 'VP9', False, False): (False, '272'),            # Fallback, 60fps (no other 8k streams)
-            ('4320p', 'VP9', False, True): (False, '272'),             # Fallback, 60fps (no other 8k streams)
+            ('4320P', 'VP9', False, True): (False, '272'),             # Fallback, 60fps (no other 8k streams)
-            ('4320p', 'VP9', True, False): (True, '272'),              # Exact match, 60fps
+            ('4320P', 'VP9', True, False): (True, '272'),              # Exact match, 60fps
-            ('4320p', 'VP9', True, True): (False, '272'),              # Fallback, 60fps (no other 8k streams)
+            ('4320P', 'VP9', True, True): (False, '272'),              # Fallback, 60fps (no other 8k streams)
         }
         for params, expected in expected_matches.items():
             resolution, vcodec, prefer_60fps, prefer_hdr = params
@@ -1409,118 +1386,3 @@ class FormatMatchingTestCase(TestCase):
             match_type, format_code = self.media.get_best_video_format()
             self.assertEqual(format_code, expected_format_code)
             self.assertEqual(match_type, expeceted_match_type)
-
-    def test_metadata_20230629(self):
-        self.media.metadata = all_test_metadata['20230629']
-        expected_matches = {
-            # (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
-            ('360p', 'AVC1', False, True): (False, '134'),             # Fallback match, no hdr
-            ('360p', 'AVC1', True, False): (False, '134'),             # Fallback match, no 60fps
-            ('360p', 'AVC1', True, True): (False, '332'),              # Fallback match, 60fps+hdr, switched to VP9
-            ('360p', 'VP9', False, False): (True, '243'),              # Exact match
-            ('360p', 'VP9', False, True): (True, '332'),               # Exact match, hdr
-            ('360p', 'VP9', True, False): (False, '332'),              # Fallback match, 60fps, extra hdr
-            ('360p', 'VP9', True, True): (True, '332'),                # Exact match, 60fps+hdr
-            ('480p', 'AVC1', False, False): (True, '135'),             # Exact match
-            ('480p', 'AVC1', False, True): (False, '135'),             # Fallback match, no hdr
-            ('480p', 'AVC1', True, False): (False, '135'),             # Fallback match, no 60fps
-            ('480p', 'AVC1', True, True): (False, '333'),              # Fallback match, 60fps+hdr, switched to VP9
-            ('480p', 'VP9', False, False): (True, '244'),              # Exact match
-            ('480p', 'VP9', False, True): (True, '333'),               # Exact match, hdr
-            ('480p', 'VP9', True, False): (False, '333'),              # Fallback match, 60fps, extra hdr
-            ('480p', 'VP9', True, True): (True, '333'),                # Exact match, 60fps+hdr
-            ('720p', 'AVC1', False, False): (True, '136'),             # Exact match
-            ('720p', 'AVC1', False, True): (False, '136'),             # Fallback match, no hdr
-            ('720p', 'AVC1', True, False): (True, '298'),              # Exact match, 60fps
-            ('720p', 'AVC1', True, True): (False, '334'),              # Fallback match, 60fps+hdr, switched to VP9
-            ('720p', 'VP9', False, False): (True, '247'),              # Exact match
-            ('720p', 'VP9', False, True): (True, '334'),               # Exact match, hdr
-            ('720p', 'VP9', True, False): (True, '302'),               # Exact match, 60fps
-            ('720p', 'VP9', True, True): (True, '334'),                # Exact match, 60fps+hdr
-            ('1440p', 'AVC1', False, False): (False, '308'),           # Fallback match, 60fps, switched to VP9 (no 1440p AVC1)
-            ('1440p', 'AVC1', False, True): (False, '336'),            # Fallback match, 60fps+hdr, switched to VP9 (no 1440p AVC1)
-            ('1440p', 'AVC1', True, False): (False, '308'),            # Fallback match, 60fps, switched to VP9 (no 1440p AVC1)
-            ('1440p', 'AVC1', True, True): (False, '336'),             # Fallback match, 60fps+hdr, switched to VP9 (no 1440p AVC1)
-            ('1440p', 'VP9', False, False): (False, '308'),            # Fallback, 60fps
-            ('1440p', 'VP9', False, True): (True, '336'),              # Exact match, hdr
-            ('1440p', 'VP9', True, False): (True, '308'),              # Exact match, 60fps
-            ('1440p', 'VP9', True, True): (True, '336'),               # Exact match, 60fps+hdr
-            ('2160p', 'AVC1', False, False): (False, '315'),           # Fallback, 60fps, switched to VP9 (no 2160p AVC1)
-            ('2160p', 'AVC1', False, True): (False, '337'),            # Fallback match, 60fps+hdr, switched to VP9 (no 2160p AVC1)
-            ('2160p', 'AVC1', True, False): (False, '315'),            # Fallback, switched to VP9 (no 2160p AVC1)
-            ('2160p', 'AVC1', True, True): (False, '337'),             # Fallback match, 60fps+hdr, switched to VP9 (no 2160p AVC1)
-            ('2160p', 'VP9', False, False): (False, '315'),            # Fallback, 60fps
-            ('2160p', 'VP9', False, True): (True, '337'),              # Exact match, hdr
-            ('2160p', 'VP9', True, False): (True, '315'),              # Exact match, 60fps
-            ('2160p', 'VP9', True, True): (True, '337'),               # Exact match, 60fps+hdr
-            ('4320p', 'AVC1', False, False): (False, '272'),           # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
-            ('4320p', 'AVC1', False, True): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
-            ('4320p', 'AVC1', True, False): (False, '272'),            # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
-            ('4320p', 'AVC1', True, True): (False, '272'),             # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
-            ('4320p', 'VP9', False, False): (False, '272'),            # Fallback, 60fps (no other 8k streams)
-            ('4320p', 'VP9', False, True): (False, '272'),             # Fallback, 60fps (no other 8k streams)
-            ('4320p', 'VP9', True, False): (True, '272'),              # Exact match, 60fps
-            ('4320p', 'VP9', True, True): (False, '272'),              # Fallback, 60fps (no other 8k streams)
-        }
-        for params, expected in expected_matches.items():
-            resolution, vcodec, prefer_60fps, prefer_hdr = params
-            expeceted_match_type, expected_format_code = expected
-            self.source.source_resolution = resolution
-            self.source.source_vcodec = vcodec
-            self.source.prefer_60fps = prefer_60fps
-            self.source.prefer_hdr = prefer_hdr
-            # The aim here is to execute the matching code to find error paths, specific testing isn't required
-            self.media.get_best_video_format()
-            self.media.get_best_audio_format()
-
-    def test_is_regex_match(self):
-        
-        self.media.metadata = all_test_metadata['boring']
-        expected_matches = {
-            ('.*'): (True),
-            ('no fancy stuff'): (True),
-            ('No fancy stuff'): (False),
-            ('(?i)No fancy stuff'): (True), #set case insensitive flag
-            ('no'): (True),
-            ('Foo'): (False),
-            ('^(?!.*fancy).*$'): (False),
-            ('^(?!.*funny).*$'): (True),
-            ('(?=.*f.*)(?=.{0,2}|.{4,})'): (True),
-            ('f{4,}'): (False),
-            ('^[^A-Z]*$'): (True),
-            ('^[^a-z]*$'): (False),
-            ('^[^\\s]*$'): (False)
-        }
-
-        for params, expected in expected_matches.items():
-            self.source.filter_text = params
-            expected_match_result = expected
-            self.assertEqual(self.source.is_regex_match(self.media.title), expected_match_result)
-
-class TasksTestCase(TestCase):
-    def setUp(self):
-        # Disable general logging for test case
-        logging.disable(logging.CRITICAL)
-
-    def test_delete_old_media(self):
-        src1 = Source.objects.create(key='aaa', name='aaa', directory='/tmp/a', delete_old_media=False, days_to_keep=14)
-        src2 = Source.objects.create(key='bbb', name='bbb', directory='/tmp/b', delete_old_media=True, days_to_keep=14)
-
-        now = timezone.now()
-
-        m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5))
-        m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25))
-        m13 = Media.objects.create(source=src1, downloaded=False, key='a13')
-
-        m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5))
-        m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25))
-        m23 = Media.objects.create(source=src2, downloaded=False, key='a23')
-
-        self.assertEquals(src1.media_source.all().count(), 3)
-        self.assertEquals(src2.media_source.all().count(), 3)
-
-        cleanup_old_media()
-
-        self.assertEquals(src1.media_source.all().count(), 3)
-        self.assertEquals(src2.media_source.all().count(), 2)
-        self.assertEquals(Media.objects.filter(pk=m22.pk).exists(), False)

tubesync/sync/urls.py

@@ -2,7 +2,7 @@ from django.urls import path
 from .views import (DashboardView, SourcesView, ValidateSourceView, AddSourceView,
                     SourceView, UpdateSourceView, DeleteSourceView, MediaView,
                     MediaThumbView, MediaItemView, MediaRedownloadView, MediaSkipView,
-                    MediaEnableView, MediaContent, TasksView, CompletedTasksView, ResetTasks,
+                    MediaEnableView, TasksView, CompletedTasksView, ResetTasks,
                     MediaServersView, AddMediaServerView, MediaServerView,
                     DeleteMediaServerView, UpdateMediaServerView)
 
@@ -28,10 +28,6 @@ urlpatterns = [
          ValidateSourceView.as_view(),
          name='validate-source'),
 
-    path('source-sync-now/<uuid:pk>',
-         SourcesView.as_view(),
-         name='source-sync-now'),
-
     path('source-add',
          AddSourceView.as_view(),
          name='add-source'),
@@ -74,10 +70,6 @@ urlpatterns = [
          MediaEnableView.as_view(),
          name='enable-media'),
 
-    path('media-content/<uuid:pk>',
-         MediaContent.as_view(),
-         name='media-content'),
-
     # Task URLs
 
     path('tasks',

tubesync/sync/utils.py

@@ -14,9 +14,9 @@ def validate_url(url, validator):
         Validate a URL against a dict of validation requirements. Returns an extracted
         part of the URL if the URL is valid, if invalid raises a ValidationError.
     '''
-    valid_scheme, valid_netlocs, valid_path, invalid_paths, valid_query, \
+    valid_scheme, valid_netloc, valid_path, invalid_paths, valid_query, \
         extract_parts = (
-            validator['scheme'], validator['domains'], validator['path_regex'],
+            validator['scheme'], validator['domain'], validator['path_regex'],
             validator['path_must_not_match'], validator['qs_args'],
             validator['extract_key']
     )
@@ -25,8 +25,8 @@ def validate_url(url, validator):
     if url_scheme != valid_scheme:
         raise ValidationError(f'invalid scheme "{url_scheme}" must be "{valid_scheme}"')
     url_netloc = str(url_parts.netloc).strip().lower()
-    if url_netloc not in valid_netlocs:
+    if url_netloc != valid_netloc:
-        raise ValidationError(f'invalid domain "{url_netloc}" must be one of "{valid_netlocs}"')
+        raise ValidationError(f'invalid domain "{url_netloc}" must be "{valid_netloc}"')
     url_path = str(url_parts.path).strip()
     matches = re.findall(valid_path, url_path)
     if not matches:
@@ -78,7 +78,7 @@ def resize_image_to_height(image, width, height):
     if scaled_width < width:
         # Width too small, stretch it
         scaled_width = width
-    image = image.resize((scaled_width, height), Image.LANCZOS)
+    image = image.resize((scaled_width, height), Image.ANTIALIAS)
     if scaled_width > width:
         # Width too large, crop it
         delta = scaled_width - width

tubesync/sync/views.py

@@ -1,24 +1,18 @@
-import glob
 import os
 import json
 from base64 import b64decode
-import pathlib
-import shutil
-import sys
 from django.conf import settings
-from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect
+from django.http import Http404
 from django.views.generic import TemplateView, ListView, DetailView
 from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateView,
                                        DeleteView)
 from django.views.generic.detail import SingleObjectMixin
-from django.core.exceptions import SuspiciousFileOperation
 from django.http import HttpResponse
 from django.urls import reverse_lazy
 from django.db import IntegrityError
-from django.db.models import Q, Count, Sum, When, Case
+from django.db.models import Q, Count, Sum
-from django.forms import Form, ValidationError
+from django.forms import ValidationError
 from django.utils.text import slugify
-from django.utils._os import safe_join
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from common.utils import append_uri_params
@@ -61,7 +55,7 @@ class DashboardView(TemplateView):
         # Disk usage
         disk_usage = Media.objects.filter(
             downloaded=True, downloaded_filesize__isnull=False
-        ).defer('metadata').aggregate(Sum('downloaded_filesize'))
+        ).aggregate(Sum('downloaded_filesize'))
         data['disk_usage_bytes'] = disk_usage['downloaded_filesize__sum']
         if not data['disk_usage_bytes']:
             data['disk_usage_bytes'] = 0
@@ -72,19 +66,18 @@ class DashboardView(TemplateView):
             data['average_bytes_per_media'] = 0
         # Latest downloads
         data['latest_downloads'] = Media.objects.filter(
-            downloaded=True, downloaded_filesize__isnull=False
+            downloaded=True
-        ).defer('metadata').order_by('-download_date')[:10]
+        ).order_by('-download_date')[:10]
         # Largest downloads
         data['largest_downloads'] = Media.objects.filter(
             downloaded=True, downloaded_filesize__isnull=False
-        ).defer('metadata').order_by('-downloaded_filesize')[:10]
+        ).order_by('-downloaded_filesize')[:10]
         # UID and GID
         data['uid'] = os.getuid()
         data['gid'] = os.getgid()
         # Config and download locations
         data['config_dir'] = str(settings.CONFIG_BASE_DIR)
         data['downloads_dir'] = str(settings.DOWNLOAD_ROOT)
-        data['database_connection'] = settings.DATABASE_CONNECTION_STR
         return data
 
 
@@ -98,27 +91,8 @@ class SourcesView(ListView):
     paginate_by = settings.SOURCES_PER_PAGE
     messages = {
         'source-deleted': _('Your selected source has been deleted.'),
-        'source-refreshed': _('The source has been scheduled to be synced now.')
     }
 
-    def get(self, *args, **kwargs):
-        if args[0].path.startswith("/source-sync-now/"):
-            sobj = Source.objects.get(pk=kwargs["pk"])
-            if sobj is None:
-                return HttpResponseNotFound()
-            
-            verbose_name = _('Index media from source "{}" once')
-            index_source_task(
-                str(sobj.pk),
-                queue=str(sobj.pk),
-                repeat=0,
-                verbose_name=verbose_name.format(sobj.name))
-            url = reverse_lazy('sync:sources')
-            url = append_uri_params(url, {'message': 'source-refreshed'})
-            return HttpResponseRedirect(url)
-        else:
-            return super().get(self, *args, **kwargs)    
-
     def __init__(self, *args, **kwargs):
         self.message = None
         super().__init__(*args, **kwargs)
@@ -130,10 +104,7 @@ class SourcesView(ListView):
 
     def get_queryset(self):
         all_sources = Source.objects.all().order_by('name')
-        return all_sources.annotate(
+        return all_sources.annotate(media_count=Count('media_source'))
-            media_count=Count('media_source'),
-            downloaded_count=Count(Case(When(media_source__downloaded=True, then=1)))
-        )
 
     def get_context_data(self, *args, **kwargs):
         data = super().get_context_data(*args, **kwargs)
@@ -196,7 +167,7 @@ class ValidateSourceView(FormView):
     validation_urls = {
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL: {
             'scheme': 'https',
-            'domains': ('m.youtube.com', 'www.youtube.com'),
+            'domain': 'www.youtube.com',
             'path_regex': '^\/(c\/)?([^\/]+)(\/videos)?$',
             'path_must_not_match': ('/playlist', '/c/playlist'),
             'qs_args': [],
@@ -205,7 +176,7 @@ class ValidateSourceView(FormView):
         },
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: {
             'scheme': 'https',
-            'domains': ('m.youtube.com', 'www.youtube.com'),
+            'domain': 'www.youtube.com',
             'path_regex': '^\/channel\/([^\/]+)(\/videos)?$',
             'path_must_not_match': ('/playlist', '/c/playlist'),
             'qs_args': [],
@@ -214,7 +185,7 @@ class ValidateSourceView(FormView):
         },
         Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: {
             'scheme': 'https',
-            'domains': ('m.youtube.com', 'www.youtube.com'),
+            'domain': 'www.youtube.com',
             'path_regex': '^\/(playlist|watch)$',
             'path_must_not_match': (),
             'qs_args': ('list',),
@@ -294,58 +265,23 @@ class ValidateSourceView(FormView):
         return append_uri_params(url, fields)
 
 
-class EditSourceMixin:
+class AddSourceView(CreateView):
-    model = Source
-    fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format',
-              'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
-              'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution',
-              'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback',
-              'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
-              'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
-              'auto_subtitles', 'sub_langs')
-    errors = {
-        'invalid_media_format': _('Invalid media format, the media format contains '
-                                  'errors or is empty. Check the table at the end of '
-                                  'this page for valid media name variables'),
-        'dir_outside_dlroot': _('You cannot specify a directory outside of the '
-                                'base directory (%BASEDIR%)')
-    }
-
-    def form_valid(self, form: Form):
-        # Perform extra validation to make sure the media_format is valid
-        obj = form.save(commit=False)
-        source_type = form.cleaned_data['media_format']
-        example_media_file = obj.get_example_media_format()
-        
-        if example_media_file == '':
-            form.add_error(
-                'media_format',
-                ValidationError(self.errors['invalid_media_format'])
-            )
-
-        # Check for suspicious file path(s)
-        try:
-            targetCheck = form.cleaned_data['directory']+"/.virt"
-            newdir = safe_join(settings.DOWNLOAD_ROOT,targetCheck)
-        except SuspiciousFileOperation:
-            form.add_error(
-                'directory',
-                ValidationError(self.errors['dir_outside_dlroot'].replace("%BASEDIR%",str(settings.DOWNLOAD_ROOT)))
-            )
-
-        if form.errors:
-            return super().form_invalid(form)
-
-        return super().form_valid(form)
-
-
-class AddSourceView(EditSourceMixin, CreateView):
     '''
         Adds a new source, optionally takes some initial data querystring values to
         prepopulate some of the more unclear values.
     '''
 
     template_name = 'sync/source-add.html'
+    model = Source
+    fields = ('source_type', 'key', 'name', 'directory', 'media_format',
+              'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
+              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
+              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
+    errors = {
+        'invalid_media_format': _('Invalid media format, the media format contains '
+                                  'errors or is empty. Check the table at the end of '
+                                  'this page for valid media name variables'),
+    }
 
     def __init__(self, *args, **kwargs):
         self.prepopulated_data = {}
@@ -372,6 +308,20 @@ class AddSourceView(EditSourceMixin, CreateView):
             initial[k] = v
         return initial
 
+    def form_valid(self, form):
+        # Perform extra validation to make sure the media_format is valid
+        obj = form.save(commit=False)
+        source_type = form.cleaned_data['media_format']
+        example_media_file = obj.get_example_media_format()
+        if example_media_file == '':
+            form.add_error(
+                'media_format',
+                ValidationError(self.errors['invalid_media_format'])
+            )
+        if form.errors:
+            return super().form_invalid(form)
+        return super().form_valid(form)
+
     def get_success_url(self):
         url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
         return append_uri_params(url, {'message': 'source-created'})
@@ -406,13 +356,37 @@ class SourceView(DetailView):
             error_message = get_error_message(error)
             setattr(error, 'error_message', error_message)
             data['errors'].append(error)
-        data['media'] = Media.objects.filter(source=self.object).order_by('-published').defer('metadata')
+        data['media'] = Media.objects.filter(source=self.object).order_by('-published')
         return data
 
 
-class UpdateSourceView(EditSourceMixin, UpdateView):
+class UpdateSourceView(UpdateView):
 
     template_name = 'sync/source-update.html'
+    model = Source
+    fields = ('source_type', 'key', 'name', 'directory', 'media_format',
+              'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
+              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
+              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
+    errors = {
+        'invalid_media_format': _('Invalid media format, the media format contains '
+                                  'errors or is empty. Check the table at the end of '
+                                  'this page for valid media name variables'),
+    }
+
+    def form_valid(self, form):
+        # Perform extra validation to make sure the media_format is valid
+        obj = form.save(commit=False)
+        source_type = form.cleaned_data['media_format']
+        example_media_file = obj.get_example_media_format()
+        if example_media_file == '':
+            form.add_error(
+                'media_format',
+                ValidationError(self.errors['invalid_media_format'])
+            )
+        if form.errors:
+            return super().form_invalid(form)
+        return super().form_valid(form)
 
     def get_success_url(self):
         url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
@@ -437,13 +411,12 @@ class DeleteSourceView(DeleteView, FormMixin):
             source = self.get_object()
             for media in Media.objects.filter(source=source):
                 if media.media_file:
-                    file_path = media.media_file.path
+                    # Delete the media file
-                    matching_files = glob.glob(os.path.splitext(file_path)[0] + '.*')
+                    delete_file(media.media_file.name)
-                    for file in matching_files:
+                    # Delete thumbnail copy if it exists
-                        delete_file(file)
+                    delete_file(media.thumbpath)
-            directory_path = source.directory_path
+                    # Delete NFO file if it exists
-            if os.path.exists(directory_path):
+                    delete_file(media.nfopath)
-                shutil.rmtree(directory_path, True)
         return super().post(request, *args, **kwargs)
 
     def get_success_url(self):
@@ -465,8 +438,6 @@ class MediaView(ListView):
 
     def __init__(self, *args, **kwargs):
         self.filter_source = None
-        self.show_skipped = False
-        self.only_skipped = False
         super().__init__(*args, **kwargs)
 
     def dispatch(self, request, *args, **kwargs):
@@ -476,30 +447,13 @@ class MediaView(ListView):
                 self.filter_source = Source.objects.get(pk=filter_by)
             except Source.DoesNotExist:
                 self.filter_source = None
-        show_skipped = request.GET.get('show_skipped', '').strip()
-        if show_skipped == 'yes':
-            self.show_skipped = True
-        if not self.show_skipped:
-            only_skipped = request.GET.get('only_skipped', '').strip()
-            if only_skipped == 'yes':
-                self.only_skipped = True
         return super().dispatch(request, *args, **kwargs)
 
     def get_queryset(self):
         if self.filter_source:
-            if self.show_skipped:
+            q = Media.objects.filter(source=self.filter_source)
-                q = Media.objects.filter(source=self.filter_source)
-            elif self.only_skipped:
-                q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=True) | Q(manual_skip=True)))
-            else:
-                q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=False) & Q(manual_skip=False)))
         else:
-            if self.show_skipped:
+            q = Media.objects.all()
-                q = Media.objects.all()
-            elif self.only_skipped:
-                q = Media.objects.filter(Q(skip=True)|Q(manual_skip=True))
-            else:
-                q = Media.objects.filter(Q(skip=False)&Q(manual_skip=False))
         return q.order_by('-published', '-created')
 
     def get_context_data(self, *args, **kwargs):
@@ -510,8 +464,6 @@ class MediaView(ListView):
             message = str(self.messages.get('filter', ''))
             data['message'] = message.format(name=self.filter_source.name)
             data['source'] = self.filter_source
-        data['show_skipped'] = self.show_skipped
-        data['only_skipped'] = self.only_skipped
         return data
 
 
@@ -654,15 +606,13 @@ class MediaSkipView(FormView, SingleObjectMixin):
         delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),))
         # If the media file exists on disk, delete it
         if self.object.media_file_exists:
-            # Delete all files which contains filename
+            delete_file(self.object.media_file.path)
-            filepath = self.object.media_file.path
+            self.object.media_file = None
-            barefilepath, fileext = os.path.splitext(filepath)
+            # If the media has an associated thumbnail copied, also delete it
-            # Get all files that start with the bare file path
+            delete_file(self.object.thumbpath)
-            all_related_files = glob.glob(f'{barefilepath}.*')
+            # If the media has an associated NFO file with it, also delete it
-            for file in all_related_files:
+            delete_file(self.object.nfopath)
-                delete_file(file)
         # Reset all download data
-        self.object.metadata = None
         self.object.downloaded = False
         self.object.downloaded_audio_codec = None
         self.object.downloaded_video_codec = None
@@ -672,7 +622,6 @@ class MediaSkipView(FormView, SingleObjectMixin):
         self.object.downloaded_filesize = None
         # Mark it to be skipped
         self.object.skip = True
-        self.object.manual_skip = True
         self.object.save()
         return super().form_valid(form)
 
@@ -701,7 +650,6 @@ class MediaEnableView(FormView, SingleObjectMixin):
     def form_valid(self, form):
         # Mark it as not skipped
         self.object.skip = False
-        self.object.manual_skip = False
         self.object.save()
         return super().form_valid(form)
 
@@ -710,52 +658,6 @@ class MediaEnableView(FormView, SingleObjectMixin):
         return append_uri_params(url, {'message': 'enabled'})
 
 
-class MediaContent(DetailView):
-    '''
-        Redirect to nginx to download the file
-    '''
-    model = Media
-
-    def __init__(self, *args, **kwargs):
-        self.object = None
-        super().__init__(*args, **kwargs)
-
-    def dispatch(self, request, *args, **kwargs):
-        self.object = self.get_object()
-        # development direct file stream - DO NOT USE PRODUCTIVLY
-        if settings.DEBUG and 'runserver' in sys.argv:
-            # get media URL
-            pth = self.object.media_file.url
-            # remove "/media-data/"
-            pth = pth.split("/media-data/",1)[1]
-            # remove "/" (incase of absolute path)
-            pth = pth.split(str(settings.DOWNLOAD_ROOT).lstrip("/"),1)
-
-            # if we do not have a "/" at the beginning, it is not a absolute path...
-            if len(pth) > 1:
-                pth = pth[1]
-            else:
-                pth = pth[0]
-            
-            
-            # build final path
-            filepth = pathlib.Path(str(settings.DOWNLOAD_ROOT) + pth)
-            
-            if filepth.exists():
-                # return file
-                response = FileResponse(open(filepth,'rb'))
-                return response
-            else:
-                return HttpResponseNotFound()
-            
-        else:
-            headers = {
-                'Content-Type': self.object.content_type,
-                'X-Accel-Redirect': self.object.media_file.url,
-            }
-            return HttpResponse(headers=headers)
-
-
 class TasksView(ListView):
     '''
         A list of tasks queued to be completed. This is, for example, scraping for new
@@ -1114,4 +1016,4 @@ class UpdateMediaServerView(FormView, SingleObjectMixin):
 
     def get_success_url(self):
         url = reverse_lazy('sync:mediaserver', kwargs={'pk': self.object.pk})
-        return append_uri_params(url, {'message': 'updated'})
+        return append_uri_params(url, {'message': 'updated'})