Compare commits
No commits in common. "main" and "v0.9" have entirely different histories.
|
|
@ -4,10 +4,12 @@ env:
|
|||
IMAGE_NAME: tubesync
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
test:
|
||||
|
|
@ -25,7 +27,7 @@ jobs:
|
|||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pipenv
|
||||
pipenv install --system --skip-lock
|
||||
pipenv install --system
|
||||
- name: Set up Django environment
|
||||
run: cp tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py
|
||||
- name: Run Django tests
|
||||
|
|
@ -33,24 +35,13 @@ jobs:
|
|||
containerise:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
- uses: actions/checkout@v2
|
||||
- name: Build the container image
|
||||
run: docker build . --tag $IMAGE_NAME
|
||||
- name: Log into GitHub Container Registry
|
||||
run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
|
||||
- name: Lowercase github username for ghcr
|
||||
id: string
|
||||
uses: ASzc/change-string-case-action@v1
|
||||
with:
|
||||
string: ${{ github.actor }}
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v2
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest
|
||||
cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest
|
||||
cache-to: type=inline
|
||||
build-args: |
|
||||
IMAGE_NAME=${{ env.IMAGE_NAME }}
|
||||
- name: Push image to GitHub Container Registry
|
||||
run: |
|
||||
LATEST_TAG=ghcr.io/meeb/$IMAGE_NAME:latest
|
||||
docker tag $IMAGE_NAME $LATEST_TAG
|
||||
docker push $LATEST_TAG
|
||||
|
|
|
|||
|
|
@ -11,28 +11,18 @@ jobs:
|
|||
containerise:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
- uses: actions/checkout@v2
|
||||
- name: Get tag
|
||||
id: tag
|
||||
uses: dawidd6/action-get-tag@v1
|
||||
- uses: docker/build-push-action@v2
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
id: vars
|
||||
run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
|
||||
- name: Build the container image
|
||||
run: docker build . --tag $IMAGE_NAME
|
||||
- name: Log into GitHub Container Registry
|
||||
run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
|
||||
- name: Lowercase github username for ghcr
|
||||
id: string
|
||||
uses: ASzc/change-string-case-action@v1
|
||||
with:
|
||||
string: ${{ github.actor }}
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v2
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
|
||||
cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
|
||||
cache-to: type=inline
|
||||
build-args: |
|
||||
IMAGE_NAME=${{ env.IMAGE_NAME }}
|
||||
- name: Push image to GitHub Container Registry
|
||||
env:
|
||||
RELEASE_TAG: ${{ steps.vars.outputs.tag }}
|
||||
run: |
|
||||
REF_TAG=ghcr.io/meeb/$IMAGE_NAME:$RELEASE_TAG
|
||||
docker tag $IMAGE_NAME $REF_TAG
|
||||
docker push $REF_TAG
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
.DS_Store
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
|
|
@ -131,6 +130,3 @@ dmypy.json
|
|||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
Pipfile.lock
|
||||
.vscode/launch.json
|
||||
|
|
|
|||
124
Dockerfile
124
Dockerfile
|
|
@ -1,9 +1,8 @@
|
|||
FROM debian:bookworm-slim
|
||||
FROM debian:buster-slim
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
ARG S6_VERSION="3.1.5.0"
|
||||
ARG FFMPEG_DATE="autobuild-2023-11-29-14-19"
|
||||
ARG FFMPEG_VERSION="112875-g47e214245b"
|
||||
ARG ARCH="amd64"
|
||||
ARG S6_VERSION="2.1.0.2"
|
||||
ARG FFMPEG_VERSION="4.3.1"
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive" \
|
||||
HOME="/root" \
|
||||
|
|
@ -11,102 +10,63 @@ ENV DEBIAN_FRONTEND="noninteractive" \
|
|||
LANG="en_US.UTF-8" \
|
||||
LC_ALL="en_US.UTF-8" \
|
||||
TERM="xterm" \
|
||||
S6_CMD_WAIT_FOR_SERVICES_MAXTIME="0"
|
||||
S6_EXPECTED_SHA256="52460473413601ff7a84ae690b161a074217ddc734990c2cdee9847166cf669e" \
|
||||
S6_DOWNLOAD="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${ARCH}.tar.gz" \
|
||||
FFMPEG_EXPECTED_SHA256="47d95c0129fba27d051748a442a44a73ce1bd38d1e3f9fe1e9dd7258c7581fa5" \
|
||||
FFMPEG_DOWNLOAD="https://tubesync.sfo2.digitaloceanspaces.com/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static.tar.xz"
|
||||
|
||||
|
||||
# Install third party software
|
||||
RUN export ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
|
||||
"linux/amd64") echo "amd64" ;; \
|
||||
"linux/arm64") echo "aarch64" ;; \
|
||||
*) echo "" ;; esac) && \
|
||||
export S6_ARCH_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
|
||||
"linux/amd64") echo "65d0d0f353d2ff9d0af202b268b4bf53a9948a5007650854855c729289085739" ;; \
|
||||
"linux/arm64") echo "3fbd14201473710a592b2189e81f00f3c8998e96d34f16bd2429c35d1bc36d00" ;; \
|
||||
*) echo "" ;; esac) && \
|
||||
export S6_DOWNLOAD_ARCH=$(case ${TARGETPLATFORM:-linux/amd64} in \
|
||||
"linux/amd64") echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-x86_64.tar.xz" ;; \
|
||||
"linux/arm64") echo "https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-aarch64.tar.xz" ;; \
|
||||
*) echo "" ;; esac) && \
|
||||
export FFMPEG_EXPECTED_SHA256=$(case ${TARGETPLATFORM:-linux/amd64} in \
|
||||
"linux/amd64") echo "36bac8c527bf390603416f749ab0dd860142b0a66f0865b67366062a9c286c8b" ;; \
|
||||
"linux/arm64") echo "8f36e45d99d2367a5c0c220ee3164fa48f4f0cec35f78204ccced8dc303bfbdc" ;; \
|
||||
*) echo "" ;; esac) && \
|
||||
export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \
|
||||
"linux/amd64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linux64-gpl.tar.xz" ;; \
|
||||
"linux/arm64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/${FFMPEG_DATE}/ffmpeg-N-${FFMPEG_VERSION}-linuxarm64-gpl.tar.xz" ;; \
|
||||
*) echo "" ;; esac) && \
|
||||
export S6_NOARCH_EXPECTED_SHA256="fd80c231e8ae1a0667b7ae2078b9ad0e1269c4d117bf447a4506815a700dbff3" && \
|
||||
export S6_DOWNLOAD_NOARCH="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-noarch.tar.xz" && \
|
||||
echo "Building for arch: ${ARCH}|${ARCH44}, downloading S6 from: ${S6_DOWNLOAD}}, expecting S6 SHA256: ${S6_EXPECTED_SHA256}" && \
|
||||
set -x && \
|
||||
RUN set -x && \
|
||||
apt-get update && \
|
||||
apt-get -y --no-install-recommends install locales && \
|
||||
echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
|
||||
locale-gen en_US.UTF-8 && \
|
||||
# Install required distro packages
|
||||
apt-get -y --no-install-recommends install curl ca-certificates binutils xz-utils && \
|
||||
apt-get -y --no-install-recommends install curl xz-utils ca-certificates binutils && \
|
||||
# Install s6
|
||||
curl -L ${S6_DOWNLOAD_NOARCH} --output /tmp/s6-overlay-noarch.tar.xz && \
|
||||
echo "${S6_NOARCH_EXPECTED_SHA256} /tmp/s6-overlay-noarch.tar.xz" | sha256sum -c - && \
|
||||
tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && \
|
||||
curl -L ${S6_DOWNLOAD_ARCH} --output /tmp/s6-overlay-${ARCH}.tar.xz && \
|
||||
echo "${S6_ARCH_EXPECTED_SHA256} /tmp/s6-overlay-${ARCH}.tar.xz" | sha256sum -c - && \
|
||||
tar -C / -Jxpf /tmp/s6-overlay-${ARCH}.tar.xz && \
|
||||
curl -L ${S6_DOWNLOAD} --output /tmp/s6-overlay-${ARCH}.tar.gz && \
|
||||
sha256sum /tmp/s6-overlay-${ARCH}.tar.gz && \
|
||||
echo "${S6_EXPECTED_SHA256} /tmp/s6-overlay-${ARCH}.tar.gz" | sha256sum -c - && \
|
||||
tar xzf /tmp/s6-overlay-${ARCH}.tar.gz -C / && \
|
||||
# Install ffmpeg
|
||||
echo "Building for arch: ${ARCH}|${ARCH44}, downloading FFMPEG from: ${FFMPEG_DOWNLOAD}, expecting FFMPEG SHA256: ${FFMPEG_EXPECTED_SHA256}" && \
|
||||
curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg-${ARCH}.tar.xz && \
|
||||
sha256sum /tmp/ffmpeg-${ARCH}.tar.xz && \
|
||||
echo "${FFMPEG_EXPECTED_SHA256} /tmp/ffmpeg-${ARCH}.tar.xz" | sha256sum -c - && \
|
||||
tar -xf /tmp/ffmpeg-${ARCH}.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffmpeg" && \
|
||||
tar -xf /tmp/ffmpeg-${ARCH}.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffprobe" && \
|
||||
curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg-${ARCH}-static.tar.xz && \
|
||||
echo "${FFMPEG_EXPECTED_SHA256} /tmp/ffmpeg-${ARCH}-static.tar.xz" | sha256sum -c - && \
|
||||
xz --decompress /tmp/ffmpeg-${ARCH}-static.tar.xz && \
|
||||
tar -xvf /tmp/ffmpeg-${ARCH}-static.tar -C /tmp && \
|
||||
install -v -s -g root -o root -m 0755 -s /tmp/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static/ffmpeg -t /usr/local/bin && \
|
||||
# Clean up
|
||||
rm -rf /tmp/s6-overlay-${ARCH}.tar.gz && \
|
||||
rm -rf /tmp/ffmpeg-${ARCH}.tar.xz && \
|
||||
apt-get -y autoremove --purge curl binutils xz-utils
|
||||
rm -rf /tmp/ffmpeg-${ARCH}-static.tar && \
|
||||
rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static && \
|
||||
apt-get -y autoremove --purge curl xz-utils binutils
|
||||
|
||||
# Copy app
|
||||
COPY tubesync /app
|
||||
COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings.py
|
||||
|
||||
# Copy over pip.conf to use piwheels
|
||||
COPY pip.conf /etc/pip.conf
|
||||
# Append container bundled software versions
|
||||
RUN echo "ffmpeg_version = '${FFMPEG_VERSION}-static'" >> /app/common/third_party_versions.py
|
||||
|
||||
# Add Pipfile
|
||||
COPY Pipfile /app/Pipfile
|
||||
COPY Pipfile.lock /app/Pipfile.lock
|
||||
|
||||
# Switch workdir to the the app
|
||||
WORKDIR /app
|
||||
|
||||
# Set up the app
|
||||
RUN set -x && \
|
||||
apt-get update && \
|
||||
# Install required distro packages
|
||||
apt-get -y install nginx-light && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
python3 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
pipenv \
|
||||
gcc \
|
||||
g++ \
|
||||
make \
|
||||
pkgconf \
|
||||
default-libmysqlclient-dev \
|
||||
libmariadb3 \
|
||||
postgresql-common \
|
||||
libpq-dev \
|
||||
libpq5 \
|
||||
libjpeg62-turbo \
|
||||
libwebp7 \
|
||||
libjpeg-dev \
|
||||
zlib1g-dev \
|
||||
libwebp-dev \
|
||||
redis-server && \
|
||||
apt-get -y --no-install-recommends install python3 python3-setuptools python3-pip python3-dev gcc make && \
|
||||
# Install pipenv
|
||||
pip3 --disable-pip-version-check install pipenv && \
|
||||
# Create a 'app' user which the application will run as
|
||||
groupadd app && \
|
||||
useradd -M -d /app -s /bin/false -g app app && \
|
||||
# Install non-distro packages
|
||||
PIPENV_VERBOSITY=64 pipenv install --system --skip-lock && \
|
||||
pipenv install --system && \
|
||||
# Make absolutely sure we didn't accidentally bundle a SQLite dev database
|
||||
rm -rf /app/db.sqlite3 && \
|
||||
# Run any required app commands
|
||||
|
|
@ -119,19 +79,10 @@ RUN set -x && \
|
|||
mkdir -p /downloads/video && \
|
||||
# Clean up
|
||||
rm /app/Pipfile && \
|
||||
rm /app/Pipfile.lock && \
|
||||
pipenv --clear && \
|
||||
apt-get -y autoremove --purge \
|
||||
python3-pip \
|
||||
python3-dev \
|
||||
gcc \
|
||||
g++ \
|
||||
make \
|
||||
default-libmysqlclient-dev \
|
||||
postgresql-common \
|
||||
libpq-dev \
|
||||
libjpeg-dev \
|
||||
zlib1g-dev \
|
||||
libwebp-dev && \
|
||||
pip3 --disable-pip-version-check uninstall -y pipenv wheel virtualenv && \
|
||||
apt-get -y autoremove --purge python3-pip python3-dev gcc make && \
|
||||
apt-get -y autoremove && \
|
||||
apt-get -y autoclean && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
|
|
@ -141,12 +92,7 @@ RUN set -x && \
|
|||
rm -rf /root && \
|
||||
mkdir -p /root && \
|
||||
chown root:root /root && \
|
||||
chmod 0755 /root
|
||||
|
||||
# Append software versions
|
||||
RUN set -x && \
|
||||
FFMPEG_VERSION=$(/usr/local/bin/ffmpeg -version | head -n 1 | awk '{ print $3 }') && \
|
||||
echo "ffmpeg_version = '${FFMPEG_VERSION}'" >> /app/common/third_party_versions.py
|
||||
chmod 0700 /root
|
||||
|
||||
# Copy root
|
||||
COPY config/root /
|
||||
|
|
@ -156,7 +102,7 @@ HEALTHCHECK --interval=1m --timeout=10s CMD /app/healthcheck.py http://127.0.0.1
|
|||
|
||||
# ENVS and ports
|
||||
ENV PYTHONPATH "/app:${PYTHONPATH}"
|
||||
EXPOSE 4848
|
||||
EXPOSE 8080
|
||||
|
||||
# Volumes
|
||||
VOLUME ["/config", "/downloads"]
|
||||
|
|
|
|||
22
Makefile
22
Makefile
|
|
@ -8,17 +8,17 @@ all: clean build
|
|||
|
||||
|
||||
dev:
|
||||
$(python) tubesync/manage.py runserver
|
||||
$(python) app/manage.py runserver
|
||||
|
||||
|
||||
build:
|
||||
mkdir -p tubesync/media
|
||||
mkdir -p tubesync/static
|
||||
$(python) tubesync/manage.py collectstatic --noinput
|
||||
mkdir -p app/media
|
||||
mkdir -p app/static
|
||||
$(python) app/manage.py collectstatic --noinput
|
||||
|
||||
|
||||
clean:
|
||||
rm -rf tubesync/static
|
||||
rm -rf app/static
|
||||
|
||||
|
||||
container: clean
|
||||
|
|
@ -29,13 +29,5 @@ runcontainer:
|
|||
$(docker) run --rm --name $(name) --env-file dev.env --log-opt max-size=50m -ti -p 4848:4848 $(image)
|
||||
|
||||
|
||||
stopcontainer:
|
||||
$(docker) stop $(name)
|
||||
|
||||
|
||||
test: build
|
||||
cd tubesync && $(python) manage.py test --verbosity=2 && cd ..
|
||||
|
||||
|
||||
shell:
|
||||
cd tubesync && $(python) manage.py shell
|
||||
test:
|
||||
$(python) app/manage.py test --verbosity=2
|
||||
|
|
|
|||
15
Pipfile
15
Pipfile
|
|
@ -4,10 +4,9 @@ url = "https://pypi.org/simple"
|
|||
verify_ssl = true
|
||||
|
||||
[dev-packages]
|
||||
autopep8 = "*"
|
||||
|
||||
[packages]
|
||||
django = "~=3.2"
|
||||
django = "*"
|
||||
django-sass-processor = "*"
|
||||
libsass = "*"
|
||||
pillow = "*"
|
||||
|
|
@ -15,11 +14,9 @@ whitenoise = "*"
|
|||
gunicorn = "*"
|
||||
django-compressor = "*"
|
||||
httptools = "*"
|
||||
youtube-dl = "*"
|
||||
django-background-tasks = "*"
|
||||
django-basicauth = "*"
|
||||
psycopg2-binary = "*"
|
||||
mysqlclient = "*"
|
||||
yt-dlp = "*"
|
||||
redis = "*"
|
||||
hiredis = "*"
|
||||
requests = {extras = ["socks"], version = "*"}
|
||||
requests = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,251 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "a4bb556fc61ee4583f9588980450b071814298ee4d1a1023fad149c14d14aaba"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"name": "pypi",
|
||||
"url": "https://pypi.org/simple",
|
||||
"verify_ssl": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"default": {
|
||||
"asgiref": {
|
||||
"hashes": [
|
||||
"sha256:5ee950735509d04eb673bd7f7120f8fa1c9e2df495394992c73234d526907e17",
|
||||
"sha256:7162a3cb30ab0609f1a4c95938fd73e8604f63bdba516a7f7d64b83ff09478f0"
|
||||
],
|
||||
"version": "==3.3.1"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
|
||||
"sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
|
||||
],
|
||||
"version": "==2020.12.5"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
"sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
|
||||
"sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
|
||||
],
|
||||
"version": "==4.0.0"
|
||||
},
|
||||
"django": {
|
||||
"hashes": [
|
||||
"sha256:169e2e7b4839a7910b393eec127fd7cbae62e80fa55f89c6510426abf673fe5f",
|
||||
"sha256:c6c0462b8b361f8691171af1fb87eceb4442da28477e12200c40420176206ba7"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.1.6"
|
||||
},
|
||||
"django-appconf": {
|
||||
"hashes": [
|
||||
"sha256:1b1d0e1069c843ebe8ae5aa48ec52403b1440402b320c3e3a206a0907e97bb06",
|
||||
"sha256:be58deb54a43d77d2e1621fe59f787681376d3cd0b8bd8e4758ef6c3a6453380"
|
||||
],
|
||||
"version": "==1.0.4"
|
||||
},
|
||||
"django-background-tasks": {
|
||||
"hashes": [
|
||||
"sha256:e1b19e8d495a276c9d64c5a1ff8b41132f75d2f58e45be71b78650dad59af9de"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.2.5"
|
||||
},
|
||||
"django-compat": {
|
||||
"hashes": [
|
||||
"sha256:3ac9a3bedc56b9365d9eb241bc5157d0c193769bf995f9a78dc1bc24e7c2331b"
|
||||
],
|
||||
"version": "==1.0.15"
|
||||
},
|
||||
"django-compressor": {
|
||||
"hashes": [
|
||||
"sha256:57ac0a696d061e5fc6fbc55381d2050f353b973fb97eee5593f39247bc0f30af",
|
||||
"sha256:d2ed1c6137ddaac5536233ec0a819e14009553fee0a869bea65d03e5285ba74f"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.4"
|
||||
},
|
||||
"django-sass-processor": {
|
||||
"hashes": [
|
||||
"sha256:9b46a12ca8bdcb397d46fbcc49e6a926ff9f76a93c5efeb23b495419fd01fc7a"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.8.2"
|
||||
},
|
||||
"gunicorn": {
|
||||
"hashes": [
|
||||
"sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626",
|
||||
"sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==20.0.4"
|
||||
},
|
||||
"httptools": {
|
||||
"hashes": [
|
||||
"sha256:0a4b1b2012b28e68306575ad14ad5e9120b34fccd02a81eb08838d7e3bbb48be",
|
||||
"sha256:3592e854424ec94bd17dc3e0c96a64e459ec4147e6d53c0a42d0ebcef9cb9c5d",
|
||||
"sha256:41b573cf33f64a8f8f3400d0a7faf48e1888582b6f6e02b82b9bd4f0bf7497ce",
|
||||
"sha256:56b6393c6ac7abe632f2294da53f30d279130a92e8ae39d8d14ee2e1b05ad1f2",
|
||||
"sha256:86c6acd66765a934e8730bf0e9dfaac6fdcf2a4334212bd4a0a1c78f16475ca6",
|
||||
"sha256:96da81e1992be8ac2fd5597bf0283d832287e20cb3cfde8996d2b00356d4e17f",
|
||||
"sha256:96eb359252aeed57ea5c7b3d79839aaa0382c9d3149f7d24dd7172b1bcecb009",
|
||||
"sha256:a2719e1d7a84bb131c4f1e0cb79705034b48de6ae486eb5297a139d6a3296dce",
|
||||
"sha256:ac0aa11e99454b6a66989aa2d44bca41d4e0f968e395a0a8f164b401fefe359a",
|
||||
"sha256:bc3114b9edbca5a1eb7ae7db698c669eb53eb8afbbebdde116c174925260849c",
|
||||
"sha256:fa3cd71e31436911a44620473e873a256851e1f53dee56669dae403ba41756a4",
|
||||
"sha256:fea04e126014169384dee76a153d4573d90d0cbd1d12185da089f73c78390437"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.1.1"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
|
||||
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
|
||||
],
|
||||
"version": "==2.10"
|
||||
},
|
||||
"libsass": {
|
||||
"hashes": [
|
||||
"sha256:1521d2a8d4b397c6ec90640a1f6b5529077035efc48ef1c2e53095544e713d1b",
|
||||
"sha256:1b2d415bbf6fa7da33ef46e549db1418498267b459978eff8357e5e823962d35",
|
||||
"sha256:25ebc2085f5eee574761ccc8d9cd29a9b436fc970546d5ef08c6fa41eb57dff1",
|
||||
"sha256:2ae806427b28bc1bb7cb0258666d854fcf92ba52a04656b0b17ba5e190fb48a9",
|
||||
"sha256:4a246e4b88fd279abef8b669206228c92534d96ddcd0770d7012088c408dff23",
|
||||
"sha256:553e5096414a8d4fb48d0a48f5a038d3411abe254d79deac5e008516c019e63a",
|
||||
"sha256:697f0f9fa8a1367ca9ec6869437cb235b1c537fc8519983d1d890178614a8903",
|
||||
"sha256:a8fd4af9f853e8bf42b1425c5e48dd90b504fa2e70d7dac5ac80b8c0a5a5fe85",
|
||||
"sha256:c9411fec76f480ffbacc97d8188322e02a5abca6fc78e70b86a2a2b421eae8a2",
|
||||
"sha256:daa98a51086d92aa7e9c8871cf1a8258124b90e2abf4697852a3dca619838618",
|
||||
"sha256:e0e60836eccbf2d9e24ec978a805cd6642fa92515fbd95e3493fee276af76f8a",
|
||||
"sha256:e64ae2587f1a683e831409aad03ba547c245ef997e1329fffadf7a866d2510b8",
|
||||
"sha256:f6852828e9e104d2ce0358b73c550d26dd86cc3a69439438c3b618811b9584f5"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.20.1"
|
||||
},
|
||||
"pillow": {
|
||||
"hashes": [
|
||||
"sha256:165c88bc9d8dba670110c689e3cc5c71dbe4bfb984ffa7cbebf1fac9554071d6",
|
||||
"sha256:1d208e670abfeb41b6143537a681299ef86e92d2a3dac299d3cd6830d5c7bded",
|
||||
"sha256:22d070ca2e60c99929ef274cfced04294d2368193e935c5d6febfd8b601bf865",
|
||||
"sha256:2353834b2c49b95e1313fb34edf18fca4d57446675d05298bb694bca4b194174",
|
||||
"sha256:39725acf2d2e9c17356e6835dccebe7a697db55f25a09207e38b835d5e1bc032",
|
||||
"sha256:3de6b2ee4f78c6b3d89d184ade5d8fa68af0848f9b6b6da2b9ab7943ec46971a",
|
||||
"sha256:47c0d93ee9c8b181f353dbead6530b26980fe4f5485aa18be8f1fd3c3cbc685e",
|
||||
"sha256:5e2fe3bb2363b862671eba632537cd3a823847db4d98be95690b7e382f3d6378",
|
||||
"sha256:604815c55fd92e735f9738f65dabf4edc3e79f88541c221d292faec1904a4b17",
|
||||
"sha256:6c5275bd82711cd3dcd0af8ce0bb99113ae8911fc2952805f1d012de7d600a4c",
|
||||
"sha256:731ca5aabe9085160cf68b2dbef95fc1991015bc0a3a6ea46a371ab88f3d0913",
|
||||
"sha256:7612520e5e1a371d77e1d1ca3a3ee6227eef00d0a9cddb4ef7ecb0b7396eddf7",
|
||||
"sha256:7916cbc94f1c6b1301ac04510d0881b9e9feb20ae34094d3615a8a7c3db0dcc0",
|
||||
"sha256:81c3fa9a75d9f1afafdb916d5995633f319db09bd773cb56b8e39f1e98d90820",
|
||||
"sha256:887668e792b7edbfb1d3c9d8b5d8c859269a0f0eba4dda562adb95500f60dbba",
|
||||
"sha256:93a473b53cc6e0b3ce6bf51b1b95b7b1e7e6084be3a07e40f79b42e83503fbf2",
|
||||
"sha256:96d4dc103d1a0fa6d47c6c55a47de5f5dafd5ef0114fa10c85a1fd8e0216284b",
|
||||
"sha256:a3d3e086474ef12ef13d42e5f9b7bbf09d39cf6bd4940f982263d6954b13f6a9",
|
||||
"sha256:b02a0b9f332086657852b1f7cb380f6a42403a6d9c42a4c34a561aa4530d5234",
|
||||
"sha256:b09e10ec453de97f9a23a5aa5e30b334195e8d2ddd1ce76cc32e52ba63c8b31d",
|
||||
"sha256:b6f00ad5ebe846cc91763b1d0c6d30a8042e02b2316e27b05de04fa6ec831ec5",
|
||||
"sha256:bba80df38cfc17f490ec651c73bb37cd896bc2400cfba27d078c2135223c1206",
|
||||
"sha256:c3d911614b008e8a576b8e5303e3db29224b455d3d66d1b2848ba6ca83f9ece9",
|
||||
"sha256:ca20739e303254287138234485579b28cb0d524401f83d5129b5ff9d606cb0a8",
|
||||
"sha256:cb192176b477d49b0a327b2a5a4979552b7a58cd42037034316b8018ac3ebb59",
|
||||
"sha256:cdbbe7dff4a677fb555a54f9bc0450f2a21a93c5ba2b44e09e54fcb72d2bd13d",
|
||||
"sha256:cf6e33d92b1526190a1de904df21663c46a456758c0424e4f947ae9aa6088bf7",
|
||||
"sha256:d355502dce85ade85a2511b40b4c61a128902f246504f7de29bbeec1ae27933a",
|
||||
"sha256:d673c4990acd016229a5c1c4ee8a9e6d8f481b27ade5fc3d95938697fa443ce0",
|
||||
"sha256:dc577f4cfdda354db3ae37a572428a90ffdbe4e51eda7849bf442fb803f09c9b",
|
||||
"sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d",
|
||||
"sha256:f50e7a98b0453f39000619d845be8b06e611e56ee6e8186f7f60c3b1e2f0feae"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==8.1.0"
|
||||
},
|
||||
"pytz": {
|
||||
"hashes": [
|
||||
"sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
|
||||
"sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
|
||||
],
|
||||
"version": "==2021.1"
|
||||
},
|
||||
"rcssmin": {
|
||||
"hashes": [
|
||||
"sha256:ca87b695d3d7864157773a61263e5abb96006e9ff0e021eff90cbe0e1ba18270"
|
||||
],
|
||||
"version": "==1.0.6"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
|
||||
"sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.25.1"
|
||||
},
|
||||
"rjsmin": {
|
||||
"hashes": [
|
||||
"sha256:0ab825839125eaca57cc59581d72e596e58a7a56fbc0839996b7528f0343a0a8",
|
||||
"sha256:211c2fe8298951663bbc02acdffbf714f6793df54bfc50e1c6c9e71b3f2559a3",
|
||||
"sha256:466fe70cc5647c7c51b3260c7e2e323a98b2b173564247f9c89e977720a0645f",
|
||||
"sha256:585e75a84d9199b68056fd4a083d9a61e2a92dfd10ff6d4ce5bdb04bc3bdbfaf",
|
||||
"sha256:6044ca86e917cd5bb2f95e6679a4192cef812122f28ee08c677513de019629b3",
|
||||
"sha256:714329db774a90947e0e2086cdddb80d5e8c4ac1c70c9f92436378dedb8ae345",
|
||||
"sha256:799890bd07a048892d8d3deb9042dbc20b7f5d0eb7da91e9483c561033b23ce2",
|
||||
"sha256:975b69754d6a76be47c0bead12367a1ca9220d08e5393f80bab0230d4625d1f4",
|
||||
"sha256:b15dc75c71f65d9493a8c7fa233fdcec823e3f1b88ad84a843ffef49b338ac32",
|
||||
"sha256:dd0f4819df4243ffe4c964995794c79ca43943b5b756de84be92b445a652fb86",
|
||||
"sha256:e3908b21ebb584ce74a6ac233bdb5f29485752c9d3be5e50c5484ed74169232c",
|
||||
"sha256:e487a7783ac4339e79ec610b98228eb9ac72178973e3dee16eba0e3feef25924",
|
||||
"sha256:ecd29f1b3e66a4c0753105baec262b331bcbceefc22fbe6f7e8bcd2067bcb4d7"
|
||||
],
|
||||
"version": "==1.1.0"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
|
||||
"sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
|
||||
],
|
||||
"version": "==1.15.0"
|
||||
},
|
||||
"sqlparse": {
|
||||
"hashes": [
|
||||
"sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0",
|
||||
"sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8"
|
||||
],
|
||||
"version": "==0.4.1"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80",
|
||||
"sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73"
|
||||
],
|
||||
"version": "==1.26.3"
|
||||
},
|
||||
"whitenoise": {
|
||||
"hashes": [
|
||||
"sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7",
|
||||
"sha256:05d00198c777028d72d8b0bbd234db605ef6d60e9410125124002518a48e515d"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==5.2.0"
|
||||
},
|
||||
"youtube-dl": {
|
||||
"hashes": [
|
||||
"sha256:b390cddbd4d605bd887d0d4063988cef0fa13f916d2e1e3564badbb22504d754",
|
||||
"sha256:e7d48cd42f3081e1e0064e69f31f2856508ef31c0fc80eeebd8e70c6a031a24d"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2021.2.10"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
}
|
||||
114
README.md
114
README.md
|
|
@ -9,10 +9,10 @@ downloaded.
|
|||
|
||||
If you want to watch YouTube videos in particular quality or settings from your local
|
||||
media server, then TubeSync is for you. Internally, TubeSync is a web interface wrapper
|
||||
on `yt-dlp` and `ffmpeg` with a task scheduler.
|
||||
on `youtube-dl` and `ffmpeg` with a task scheduler.
|
||||
|
||||
There are several other web interfaces to YouTube and `yt-dlp` all with varying
|
||||
features and implementations. TubeSync's largest difference is full PVR experience of
|
||||
There are several other web interfaces to YouTube and `youtube-dl` all with varying
|
||||
features and implemenations. TubeSync's largest difference is full PVR experience of
|
||||
updating media servers and better selection of media formats. Additionally, to be as
|
||||
hands-free as possible, TubeSync has gradual retrying of failures with back-off timers
|
||||
so media which fails to download will be retried for an extended period making it,
|
||||
|
|
@ -22,9 +22,12 @@ hopefully, quite reliable.
|
|||
# Latest container image
|
||||
|
||||
```yaml
|
||||
ghcr.io/meeb/tubesync:latest
|
||||
ghcr.io/meeb/tubesync:v0.9
|
||||
```
|
||||
|
||||
**NOTE: the `:latest` tag does exist, but will contain in-development commits and may
|
||||
be broken. Use at your own risk.**
|
||||
|
||||
# Screenshots
|
||||
|
||||
### Dashboard
|
||||
|
|
@ -69,12 +72,11 @@ currently just Plex, to complete the PVR experience.
|
|||
# Installation
|
||||
|
||||
TubeSync is designed to be run in a container, such as via Docker or Podman. It also
|
||||
works in a Docker Compose stack. `amd64` (most desktop PCs and servers) and `arm64`
|
||||
(modern ARM computers, such as the Rasperry Pi 3 or later) are supported.
|
||||
works in a Docker Compose stack. Only `amd64` is initially supported.
|
||||
|
||||
Example (with Docker on *nix):
|
||||
|
||||
First find the user ID and group ID you want to run TubeSync as, if you're not
|
||||
First find your the user ID and group ID you want to run TubeSync as, if you're not
|
||||
sure what this is it's probably your current user ID and group ID:
|
||||
|
||||
```bash
|
||||
|
|
@ -99,8 +101,8 @@ $ mkdir /some/directory/tubesync-downloads
|
|||
Finally, download and run the container:
|
||||
|
||||
```bash
|
||||
# Pull image
|
||||
$ docker pull ghcr.io/meeb/tubesync:latest
|
||||
# Pull a versioned image
|
||||
$ docker pull ghcr.io/meeb/tubesync:v0.9
|
||||
# Start the container using your user ID and group ID
|
||||
$ docker run \
|
||||
-d \
|
||||
|
|
@ -111,21 +113,19 @@ $ docker run \
|
|||
-v /some/directory/tubesync-config:/config \
|
||||
-v /some/directory/tubesync-downloads:/downloads \
|
||||
-p 4848:4848 \
|
||||
ghcr.io/meeb/tubesync:latest
|
||||
ghcr.io/meeb/tubesync:v0.9
|
||||
```
|
||||
|
||||
Once running, open `http://localhost:4848` in your browser and you should see the
|
||||
TubeSync dashboard. If you do, you can proceed to adding some sources (YouTube channels
|
||||
and playlists). If not, check `docker logs tubesync` to see what errors might be
|
||||
occurring, typical ones are file permission issues.
|
||||
occuring, typical ones are file permission issues.
|
||||
|
||||
Alternatively, for Docker Compose, you can use something like:
|
||||
|
||||
```yml
|
||||
version: '3.7'
|
||||
services:
|
||||
```yaml
|
||||
tubesync:
|
||||
image: ghcr.io/meeb/tubesync:latest
|
||||
image: ghcr.io/meeb/tubesync:v0.9
|
||||
container_name: tubesync
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
|
|
@ -139,41 +139,6 @@ services:
|
|||
- PGID=1000
|
||||
```
|
||||
|
||||
|
||||
## Optional authentication
|
||||
|
||||
Available in `v1.0` (or `:latest`)and later. If you want to enable a basic username and
|
||||
password to be required to access the TubeSync dashboard you can set them with the
|
||||
following environment variables:
|
||||
|
||||
```bash
|
||||
HTTP_USER
|
||||
HTTP_PASS
|
||||
```
|
||||
|
||||
For example, in the `docker run ...` line add in:
|
||||
|
||||
```bash
|
||||
...
|
||||
-e HTTP_USER=some-username \
|
||||
-e HTTP_PASS=some-secure-password \
|
||||
...
|
||||
```
|
||||
|
||||
Or in your Docker Compose file you would add in:
|
||||
|
||||
```yaml
|
||||
...
|
||||
environment:
|
||||
- HTTP_USER=some-username
|
||||
- HTTP_PASS=some-secure-password
|
||||
...
|
||||
```
|
||||
|
||||
When BOTH `HTTP_USER` and `HTTP_PASS` are set then basic HTTP authentication will be
|
||||
enabled.
|
||||
|
||||
|
||||
# Updating
|
||||
|
||||
To update, you can just pull a new version of the container image as they are released.
|
||||
|
|
@ -236,12 +201,9 @@ $ docker logs --follow tubesync
|
|||
Once you're happy using TubeSync there are some advanced usage guides for more complex
|
||||
and less common features:
|
||||
|
||||
* [Import existing media into TubeSync](https://github.com/meeb/tubesync/blob/main/docs/import-existing-media.md)
|
||||
* [Sync or create missing metadata files](https://github.com/meeb/tubesync/blob/main/docs/create-missing-metadata.md)
|
||||
* [Reset tasks from the command line](https://github.com/meeb/tubesync/blob/main/docs/reset-tasks.md)
|
||||
* [Using PostgreSQL, MySQL or MariaDB as database backends](https://github.com/meeb/tubesync/blob/main/docs/other-database-backends.md)
|
||||
* [Using cookies](https://github.com/meeb/tubesync/blob/main/docs/using-cookies.md)
|
||||
* [Reset metadata](https://github.com/meeb/tubesync/blob/main/docs/reset-metadata.md)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# Warnings
|
||||
|
|
@ -283,7 +245,7 @@ automatically.
|
|||
### Does TubeSync support any other video platforms?
|
||||
|
||||
At the moment, no. This is a pre-release. The library TubeSync uses that does most
|
||||
of the downloading work, `yt-dlp`, supports many hundreds of video sources so it's
|
||||
of the downloading work, `youtube-dl`, supports many hundreds of video sources so it's
|
||||
likely more will be added to TubeSync if there is demand for it.
|
||||
|
||||
### Is there a progress bar?
|
||||
|
|
@ -295,27 +257,27 @@ your install is doing check the container logs.
|
|||
|
||||
### Are there alerts when a download is complete?
|
||||
|
||||
No, this feature is best served by existing services such as the excellent
|
||||
No, this feature is best served by existing services such as the execelent
|
||||
[Tautulli](https://tautulli.com/) which can monitor your Plex server and send alerts
|
||||
that way.
|
||||
|
||||
### There are errors in my "tasks" tab!
|
||||
### There's errors in my "tasks" tab!
|
||||
|
||||
You only really need to worry about these if there is a permanent failure. Some errors
|
||||
are temporary and will be retried for you automatically, such as a download got
|
||||
interrupted and will be tried again later. Sources with permanent errors (such as no
|
||||
are temproary and will be retried for you automatically, such as a download got
|
||||
interrupted and will be tried again later. Sources with permanet errors (such as no
|
||||
media available because you got a channel name wrong) will be shown as errors on the
|
||||
"sources" tab.
|
||||
|
||||
### What is TubeSync written in?
|
||||
|
||||
Python3 using Django, embedding yt-dlp. It's pretty much glue between other much
|
||||
Python3 using Django, embedding youtube-dl. It's pretty much glue between other much
|
||||
larger libraries.
|
||||
|
||||
Notable libraries and software used:
|
||||
|
||||
* [Django](https://www.djangoproject.com/)
|
||||
* [yt-dlp](https://github.com/yt-dlp/yt-dlp)
|
||||
* [youtube-dl](https://yt-dl.org/)
|
||||
* [ffmpeg](https://ffmpeg.org/)
|
||||
* [Django Background Tasks](https://github.com/arteria/django-background-tasks/)
|
||||
* [django-sass](https://github.com/coderedcorp/django-sass/)
|
||||
|
|
@ -325,7 +287,7 @@ See the [Pipefile](https://github.com/meeb/tubesync/blob/main/Pipfile) for a ful
|
|||
|
||||
### Can I get access to the full Django admin?
|
||||
|
||||
Yes, although pretty much all operations are available through the front-end interface
|
||||
Yes, although pretty much all operations are available through the front end interface
|
||||
and you can probably break things by playing in the admin. If you still want to access
|
||||
it you can run:
|
||||
|
||||
|
|
@ -338,9 +300,7 @@ can log in at http://localhost:4848/admin
|
|||
|
||||
### Are there user accounts or multi-user support?
|
||||
|
||||
There is support for basic HTTP authentication by setting the `HTTP_USER` and
|
||||
`HTTP_PASS` environment variables. There is not support for multi-user or user
|
||||
management.
|
||||
No not at the moment. This could be added later if there is demand for it.
|
||||
|
||||
### Does TubeSync support HTTPS?
|
||||
|
||||
|
|
@ -351,10 +311,6 @@ etc.). Configuration of this is beyond the scope of this README.
|
|||
|
||||
Just `amd64` for the moment. Others may be made available if there is demand.
|
||||
|
||||
### The pipenv install fails with "Locking failed"!
|
||||
|
||||
Make sure that you have `mysql_config` or `mariadb_config` available, as required by the python module `mysqlclient`. On Debian-based systems this is usually found in the package `libmysqlclient-dev`
|
||||
|
||||
|
||||
# Advanced configuration
|
||||
|
||||
|
|
@ -363,25 +319,21 @@ There are a number of other environment variables you can set. These are, mostly
|
|||
useful if you are manually installing TubeSync in some other environment. These are:
|
||||
|
||||
| Name | What | Example |
|
||||
| --------------------------- | ------------------------------------------------------------ | ------------------------------------ |
|
||||
| ------------------------ | ------------------------------------------------------------ | ---------------------------------- |
|
||||
| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l |
|
||||
| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ |
|
||||
| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME | /somepath |
|
||||
| TUBESYNC_DEBUG | Enable debugging | True |
|
||||
| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 |
|
||||
| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com |
|
||||
| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True
|
||||
| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS | tubesync.example.com,otherhost.com |
|
||||
| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 |
|
||||
| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 |
|
||||
| LISTEN_PORT | Port number for gunicorn to listen on | 8080 |
|
||||
| HTTP_USER | Sets the username for HTTP basic authentication | some-username |
|
||||
| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password |
|
||||
| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database |
|
||||
|
||||
|
||||
# Manual, non-containerised, installation
|
||||
|
||||
As a relatively normal Django app you can run TubeSync without the container. Beyond
|
||||
following this rough guide, you are on your own and should be knowledgeable about
|
||||
following this rough guide you are on your own and should be knowledgeable about
|
||||
installing and running WSGI-based Python web applications before attempting this.
|
||||
|
||||
1. Clone or download this repo
|
||||
|
|
@ -392,7 +344,7 @@ installing and running WSGI-based Python web applications before attempting this
|
|||
`tubesync/tubesync/local_settings.py` and edit it as appropriate
|
||||
5. Run migrations with `./manage.py migrate`
|
||||
6. Collect static files with `./manage.py collectstatic`
|
||||
6. Set up your prefered WSGI server, such as `gunicorn` pointing it to the application
|
||||
6. Set up your prefered WSGI server, such as `gunicorn` poiting it to the application
|
||||
in `tubesync/tubesync/wsgi.py`
|
||||
7. Set up your proxy server such as `nginx` and forward it to the WSGI server
|
||||
8. Check the web interface is working
|
||||
|
|
@ -404,7 +356,7 @@ installing and running WSGI-based Python web applications before attempting this
|
|||
|
||||
# Tests
|
||||
|
||||
There is a moderately comprehensive test suite focusing on the custom media format
|
||||
There is a moderately comprehensive test suite focussing on the custom media format
|
||||
matching logic and that the front-end interface works. You can run it via Django:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
#!/usr/bin/with-contenv bash
|
||||
|
||||
# Change runtime user UID and GID
|
||||
PUID=${PUID:-911}
|
||||
PGID=${PGID:-911}
|
||||
groupmod -o -g "$PGID" app
|
||||
usermod -o -u "$PUID" app
|
||||
|
||||
# Reset permissions
|
||||
chown -R app:app /run/app && \
|
||||
chmod -R 0700 /run/app && \
|
||||
chown -R app:app /config && \
|
||||
chmod -R 0755 /config && \
|
||||
chown -R app:app /downloads && \
|
||||
chmod -R 0755 /downloads && \
|
||||
chown -R root:app /app && \
|
||||
chmod -R 0750 /app && \
|
||||
chown -R app:app /app/common/static && \
|
||||
chmod -R 0750 /app/common/static && \
|
||||
chown -R app:app /app/static && \
|
||||
chmod -R 0750 /app/static && \
|
||||
find /app -type f ! -iname healthcheck.py -exec chmod 640 {} \; && \
|
||||
chmod 0755 /app/healthcheck.py
|
||||
|
||||
# Run migrations
|
||||
exec s6-setuidgid app \
|
||||
/usr/bin/python3 /app/manage.py migrate
|
||||
|
|
@ -79,11 +79,6 @@ http {
|
|||
proxy_connect_timeout 10;
|
||||
}
|
||||
|
||||
# File dwnload and streaming
|
||||
location /media-data/ {
|
||||
internal;
|
||||
alias /downloads/;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,46 +0,0 @@
|
|||
bind 127.0.0.1
|
||||
protected-mode yes
|
||||
port 6379
|
||||
tcp-backlog 511
|
||||
timeout 0
|
||||
tcp-keepalive 300
|
||||
daemonize no
|
||||
supervised no
|
||||
loglevel notice
|
||||
logfile ""
|
||||
databases 1
|
||||
always-show-logo no
|
||||
save ""
|
||||
dir /var/lib/redis
|
||||
maxmemory 64mb
|
||||
maxmemory-policy noeviction
|
||||
lazyfree-lazy-eviction no
|
||||
lazyfree-lazy-expire no
|
||||
lazyfree-lazy-server-del no
|
||||
replica-lazy-flush no
|
||||
lazyfree-lazy-user-del no
|
||||
oom-score-adj no
|
||||
oom-score-adj-values 0 200 800
|
||||
appendonly no
|
||||
appendfsync no
|
||||
lua-time-limit 5000
|
||||
slowlog-log-slower-than 10000
|
||||
slowlog-max-len 128
|
||||
latency-monitor-threshold 0
|
||||
notify-keyspace-events ""
|
||||
hash-max-ziplist-entries 512
|
||||
hash-max-ziplist-value 64
|
||||
list-max-ziplist-size -2
|
||||
list-compress-depth 0
|
||||
set-max-intset-entries 512
|
||||
zset-max-ziplist-entries 128
|
||||
zset-max-ziplist-value 64
|
||||
hll-sparse-max-bytes 3000
|
||||
stream-node-max-bytes 4096
|
||||
stream-node-max-entries 100
|
||||
activerehashing yes
|
||||
client-output-buffer-limit normal 0 0 0
|
||||
client-output-buffer-limit replica 256mb 64mb 60
|
||||
client-output-buffer-limit pubsub 32mb 8mb 60
|
||||
hz 10
|
||||
dynamic-hz yes
|
||||
|
|
@ -1 +0,0 @@
|
|||
gunicorn
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
#!/usr/bin/with-contenv bash
|
||||
|
||||
UMASK_SET=${UMASK_SET:-022}
|
||||
umask "$UMASK_SET"
|
||||
|
||||
cd /app || exit
|
||||
|
||||
PIDFILE=/run/app/celery-beat.pid
|
||||
SCHEDULE=/tmp/tubesync-celerybeat-schedule
|
||||
|
||||
if [ -f "${PIDFILE}" ]
|
||||
then
|
||||
PID=$(cat $PIDFILE)
|
||||
echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
|
||||
if kill -0 $PID
|
||||
then
|
||||
echo "Killing old gunicorn process with PID: ${PID}"
|
||||
kill -9 $PID
|
||||
fi
|
||||
echo "Removing stale PID file: ${PIDFILE}"
|
||||
rm ${PIDFILE}
|
||||
fi
|
||||
|
||||
#exec s6-setuidgid app \
|
||||
# /usr/local/bin/celery --workdir /app -A tubesync beat --pidfile ${PIDFILE} -s ${SCHEDULE}
|
||||
|
|
@ -1 +0,0 @@
|
|||
longrun
|
||||
|
|
@ -1 +0,0 @@
|
|||
gunicorn
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
#!/usr/bin/with-contenv bash
|
||||
|
||||
UMASK_SET=${UMASK_SET:-022}
|
||||
umask "$UMASK_SET"
|
||||
|
||||
cd /app || exit
|
||||
|
||||
PIDFILE=/run/app/celery-worker.pid
|
||||
|
||||
if [ -f "${PIDFILE}" ]
|
||||
then
|
||||
PID=$(cat $PIDFILE)
|
||||
echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
|
||||
if kill -0 $PID
|
||||
then
|
||||
echo "Killing old gunicorn process with PID: ${PID}"
|
||||
kill -9 $PID
|
||||
fi
|
||||
echo "Removing stale PID file: ${PIDFILE}"
|
||||
rm ${PIDFILE}
|
||||
fi
|
||||
|
||||
#exec s6-setuidgid app \
|
||||
# /usr/local/bin/celery --workdir /app -A tubesync worker --pidfile ${PIDFILE} -l INFO
|
||||
|
|
@ -1 +0,0 @@
|
|||
longrun
|
||||
|
|
@ -1 +0,0 @@
|
|||
tubesync-init
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
#!/command/with-contenv bash
|
||||
|
||||
UMASK_SET=${UMASK_SET:-022}
|
||||
umask "$UMASK_SET"
|
||||
|
||||
cd /app || exit
|
||||
|
||||
PIDFILE=/run/app/gunicorn.pid
|
||||
|
||||
if [ -f "${PIDFILE}" ]
|
||||
then
|
||||
PID=$(cat $PIDFILE)
|
||||
echo "Unexpected PID file exists at ${PIDFILE} with PID: ${PID}"
|
||||
if kill -0 $PID
|
||||
then
|
||||
echo "Killing old gunicorn process with PID: ${PID}"
|
||||
kill -9 $PID
|
||||
fi
|
||||
echo "Removing stale PID file: ${PIDFILE}"
|
||||
rm ${PIDFILE}
|
||||
fi
|
||||
|
||||
exec s6-setuidgid app \
|
||||
/usr/local/bin/gunicorn -c /app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application
|
||||
|
|
@ -1 +0,0 @@
|
|||
longrun
|
||||
|
|
@ -1 +0,0 @@
|
|||
gunicorn
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
#!/command/with-contenv bash
|
||||
|
||||
cd /
|
||||
|
||||
/usr/sbin/nginx
|
||||
|
|
@ -1 +0,0 @@
|
|||
longrun
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
#!/command/with-contenv bash
|
||||
|
||||
exec s6-setuidgid redis \
|
||||
/usr/bin/redis-server /etc/redis/redis.conf
|
||||
|
|
@ -1 +0,0 @@
|
|||
longrun
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
#!/command/with-contenv bash
|
||||
|
||||
# Change runtime user UID and GID
|
||||
PUID="${PUID:-911}"
|
||||
PUID="${PUID:-911}"
|
||||
groupmod -o -g "$PGID" app
|
||||
usermod -o -u "$PUID" app
|
||||
|
||||
# Reset permissions
|
||||
chown -R app:app /run/app
|
||||
chmod -R 0700 /run/app
|
||||
chown -R app:app /config
|
||||
chmod -R 0755 /config
|
||||
chown -R root:app /app
|
||||
chmod -R 0750 /app
|
||||
chown -R app:app /app/common/static
|
||||
chmod -R 0750 /app/common/static
|
||||
chown -R app:app /app/static
|
||||
chmod -R 0750 /app/static
|
||||
find /app -type f ! -iname healthcheck.py -exec chmod 640 {} \;
|
||||
chmod 0755 /app/healthcheck.py
|
||||
|
||||
# Optionally reset the download dir permissions
|
||||
TUBESYNC_RESET_DOWNLOAD_DIR="${TUBESYNC_RESET_DOWNLOAD_DIR:-True}"
|
||||
if [ "$TUBESYNC_RESET_DOWNLOAD_DIR" == "True" ]
|
||||
then
|
||||
echo "TUBESYNC_RESET_DOWNLOAD_DIR=True, Resetting /downloads directory permissions"
|
||||
chown -R app:app /downloads
|
||||
chmod -R 0755 /downloads
|
||||
fi
|
||||
|
||||
# Run migrations
|
||||
exec s6-setuidgid app \
|
||||
/usr/bin/python3 /app/manage.py migrate
|
||||
|
|
@ -1 +0,0 @@
|
|||
oneshot
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
#!/command/execlineb -P
|
||||
|
||||
/etc/s6-overlay/s6-rc.d/tubesync-init/run
|
||||
|
|
@ -1 +0,0 @@
|
|||
gunicorn
|
||||
|
|
@ -1 +0,0 @@
|
|||
longrun
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
#!/usr/bin/with-contenv bash
|
||||
|
||||
UMASK_SET=${UMASK_SET:-022}
|
||||
umask "$UMASK_SET"
|
||||
|
||||
cd /app || exit
|
||||
|
||||
exec s6-setuidgid app \
|
||||
/usr/local/bin/gunicorn -c /app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
#!/usr/bin/with-contenv bash
|
||||
|
||||
cd /
|
||||
|
||||
/usr/sbin/nginx
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
#!/command/with-contenv bash
|
||||
#!/usr/bin/with-contenv bash
|
||||
|
||||
exec s6-setuidgid app \
|
||||
/usr/bin/python3 /app/manage.py process_tasks
|
||||
|
|
@ -1,132 +0,0 @@
|
|||
# TubeSync
|
||||
|
||||
## Advanced usage guide - using other database backends
|
||||
|
||||
This is a new feature in v1.0 of TubeSync and later. It allows you to use a custom
|
||||
existing external database server instead of the default SQLite database. You may want
|
||||
to use this if you encounter performance issues with adding very large or a large
|
||||
number of channels and database write contention (as shown by errors in the log)
|
||||
become an issue.
|
||||
|
||||
## Requirements
|
||||
|
||||
TubeSync supports SQLite (the automatic default) as well as PostgreSQL, MySQL and
|
||||
MariaDB. For MariaDB just follow the MySQL instructions as the driver is the same.
|
||||
|
||||
You should start with a blank install of TubeSync. Migrating to a new database will
|
||||
reset your database. If you are comfortable with Django you can export and re-import
|
||||
existing database data with:
|
||||
|
||||
```bash
|
||||
$ docker exec -i tubesync python3 /app/manage.py dumpdata > some-file.json
|
||||
```
|
||||
|
||||
Then change you database backend over, then use
|
||||
|
||||
```bash
|
||||
$ cat some-file.json | docker exec -i tubesync python3 /app/manage.py loaddata - --format=json
|
||||
```
|
||||
|
||||
As detailed in the Django documentation:
|
||||
|
||||
https://docs.djangoproject.com/en/3.1/ref/django-admin/#dumpdata
|
||||
|
||||
and:
|
||||
|
||||
https://docs.djangoproject.com/en/3.1/ref/django-admin/#loaddata
|
||||
|
||||
Further instructions are beyond the scope of TubeSync documenation and you should refer
|
||||
to Django documentation for more details.
|
||||
|
||||
If you are not comfortable with the above, then skip the `dumpdata` steps, however
|
||||
remember you will start again with a completely new database.
|
||||
|
||||
## Steps
|
||||
|
||||
### 1. Create a database in your external database server
|
||||
|
||||
You need to create a database and a user with permissions to access the database in
|
||||
your chosen external database server. Steps vary between PostgreSQL, MySQL and MariaDB
|
||||
so this is up to you to work out.
|
||||
|
||||
### 2. Set the database connection string environment variable
|
||||
|
||||
You need to provide the database connection details to TubeSync via an environment
|
||||
variable. The environment variable name is `DATABASE_CONNECTION` and the format is the
|
||||
standard URL-style string. Examples are:
|
||||
|
||||
`postgresql://tubesync:password@localhost:5432/tubesync`
|
||||
|
||||
and
|
||||
|
||||
`mysql://tubesync:password@localhost:3306/tubesync`
|
||||
|
||||
*Important note:* For MySQL databases make SURE you create the tubesync database with
|
||||
`utf8mb4` encoding, like:
|
||||
|
||||
`CREATE DATABASE tubesync CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;`
|
||||
|
||||
Without `utf8mb4` encoding things like emojis in video titles (or any extended UTF8
|
||||
characters) can cause issues.
|
||||
|
||||
### 3. Start TubeSync and check the logs
|
||||
|
||||
Once you start TubeSync with the new database connection you should see the folling log
|
||||
entry in the container or stdout logs:
|
||||
|
||||
`2021-04-04 22:42:17,912 [tubesync/INFO] Using database connection: django.db.backends.postgresql://tubesync:[hidden]@localhost:5432/tubesync`
|
||||
|
||||
If you see a line similar to the above and the web interface loads, congratulations,
|
||||
you are now using an external database server for your TubeSync data!
|
||||
|
||||
## Database Compression (For MariaDB)
|
||||
With a lot of media files the `sync_media` table grows in size quickly.
|
||||
You can save space using column compression using the following steps while using MariaDB:
|
||||
|
||||
1. Stop tubesync
|
||||
2. Execute `ALTER TABLE sync_media MODIFY metadata LONGTEXT COMPRESSED;` on database tubesync
|
||||
3. Start tunesync and confirm the connection still works.
|
||||
|
||||
## Docker Compose
|
||||
|
||||
If you're using Docker Compose and simply want to connect to another container with
|
||||
the DB for the performance benefits, a configuration like this would be enough:
|
||||
|
||||
```
|
||||
tubesync-db:
|
||||
image: postgres:15.2
|
||||
container_name: tubesync-db
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /<path/to>/init.sql:/docker-entrypoint-initdb.d/init.sql
|
||||
- /<path/to>/tubesync-db:/var/lib/postgresql/data
|
||||
environment:
|
||||
- POSTGRES_USER=postgres
|
||||
- POSTGRES_PASSWORD=testpassword
|
||||
|
||||
tubesync:
|
||||
image: ghcr.io/meeb/tubesync:latest
|
||||
container_name: tubesync
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- 4848:4848
|
||||
volumes:
|
||||
- /<path/to>/tubesync/config:/config
|
||||
- /<path/to>/YouTube:/downloads
|
||||
environment:
|
||||
- DATABASE_CONNECTION=postgresql://postgres:testpassword@tubesync-db:5432/tubesync
|
||||
depends_on:
|
||||
- tubesync-db
|
||||
```
|
||||
|
||||
Note that an `init.sql` file is needed to initialize the `tubesync`
|
||||
database before it can be written to. This file should contain:
|
||||
|
||||
```
|
||||
CREATE DATABASE tubesync;
|
||||
```
|
||||
|
||||
|
||||
Then it must be mapped to `/docker-entrypoint-initdb.d/init.sql` for it
|
||||
to be executed on first startup of the container. See the `tubesync-db`
|
||||
volume mapping above for how to do this.
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
# TubeSync
|
||||
|
||||
## Advanced usage guide - reset media metadata from the command line
|
||||
|
||||
This command allows you to reset all media item metadata. You might want to use
|
||||
this if you have a lot of media items with invalid metadata and you want to
|
||||
wipe it which triggers the metadata to be redownloaded.
|
||||
|
||||
|
||||
## Requirements
|
||||
|
||||
You have added some sources and media
|
||||
|
||||
## Steps
|
||||
|
||||
### 1. Run the reset tasks command
|
||||
|
||||
Execute the following Django command:
|
||||
|
||||
`./manage.py reset-metadata`
|
||||
|
||||
When deploying TubeSync inside a container, you can execute this with:
|
||||
|
||||
`docker exec -ti tubesync python3 /app/manage.py reset-metadata`
|
||||
|
||||
This command will log what its doing to the terminal when you run it.
|
||||
|
||||
When this is run, new tasks will be immediately created so all your media
|
||||
items will start downloading updated metadata straight away, any missing information
|
||||
such as thumbnails will be redownloaded, etc.
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
# TubeSync
|
||||
|
||||
## Advanced usage guide - reset tasks from the command line
|
||||
|
||||
This is a new feature in v1.0 of TubeSync and later. It allows you to reset all
|
||||
scheduled tasks from the command line as well as the "reset tasks" button in the
|
||||
"tasks" tab of the dashboard.
|
||||
|
||||
This is useful for TubeSync installations where you may have a lot of media and
|
||||
sources added and the "reset tasks" button may take too long to the extent where
|
||||
the page times out (with a 502 error or similar issue).
|
||||
|
||||
## Requirements
|
||||
|
||||
You have added some sources and media
|
||||
|
||||
## Steps
|
||||
|
||||
### 1. Run the reset tasks command
|
||||
|
||||
Execute the following Django command:
|
||||
|
||||
`./manage.py reset-tasks`
|
||||
|
||||
When deploying TubeSync inside a container, you can execute this with:
|
||||
|
||||
`docker exec -ti tubesync python3 /app/manage.py reset-tasks`
|
||||
|
||||
This command will log what its doing to the terminal when you run it.
|
||||
|
||||
When this is run, new tasks will be immediately created so all your sources will be
|
||||
indexed again straight away, any missing information such as thumbnails will be
|
||||
redownloaded, etc.
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
# TubeSync
|
||||
|
||||
## Advanced usage guide - using exported cookies
|
||||
|
||||
This is a new feature in v0.10 of TubeSync and later. It allows you to use the cookies
|
||||
file exported from your browser in "Netscape" format with TubeSync to authenticate
|
||||
to YouTube. This can bypass some throttling, age restrictions and other blocks at
|
||||
YouTube.
|
||||
|
||||
**IMPORTANT NOTE**: Using cookies exported from your browser that is authenticated
|
||||
to YouTube identifes your Google account as using TubeSync. This may result in
|
||||
potential account impacts and is entirely at your own risk. Do not use this
|
||||
feature unless you really know what you're doing.
|
||||
|
||||
## Requirements
|
||||
|
||||
Have a browser that supports exporting your cookies and be logged into YouTube.
|
||||
|
||||
## Steps
|
||||
|
||||
### 1. Export your cookies
|
||||
|
||||
You need to export cookies for youtube.com from your browser, you can either do
|
||||
this manually or there are plug-ins to automate this for you. This file must be
|
||||
in the "Netscape" cookie export format.
|
||||
|
||||
Save your cookies as a `cookies.txt` file.
|
||||
|
||||
### 2. Import into TubeSync
|
||||
|
||||
Drop the `cookies.txt` file into your TubeSync `config` directory.
|
||||
|
||||
If detected correctly, you will see something like this in the worker or container
|
||||
logs:
|
||||
|
||||
```
|
||||
YYYY-MM-DD HH:MM:SS,mmm [tubesync/INFO] [youtube-dl] using cookies.txt from: /config/cookies.txt
|
||||
```
|
||||
|
||||
If you see that line it's working correctly.
|
||||
|
||||
If you see errors in your logs like this:
|
||||
|
||||
```
|
||||
http.cookiejar.LoadError: '/config/cookies.txt' does not look like a Netscape format cookies file
|
||||
```
|
||||
|
||||
Then your `cookies.txt` file was not generated or created correctly as it's not
|
||||
in the required "Netscape" format. You can fix this by exporting your `cookies.txt`
|
||||
in the correct "Netscape" format.
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
from django.conf import settings
|
||||
from .third_party_versions import yt_dlp_version, ffmpeg_version
|
||||
from .third_party_versions import youtube_dl_version, ffmpeg_version
|
||||
|
||||
|
||||
def app_details(request):
|
||||
return {
|
||||
'app_version': str(settings.VERSION),
|
||||
'yt_dlp_version': yt_dlp_version,
|
||||
'youtube_dl_version': youtube_dl_version,
|
||||
'ffmpeg_version': ffmpeg_version,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,10 +20,3 @@ class DownloadFailedException(Exception):
|
|||
exist.
|
||||
'''
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseConnectionError(Exception):
|
||||
'''
|
||||
Raised when parsing or initially connecting to a database.
|
||||
'''
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,14 +1,10 @@
|
|||
import logging
|
||||
from django.conf import settings
|
||||
|
||||
|
||||
logging_level = logging.DEBUG if settings.DEBUG else logging.INFO
|
||||
|
||||
|
||||
log = logging.getLogger('tubesync')
|
||||
log.setLevel(logging_level)
|
||||
log.setLevel(logging.DEBUG)
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging_level)
|
||||
ch.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter('%(asctime)s [%(name)s/%(levelname)s] %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
log.addHandler(ch)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
from django.conf import settings
|
||||
from django.forms import BaseForm
|
||||
from basicauth.middleware import BasicAuthMiddleware as BaseBasicAuthMiddleware
|
||||
|
||||
|
||||
class MaterializeDefaultFieldsMiddleware:
|
||||
|
|
@ -21,12 +19,3 @@ class MaterializeDefaultFieldsMiddleware:
|
|||
for _, field in v.fields.items():
|
||||
field.widget.attrs.update({'class':'browser-default'})
|
||||
return response
|
||||
|
||||
|
||||
class BasicAuthMiddleware(BaseBasicAuthMiddleware):
|
||||
|
||||
def process_request(self, request):
|
||||
bypass_uris = getattr(settings, 'BASICAUTH_ALWAYS_ALLOW_URIS', [])
|
||||
if request.path in bypass_uris:
|
||||
return None
|
||||
return super().process_request(request)
|
||||
|
|
|
|||
|
|
@ -1,20 +1,20 @@
|
|||
@font-face {
|
||||
font-family: 'roboto';
|
||||
src: url('../fonts/roboto/roboto-light.woff') format('woff');
|
||||
font-weight: lighter;
|
||||
font-style: normal;
|
||||
}
|
||||
|
||||
@font-face {
|
||||
font-family: 'roboto';
|
||||
src: url('../fonts/roboto/roboto-regular.woff') format('woff');
|
||||
font-family: 'roboto-light';
|
||||
src: url('/static/fonts/roboto/roboto-light.woff') format('woff');
|
||||
font-weight: normal;
|
||||
font-style: normal;
|
||||
}
|
||||
|
||||
@font-face {
|
||||
font-family: 'roboto';
|
||||
src: url('../fonts/roboto/roboto-bold.woff') format('woff');
|
||||
font-family: 'roboto-regular';
|
||||
src: url('/static/fonts/roboto/roboto-regular.woff') format('woff');
|
||||
font-weight: normal;
|
||||
font-style: normal;
|
||||
}
|
||||
|
||||
@font-face {
|
||||
font-family: 'roboto-bold';
|
||||
src: url('/static/fonts/roboto/roboto-bold.woff') format('woff');
|
||||
font-weight: bold;
|
||||
font-style: normal;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -181,10 +181,8 @@ main {
|
|||
display: inline-block;
|
||||
font-weight: bold;
|
||||
text-decoration: none;
|
||||
padding: 5px 8px 4px 8px;
|
||||
margin: 0 3px 6px 3px;
|
||||
min-width: 40px;
|
||||
min-height: 40px;
|
||||
padding: 5px 10px 5px 10px;
|
||||
margin: 0 3px 0 3px;
|
||||
background-color: $pagination-background-colour;
|
||||
color: $pagination-text-colour;
|
||||
border: 2px $pagination-border-colour solid;
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
$font-family: 'roboto', Arial, Helvetica, sans-serif;
|
||||
$font-family: 'roboto-regular', Arial, Helvetica, sans-serif;
|
||||
$font-size: 1.05rem;
|
||||
|
|
|
|||
|
|
@ -65,7 +65,6 @@ readers do not read off random characters that represent icons */
|
|||
.#{$fa-css-prefix}-arrows-alt-h:before { content: fa-content($fa-var-arrows-alt-h); }
|
||||
.#{$fa-css-prefix}-arrows-alt-v:before { content: fa-content($fa-var-arrows-alt-v); }
|
||||
.#{$fa-css-prefix}-artstation:before { content: fa-content($fa-var-artstation); }
|
||||
.#{$fa-css-prefix}-arrow-rotate-right:before { content: fa-content($fa-var-arrow-rotate-right); }
|
||||
.#{$fa-css-prefix}-assistive-listening-systems:before { content: fa-content($fa-var-assistive-listening-systems); }
|
||||
.#{$fa-css-prefix}-asterisk:before { content: fa-content($fa-var-asterisk); }
|
||||
.#{$fa-css-prefix}-asymmetrik:before { content: fa-content($fa-var-asymmetrik); }
|
||||
|
|
|
|||
|
|
@ -80,7 +80,6 @@ $fa-var-arrow-right: \f061;
|
|||
$fa-var-arrow-up: \f062;
|
||||
$fa-var-arrows-alt: \f0b2;
|
||||
$fa-var-arrows-alt-h: \f337;
|
||||
$fa-var-arrow-rotate-right: \f01e;
|
||||
$fa-var-arrows-alt-v: \f338;
|
||||
$fa-var-artstation: \f77a;
|
||||
$fa-var-assistive-listening-systems: \f2a2;
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
// Text Label Style
|
||||
+ span:not(.lever) {
|
||||
position: relative;
|
||||
padding-left: 27px;
|
||||
padding-left: 35px;
|
||||
cursor: pointer;
|
||||
display: inline-block;
|
||||
height: 25px;
|
||||
|
|
|
|||
|
|
@ -17,16 +17,3 @@ html {
|
|||
visibility: visible;
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.flex-collection-container {
|
||||
display: flex !important;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.flex-grow {
|
||||
flex-grow: 1;
|
||||
}
|
||||
|
||||
.help-text > i {
|
||||
padding-right: 6px;
|
||||
}
|
||||
|
|
@ -57,7 +57,7 @@
|
|||
</p>
|
||||
<p>
|
||||
<a href="https://github.com/meeb/tubesync" class="nowrap" target="_blank"><i class="fab fa-github"></i> TubeSync</a> version <strong>{{ app_version }}</strong> with
|
||||
<a href="https://github.com/yt-dlp/yt-dlp" class="nowrap" target="_blank"><i class="fas fa-link"></i> yt-dlp</a> version <strong>{{ yt_dlp_version }}</strong> and
|
||||
<a href="https://yt-dl.org/" class="nowrap" target="_blank"><i class="fas fa-link"></i> youtube-dl</a> version <strong>{{ youtube_dl_version }}</strong> and
|
||||
<a href="https://ffmpeg.org/" class="nowrap" target="_blank"><i class="fas fa-link"></i> FFmpeg</a> version <strong>{{ ffmpeg_version }}</strong>.
|
||||
</p>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
<div class="col s12">
|
||||
<div class="pagination">
|
||||
{% for i in paginator.page_range %}
|
||||
<a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}{% if show_skipped %}&show_skipped=yes{% endif %}{% if only_skipped %}&only_skipped=yes{% endif %}">{{ i }}</a>
|
||||
<a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}">{{ i }}</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@ import os.path
|
|||
from django.conf import settings
|
||||
from django.test import TestCase, Client
|
||||
from .testutils import prevent_request_warnings
|
||||
from .utils import parse_database_connection_string, clean_filename
|
||||
from .errors import DatabaseConnectionError
|
||||
|
||||
|
||||
class ErrorPageTestCase(TestCase):
|
||||
|
|
@ -63,75 +61,3 @@ class CommonStaticTestCase(TestCase):
|
|||
favicon_real_path = os.path.join(os.sep.join(root_parts),
|
||||
os.sep.join(url_parts))
|
||||
self.assertTrue(os.path.exists(favicon_real_path))
|
||||
|
||||
|
||||
class UtilsTestCase(TestCase):
|
||||
|
||||
def test_parse_database_connection_string(self):
|
||||
database_dict = parse_database_connection_string(
|
||||
'postgresql://tubesync:password@localhost:5432/tubesync')
|
||||
self.assertEqual(database_dict,
|
||||
{
|
||||
'DRIVER': 'postgresql',
|
||||
'ENGINE': 'django.db.backends.postgresql',
|
||||
'USER': 'tubesync',
|
||||
'PASSWORD': 'password',
|
||||
'HOST': 'localhost',
|
||||
'PORT': 5432,
|
||||
'NAME': 'tubesync',
|
||||
'CONN_MAX_AGE': 300,
|
||||
'OPTIONS': {},
|
||||
}
|
||||
)
|
||||
database_dict = parse_database_connection_string(
|
||||
'mysql://tubesync:password@localhost:3306/tubesync')
|
||||
self.assertEqual(database_dict,
|
||||
{
|
||||
'DRIVER': 'mysql',
|
||||
'ENGINE': 'django.db.backends.mysql',
|
||||
'USER': 'tubesync',
|
||||
'PASSWORD': 'password',
|
||||
'HOST': 'localhost',
|
||||
'PORT': 3306,
|
||||
'NAME': 'tubesync',
|
||||
'CONN_MAX_AGE': 300,
|
||||
'OPTIONS': {'charset': 'utf8mb4'}
|
||||
}
|
||||
)
|
||||
# Invalid driver
|
||||
with self.assertRaises(DatabaseConnectionError):
|
||||
parse_database_connection_string(
|
||||
'test://tubesync:password@localhost:5432/tubesync')
|
||||
# No username
|
||||
with self.assertRaises(DatabaseConnectionError):
|
||||
parse_database_connection_string(
|
||||
'postgresql://password@localhost:5432/tubesync')
|
||||
# No database name
|
||||
with self.assertRaises(DatabaseConnectionError):
|
||||
parse_database_connection_string(
|
||||
'postgresql://tubesync:password@5432')
|
||||
# Invalid port
|
||||
with self.assertRaises(DatabaseConnectionError):
|
||||
parse_database_connection_string(
|
||||
'postgresql://tubesync:password@localhost:test/tubesync')
|
||||
# Invalid port
|
||||
with self.assertRaises(DatabaseConnectionError):
|
||||
parse_database_connection_string(
|
||||
'postgresql://tubesync:password@localhost:65537/tubesync')
|
||||
# Invalid username or password
|
||||
with self.assertRaises(DatabaseConnectionError):
|
||||
parse_database_connection_string(
|
||||
'postgresql://tubesync:password:test@localhost:5432/tubesync')
|
||||
# Invalid database name
|
||||
with self.assertRaises(DatabaseConnectionError):
|
||||
parse_database_connection_string(
|
||||
'postgresql://tubesync:password@localhost:5432/tubesync/test')
|
||||
|
||||
def test_clean_filename(self):
|
||||
self.assertEqual(clean_filename('a'), 'a')
|
||||
self.assertEqual(clean_filename('a\t'), 'a')
|
||||
self.assertEqual(clean_filename('a\n'), 'a')
|
||||
self.assertEqual(clean_filename('a a'), 'a a')
|
||||
self.assertEqual(clean_filename('a a'), 'a a')
|
||||
self.assertEqual(clean_filename('a\t\t\ta'), 'a a')
|
||||
self.assertEqual(clean_filename('a\t\t\ta\t\t\t'), 'a a')
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from yt_dlp import version as yt_dlp_version
|
||||
from youtube_dl import version as yt_version
|
||||
|
||||
|
||||
yt_dlp_version = str(yt_dlp_version.__version__)
|
||||
youtube_dl_version = str(yt_version.__version__)
|
||||
ffmpeg_version = '(shared install)'
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,96 +1,4 @@
|
|||
import string
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlunsplit, urlencode, urlparse
|
||||
from yt_dlp.utils import LazyList
|
||||
from .errors import DatabaseConnectionError
|
||||
|
||||
|
||||
def parse_database_connection_string(database_connection_string):
|
||||
'''
|
||||
Parses a connection string in a URL style format, such as:
|
||||
postgresql://tubesync:password@localhost:5432/tubesync
|
||||
mysql://someuser:somepassword@localhost:3306/tubesync
|
||||
into a Django-compatible settings.DATABASES dict format.
|
||||
'''
|
||||
valid_drivers = ('postgresql', 'mysql')
|
||||
default_ports = {
|
||||
'postgresql': 5432,
|
||||
'mysql': 3306,
|
||||
}
|
||||
django_backends = {
|
||||
'postgresql': 'django.db.backends.postgresql',
|
||||
'mysql': 'django.db.backends.mysql',
|
||||
}
|
||||
backend_options = {
|
||||
'postgresql': {},
|
||||
'mysql': {
|
||||
'charset': 'utf8mb4',
|
||||
}
|
||||
}
|
||||
try:
|
||||
parts = urlparse(str(database_connection_string))
|
||||
except Exception as e:
|
||||
raise DatabaseConnectionError(f'Failed to parse "{database_connection_string}" '
|
||||
f'as a database connection string: {e}') from e
|
||||
driver = parts.scheme
|
||||
user_pass_host_port = parts.netloc
|
||||
database = parts.path
|
||||
if driver not in valid_drivers:
|
||||
raise DatabaseConnectionError(f'Database connection string '
|
||||
f'"{database_connection_string}" specified an '
|
||||
f'invalid driver, must be one of {valid_drivers}')
|
||||
django_driver = django_backends.get(driver)
|
||||
host_parts = user_pass_host_port.split('@')
|
||||
if len(host_parts) != 2:
|
||||
raise DatabaseConnectionError(f'Database connection string netloc must be in '
|
||||
f'the format of user:pass@host')
|
||||
user_pass, host_port = host_parts
|
||||
user_pass_parts = user_pass.split(':')
|
||||
if len(user_pass_parts) != 2:
|
||||
raise DatabaseConnectionError(f'Database connection string netloc must be in '
|
||||
f'the format of user:pass@host')
|
||||
username, password = user_pass_parts
|
||||
host_port_parts = host_port.split(':')
|
||||
if len(host_port_parts) == 1:
|
||||
# No port number, assign a default port
|
||||
hostname = host_port_parts[0]
|
||||
port = default_ports.get(driver)
|
||||
elif len(host_port_parts) == 2:
|
||||
# Host name and port number
|
||||
hostname, port = host_port_parts
|
||||
try:
|
||||
port = int(port)
|
||||
except (ValueError, TypeError) as e:
|
||||
raise DatabaseConnectionError(f'Database connection string contained an '
|
||||
f'invalid port, ports must be integers: '
|
||||
f'{e}') from e
|
||||
if not 0 < port < 63336:
|
||||
raise DatabaseConnectionError(f'Database connection string contained an '
|
||||
f'invalid port, ports must be between 1 and '
|
||||
f'65535, got {port}')
|
||||
else:
|
||||
# Malformed
|
||||
raise DatabaseConnectionError(f'Database connection host must be a hostname or '
|
||||
f'a hostname:port combination')
|
||||
if database.startswith('/'):
|
||||
database = database[1:]
|
||||
if not database:
|
||||
raise DatabaseConnectionError(f'Database connection string path must be a '
|
||||
f'string in the format of /databasename')
|
||||
if '/' in database:
|
||||
raise DatabaseConnectionError(f'Database connection string path can only '
|
||||
f'contain a single string name, got: {database}')
|
||||
return {
|
||||
'DRIVER': driver,
|
||||
'ENGINE': django_driver,
|
||||
'NAME': database,
|
||||
'USER': username,
|
||||
'PASSWORD': password,
|
||||
'HOST': hostname,
|
||||
'PORT': port,
|
||||
'CONN_MAX_AGE': 300,
|
||||
'OPTIONS': backend_options.get(driver),
|
||||
}
|
||||
from urllib.parse import urlunsplit, urlencode
|
||||
|
||||
|
||||
def get_client_ip(request):
|
||||
|
|
@ -114,18 +22,5 @@ def clean_filename(filename):
|
|||
to_scrub = '<>\/:*?"|%'
|
||||
for char in to_scrub:
|
||||
filename = filename.replace(char, '')
|
||||
clean_filename = ''
|
||||
for c in filename:
|
||||
if c in string.whitespace:
|
||||
c = ' '
|
||||
if ord(c) > 30:
|
||||
clean_filename += c
|
||||
return clean_filename.strip()
|
||||
|
||||
|
||||
def json_serial(obj):
|
||||
if isinstance(obj, datetime):
|
||||
return obj.isoformat()
|
||||
if isinstance(obj, LazyList):
|
||||
return list(obj)
|
||||
raise TypeError(f'Type {type(obj)} is not json_serial()-able')
|
||||
filename = ''.join([c for c in filename if ord(c) > 30])
|
||||
return ' '.join(filename.split())
|
||||
|
|
|
|||
|
|
@ -1,109 +0,0 @@
|
|||
from django.forms import MultipleChoiceField, CheckboxSelectMultiple, Field, TypedMultipleChoiceField
|
||||
from django.db import models
|
||||
from typing import Any, Optional, Dict
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
# this is a form field!
|
||||
class CustomCheckboxSelectMultiple(CheckboxSelectMultiple):
|
||||
template_name = 'widgets/checkbox_select.html'
|
||||
option_template_name = 'widgets/checkbox_option.html'
|
||||
|
||||
def get_context(self, name: str, value: Any, attrs) -> Dict[str, Any]:
|
||||
ctx = super().get_context(name, value, attrs)['widget']
|
||||
ctx["multipleChoiceProperties"] = []
|
||||
for _group, options, _index in ctx["optgroups"]:
|
||||
for option in options:
|
||||
if not isinstance(value,str) and not isinstance(value,list) and ( option["value"] in value.selected_choices or ( value.allow_all and value.all_choice in value.selected_choices ) ):
|
||||
checked = True
|
||||
else:
|
||||
checked = False
|
||||
|
||||
ctx["multipleChoiceProperties"].append({
|
||||
"template_name": option["template_name"],
|
||||
"type": option["type"],
|
||||
"value": option["value"],
|
||||
"label": option["label"],
|
||||
"name": option["name"],
|
||||
"checked": checked})
|
||||
|
||||
return { 'widget': ctx }
|
||||
|
||||
# this is a database field!
|
||||
class CommaSepChoiceField(models.Field):
|
||||
"Implements comma-separated storage of lists"
|
||||
|
||||
def __init__(self, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, *args, **kwargs):
|
||||
self.separator = separator
|
||||
self.possible_choices = possible_choices
|
||||
self.selected_choices = []
|
||||
self.allow_all = allow_all
|
||||
self.all_label = all_label
|
||||
self.all_choice = all_choice
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def deconstruct(self):
|
||||
name, path, args, kwargs = super().deconstruct()
|
||||
if self.separator != ",":
|
||||
kwargs['separator'] = self.separator
|
||||
kwargs['possible_choices'] = self.possible_choices
|
||||
return name, path, args, kwargs
|
||||
|
||||
def db_type(self, connection):
|
||||
return 'text'
|
||||
|
||||
def get_my_choices(self):
|
||||
choiceArray = []
|
||||
if self.possible_choices is None:
|
||||
return choiceArray
|
||||
if self.allow_all:
|
||||
choiceArray.append((self.all_choice, _(self.all_label)))
|
||||
|
||||
for t in self.possible_choices:
|
||||
choiceArray.append(t)
|
||||
|
||||
return choiceArray
|
||||
|
||||
def formfield(self, **kwargs):
|
||||
# This is a fairly standard way to set up some defaults
|
||||
# while letting the caller override them.
|
||||
defaults = {'form_class': MultipleChoiceField,
|
||||
'choices': self.get_my_choices,
|
||||
'widget': CustomCheckboxSelectMultiple,
|
||||
'label': '',
|
||||
'required': False}
|
||||
defaults.update(kwargs)
|
||||
#del defaults.required
|
||||
return super().formfield(**defaults)
|
||||
|
||||
def deconstruct(self):
|
||||
name, path, args, kwargs = super().deconstruct()
|
||||
# Only include kwarg if it's not the default
|
||||
if self.separator != ",":
|
||||
kwargs['separator'] = self.separator
|
||||
return name, path, args, kwargs
|
||||
|
||||
def from_db_value(self, value, expr, conn):
|
||||
if value is None:
|
||||
self.selected_choices = []
|
||||
else:
|
||||
self.selected_choices = value.split(",")
|
||||
|
||||
return self
|
||||
|
||||
def get_prep_value(self, value):
|
||||
if value is None:
|
||||
return ""
|
||||
if not isinstance(value,list):
|
||||
return ""
|
||||
|
||||
if self.all_choice not in value:
|
||||
return ",".join(value)
|
||||
else:
|
||||
return self.all_choice
|
||||
|
||||
def get_text_for_value(self, val):
|
||||
fval = [i for i in self.possible_choices if i[0] == val]
|
||||
if len(fval) <= 0:
|
||||
return []
|
||||
else:
|
||||
return fval[0][1]
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
import os
|
||||
import uuid
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.db.models import signals
|
||||
from common.logger import log
|
||||
from sync.models import Source, Media, MediaServer
|
||||
from sync.signals import media_post_delete
|
||||
from sync.tasks import rescan_media_server
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
|
||||
help = ('Deletes a source by UUID')
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument('--source', action='store', required=True, help='Source UUID')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
source_uuid_str = options.get('source', '')
|
||||
try:
|
||||
source_uuid = uuid.UUID(source_uuid_str)
|
||||
except Exception as e:
|
||||
raise CommandError(f'Failed to parse source UUID: {e}')
|
||||
log.info(f'Deleting source with UUID: {source_uuid}')
|
||||
# Fetch the source by UUID
|
||||
try:
|
||||
source = Source.objects.get(uuid=source_uuid)
|
||||
except Source.DoesNotExist:
|
||||
raise CommandError(f'Source does not exist with '
|
||||
f'UUID: {source_uuid}')
|
||||
# Detach post-delete signal for Media so we don't spam media servers
|
||||
signals.post_delete.disconnect(media_post_delete, sender=Media)
|
||||
# Delete the source, triggering pre-delete signals for each media item
|
||||
log.info(f'Found source with UUID "{source.uuid}" with name '
|
||||
f'"{source.name}" and deleting it, this may take some time!')
|
||||
source.delete()
|
||||
# Update any media servers
|
||||
for mediaserver in MediaServer.objects.all():
|
||||
log.info(f'Scheduling media server updates')
|
||||
verbose_name = _('Request media server rescan for "{}"')
|
||||
rescan_media_server(
|
||||
str(mediaserver.pk),
|
||||
priority=0,
|
||||
verbose_name=verbose_name.format(mediaserver),
|
||||
remove_existing_tasks=True
|
||||
)
|
||||
# Re-attach signals
|
||||
signals.post_delete.connect(media_post_delete, sender=Media)
|
||||
# All done
|
||||
log.info('Done')
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
import os
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from common.logger import log
|
||||
from sync.models import Source, Media, MediaServer
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
|
||||
help = ('Lists sources')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
log.info('Listing sources...')
|
||||
for source in Source.objects.all():
|
||||
log.info(f' - {source.uuid}: {source.name}')
|
||||
log.info('Done')
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
from django.core.management.base import BaseCommand
|
||||
from sync.models import Media
|
||||
|
||||
|
||||
from common.logger import log
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
|
||||
help = 'Resets all media item metadata'
|
||||
|
||||
def handle(self, *args, **options):
|
||||
log.info('Resettings all media metadata...')
|
||||
# Delete all metadata
|
||||
Media.objects.update(metadata=None)
|
||||
# Trigger the save signal on each media item
|
||||
for item in Media.objects.all():
|
||||
item.save()
|
||||
log.info('Done')
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from background_task.models import Task
|
||||
from sync.models import Source
|
||||
from sync.tasks import index_source_task
|
||||
|
||||
|
||||
from common.logger import log
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
|
||||
help = 'Resets all tasks'
|
||||
|
||||
def handle(self, *args, **options):
|
||||
log.info('Resettings all tasks...')
|
||||
# Delete all tasks
|
||||
Task.objects.all().delete()
|
||||
# Iter all tasks
|
||||
for source in Source.objects.all():
|
||||
# Recreate the initial indexing task
|
||||
log.info(f'Resetting tasks for source: {source}')
|
||||
verbose_name = _('Index media from source "{}"')
|
||||
index_source_task(
|
||||
str(source.pk),
|
||||
repeat=source.index_schedule,
|
||||
queue=str(source.pk),
|
||||
priority=5,
|
||||
verbose_name=verbose_name.format(source.name)
|
||||
)
|
||||
# This also chains down to call each Media objects .save() as well
|
||||
source.save()
|
||||
log.info('Done')
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
import json
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from sync.youtube import get_media_info
|
||||
from common.utils import json_serial
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
|
|
@ -15,6 +14,5 @@ class Command(BaseCommand):
|
|||
url = options['url']
|
||||
self.stdout.write(f'Showing information for URL: {url}')
|
||||
info = get_media_info(url)
|
||||
d = json.dumps(info, indent=4, sort_keys=True, default=json_serial)
|
||||
self.stdout.write(d)
|
||||
self.stdout.write(json.dumps(info, indent=4, sort_keys=True))
|
||||
self.stdout.write('Done')
|
||||
|
|
|
|||
|
|
@ -53,8 +53,6 @@ def get_best_audio_format(media):
|
|||
# If the format has a video stream, skip it
|
||||
if fmt['vcodec'] is not None:
|
||||
continue
|
||||
if not fmt['acodec']:
|
||||
continue
|
||||
audio_formats.append(fmt)
|
||||
audio_formats = list(reversed(sorted(audio_formats, key=lambda k: k['abr'])))
|
||||
if not audio_formats:
|
||||
|
|
@ -90,8 +88,6 @@ def get_best_video_format(media):
|
|||
# If the format has an audio stream, skip it
|
||||
if fmt['acodec'] is not None:
|
||||
continue
|
||||
if not fmt['vcodec']:
|
||||
continue
|
||||
if media.source.source_resolution.strip().upper() == fmt['format']:
|
||||
video_formats.append(fmt)
|
||||
# Check we matched some streams
|
||||
|
|
|
|||
|
|
@ -44,9 +44,7 @@ class PlexMediaServer(MediaServer):
|
|||
'<p>The <strong>libraries</strong> is a comma-separated list of Plex '
|
||||
'library or section IDs, you can find out how to get your library or '
|
||||
'section IDs <a href="https://support.plex.tv/articles/201242707-plex-'
|
||||
'media-scanner-via-command-line/#toc-1" target="_blank">here</a> or '
|
||||
'<a href="https://www.plexopedia.com/plex-media-server/api/server/libraries/" '
|
||||
'target="_blank">here</a></p>.')
|
||||
'media-scanner-via-command-line/#toc-1" target="_blank">here</a>.</p>')
|
||||
|
||||
def make_request(self, uri='/', params={}):
|
||||
headers = {'User-Agent': 'TubeSync'}
|
||||
|
|
@ -126,7 +124,7 @@ class PlexMediaServer(MediaServer):
|
|||
# Seems we have a valid library sections page, get the library IDs
|
||||
remote_libraries = {}
|
||||
try:
|
||||
for parent in parsed_response.iter('MediaContainer'):
|
||||
for parent in parsed_response.getiterator('MediaContainer'):
|
||||
for d in parent:
|
||||
library_id = d.attrib['key']
|
||||
library_name = d.attrib['title']
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
# Generated by Django 3.2.7 on 2021-09-24 05:54
|
||||
|
||||
import django.core.files.storage
|
||||
from django.db import migrations, models
|
||||
import sync.models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0009_auto_20210218_0442'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='media',
|
||||
name='media_file',
|
||||
field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='source',
|
||||
name='index_schedule',
|
||||
field=models.IntegerField(choices=[(3600, 'Every hour'), (7200, 'Every 2 hours'), (10800, 'Every 3 hours'), (14400, 'Every 4 hours'), (18000, 'Every 5 hours'), (21600, 'Every 6 hours'), (43200, 'Every 12 hours'), (86400, 'Every 24 hours'), (259200, 'Every 3 days'), (604800, 'Every 7 days'), (0, 'Never')], db_index=True, default=86400, help_text='Schedule of how often to index the source for new media', verbose_name='index schedule'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='source',
|
||||
name='media_format',
|
||||
field=models.CharField(default='{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
# Generated by Django 3.2.11 on 2022-02-01 16:54
|
||||
|
||||
import django.core.files.storage
|
||||
from django.db import migrations, models
|
||||
import sync.models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0010_auto_20210924_0554'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='write_json',
|
||||
field=models.BooleanField(
|
||||
default=False, help_text='Write a JSON file with the media info, these may be detected and used by some media servers', verbose_name='write json'),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Generated by Django 3.2.12 on 2022-04-06 06:19
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0011_auto_20220201_1654'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='media',
|
||||
name='downloaded_format',
|
||||
field=models.CharField(blank=True, help_text='Video format (resolution) of the downloaded media', max_length=30, null=True, verbose_name='downloaded format'),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
# Generated by Django 3.2.12 on 2022-04-06 06:19
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
def fix_media_file(apps, schema_editor):
|
||||
Media = apps.get_model('sync', 'Media')
|
||||
for media in Media.objects.filter(downloaded=True):
|
||||
download_dir = str(settings.DOWNLOAD_ROOT)
|
||||
|
||||
if media.media_file.name.startswith(download_dir):
|
||||
media.media_file.name = media.media_file.name[len(download_dir) + 1:]
|
||||
media.save()
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0012_alter_media_downloaded_format'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(fix_media_file)
|
||||
]
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
# Generated by Django 3.2.15 on 2022-12-28 20:33
|
||||
|
||||
import django.core.files.storage
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
import sync.models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0013_fix_elative_media_file'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='media',
|
||||
name='media_file',
|
||||
field=models.FileField(blank=True, help_text='Media file', max_length=255, null=True, storage=django.core.files.storage.FileSystemStorage(base_url='/media-data/', location=str(settings.DOWNLOAD_ROOT)), upload_to=sync.models.get_media_file_path, verbose_name='media file'),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
# Generated by Django 3.2.17 on 2023-02-13 06:03
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0014_alter_media_media_file'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='media',
|
||||
name='manual_skip',
|
||||
field=models.BooleanField(db_index=True, default=False, help_text='Media marked as "skipped", won\' be downloaded', verbose_name='manual_skip'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='media',
|
||||
name='skip',
|
||||
field=models.BooleanField(db_index=True, default=False, help_text='INTERNAL FLAG - Media will be skipped and not downloaded', verbose_name='skip'),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
# Generated by Django 3.2.18 on 2023-02-14 20:52
|
||||
|
||||
from django.db import migrations, models
|
||||
import sync.models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0015_auto_20230213_0603'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='embed_metadata',
|
||||
field=models.BooleanField(default=False, help_text='Embed metadata from source into file', verbose_name='embed metadata'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='embed_thumbnail',
|
||||
field=models.BooleanField(default=False, help_text='Embed thumbnail into the file', verbose_name='embed thumbnail'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='enable_sponsorblock',
|
||||
field=models.BooleanField(default=True, help_text='Use SponsorBlock?', verbose_name='enable sponsorblock'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='sponsorblock_categories',
|
||||
field=sync.models.CommaSepChoiceField(default='all', possible_choices=(('all', 'All'), ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'), ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'), ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'))),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
# Generated by Django 3.2.18 on 2023-02-20 02:23
|
||||
|
||||
from django.db import migrations
|
||||
import sync.fields
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0016_auto_20230214_2052'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='source',
|
||||
name='sponsorblock_categories',
|
||||
field=sync.fields.CommaSepChoiceField(default='all', help_text='Select the sponsorblocks you want to enforce', separator=''),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
# Generated by pac
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0017_alter_source_sponsorblock_categories'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='write_subtitles',
|
||||
field=models.BooleanField(default=False, help_text='Download video subtitles', verbose_name='write subtitles'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='auto_subtitles',
|
||||
field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto subtitles'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='sub_langs',
|
||||
field=models.CharField(default='en', help_text='List of subtitles langs to download comma-separated. Example: en,fr',max_length=30),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
# Generated by pac
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0018_source_subtitles'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='delete_removed_media',
|
||||
field=models.BooleanField(default=False, help_text='Delete media that is no longer on this playlist', verbose_name='delete removed media'),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
# Generated by Django 3.2.22 on 2023-10-24 17:25
|
||||
|
||||
import django.core.validators
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0019_add_delete_removed_media'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='filter_text',
|
||||
field=models.CharField(blank=True, default='', help_text='Regex compatible filter string for video titles', max_length=100, verbose_name='filter string'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='source',
|
||||
name='auto_subtitles',
|
||||
field=models.BooleanField(default=False, help_text='Accept auto-generated subtitles', verbose_name='accept auto-generated subs'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='source',
|
||||
name='sub_langs',
|
||||
field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z]+,)*(\\-?[\\_\\.a-zA-Z]+){1}$')], verbose_name='subs langs'),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
# Generated by pac
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sync', '0020_auto_20231024_1825'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='source',
|
||||
name='delete_files_on_disk',
|
||||
field=models.BooleanField(default=False, help_text='Delete files on disk when they are removed from TubeSync', verbose_name='delete files on disk'),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
import os
|
||||
import uuid
|
||||
import json
|
||||
import re
|
||||
from xml.etree import ElementTree
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime, timedelta
|
||||
|
|
@ -9,7 +8,6 @@ from pathlib import Path
|
|||
from django.conf import settings
|
||||
from django.db import models
|
||||
from django.core.files.storage import FileSystemStorage
|
||||
from django.core.validators import RegexValidator
|
||||
from django.utils.text import slugify
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
|
@ -21,9 +19,10 @@ from .utils import seconds_to_timestr, parse_media_format
|
|||
from .matching import (get_best_combined_format, get_best_audio_format,
|
||||
get_best_video_format)
|
||||
from .mediaservers import PlexMediaServer
|
||||
from .fields import CommaSepChoiceField
|
||||
|
||||
media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/')
|
||||
|
||||
media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT))
|
||||
|
||||
|
||||
class Source(models.Model):
|
||||
'''
|
||||
|
|
@ -107,43 +106,6 @@ class Source(models.Model):
|
|||
EXTENSION_MKV = 'mkv'
|
||||
EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
|
||||
|
||||
# as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py
|
||||
SPONSORBLOCK_CATEGORIES_CHOICES = (
|
||||
('sponsor', 'Sponsor'),
|
||||
('intro', 'Intermission/Intro Animation'),
|
||||
('outro', 'Endcards/Credits'),
|
||||
('selfpromo', 'Unpaid/Self Promotion'),
|
||||
('preview', 'Preview/Recap'),
|
||||
('filler', 'Filler Tangent'),
|
||||
('interaction', 'Interaction Reminder'),
|
||||
('music_offtopic', 'Non-Music Section'),
|
||||
)
|
||||
|
||||
sponsorblock_categories = CommaSepChoiceField(
|
||||
_(''),
|
||||
possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES,
|
||||
all_choice='all',
|
||||
allow_all=True,
|
||||
all_label='(all options)',
|
||||
default='all',
|
||||
help_text=_('Select the sponsorblocks you want to enforce')
|
||||
)
|
||||
embed_metadata = models.BooleanField(
|
||||
_('embed metadata'),
|
||||
default=False,
|
||||
help_text=_('Embed metadata from source into file')
|
||||
)
|
||||
embed_thumbnail = models.BooleanField(
|
||||
_('embed thumbnail'),
|
||||
default=False,
|
||||
help_text=_('Embed thumbnail into the file')
|
||||
)
|
||||
enable_sponsorblock = models.BooleanField(
|
||||
_('enable sponsorblock'),
|
||||
default=True,
|
||||
help_text=_('Use SponsorBlock?')
|
||||
)
|
||||
|
||||
# Fontawesome icons used for the source on the front end
|
||||
ICONS = {
|
||||
SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
|
||||
|
|
@ -196,9 +158,6 @@ class Source(models.Model):
|
|||
EVERY_6_HOURS = 21600, _('Every 6 hours')
|
||||
EVERY_12_HOURS = 43200, _('Every 12 hours')
|
||||
EVERY_24_HOURS = 86400, _('Every 24 hours')
|
||||
EVERY_3_DAYS = 259200, _('Every 3 days')
|
||||
EVERY_7_DAYS = 604800, _('Every 7 days')
|
||||
NEVER = 0, _('Never')
|
||||
|
||||
uuid = models.UUIDField(
|
||||
_('uuid'),
|
||||
|
|
@ -259,7 +218,7 @@ class Source(models.Model):
|
|||
_('index schedule'),
|
||||
choices=IndexSchedule.choices,
|
||||
db_index=True,
|
||||
default=IndexSchedule.EVERY_24_HOURS,
|
||||
default=IndexSchedule.EVERY_6_HOURS,
|
||||
help_text=_('Schedule of how often to index the source for new media')
|
||||
)
|
||||
download_media = models.BooleanField(
|
||||
|
|
@ -284,23 +243,6 @@ class Source(models.Model):
|
|||
help_text=_('If "delete old media" is ticked, the number of days after which '
|
||||
'to automatically delete media')
|
||||
)
|
||||
filter_text = models.CharField(
|
||||
_('filter string'),
|
||||
max_length=100,
|
||||
default='',
|
||||
blank=True,
|
||||
help_text=_('Regex compatible filter string for video titles')
|
||||
)
|
||||
delete_removed_media = models.BooleanField(
|
||||
_('delete removed media'),
|
||||
default=False,
|
||||
help_text=_('Delete media that is no longer on this playlist')
|
||||
)
|
||||
delete_files_on_disk = models.BooleanField(
|
||||
_('delete files on disk'),
|
||||
default=False,
|
||||
help_text=_('Delete files on disk when they are removed from TubeSync')
|
||||
)
|
||||
source_resolution = models.CharField(
|
||||
_('source resolution'),
|
||||
max_length=8,
|
||||
|
|
@ -353,41 +295,12 @@ class Source(models.Model):
|
|||
default=False,
|
||||
help_text=_('Write an NFO file in XML with the media info, these may be detected and used by some media servers')
|
||||
)
|
||||
write_json = models.BooleanField(
|
||||
_('write json'),
|
||||
default=False,
|
||||
help_text=_('Write a JSON file with the media info, these may be detected and used by some media servers')
|
||||
)
|
||||
has_failed = models.BooleanField(
|
||||
_('has failed'),
|
||||
default=False,
|
||||
help_text=_('Source has failed to index media')
|
||||
)
|
||||
|
||||
write_subtitles = models.BooleanField(
|
||||
_('write subtitles'),
|
||||
default=False,
|
||||
help_text=_('Download video subtitles')
|
||||
)
|
||||
|
||||
auto_subtitles = models.BooleanField(
|
||||
_('accept auto-generated subs'),
|
||||
default=False,
|
||||
help_text=_('Accept auto-generated subtitles')
|
||||
)
|
||||
sub_langs = models.CharField(
|
||||
_('subs langs'),
|
||||
max_length=30,
|
||||
default='en',
|
||||
help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'),
|
||||
validators=[
|
||||
RegexValidator(
|
||||
regex=r"^(\-?[\_\.a-zA-Z]+,)*(\-?[\_\.a-zA-Z]+){1}$",
|
||||
message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat')
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
|
|
@ -471,14 +384,10 @@ class Source(models.Model):
|
|||
@property
|
||||
def directory_path(self):
|
||||
download_dir = Path(media_file_storage.location)
|
||||
return download_dir / self.type_directory_path
|
||||
|
||||
@property
|
||||
def type_directory_path(self):
|
||||
if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO:
|
||||
return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory
|
||||
return download_dir / settings.DOWNLOAD_AUDIO_DIR / self.directory
|
||||
else:
|
||||
return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory
|
||||
return download_dir / settings.DOWNLOAD_VIDEO_DIR / self.directory
|
||||
|
||||
def make_directory(self):
|
||||
return os.makedirs(self.directory_path, exist_ok=True)
|
||||
|
|
@ -516,20 +425,19 @@ class Source(models.Model):
|
|||
fmt.append('60fps')
|
||||
if self.prefer_hdr:
|
||||
fmt.append('hdr')
|
||||
now = timezone.now()
|
||||
return {
|
||||
'yyyymmdd': now.strftime('%Y%m%d'),
|
||||
'yyyy_mm_dd': now.strftime('%Y-%m-%d'),
|
||||
'yyyy': now.strftime('%Y'),
|
||||
'mm': now.strftime('%m'),
|
||||
'dd': now.strftime('%d'),
|
||||
'yyyymmdd': timezone.now().strftime('%Y%m%d'),
|
||||
'yyyy_mm_dd': timezone.now().strftime('%Y-%m-%d'),
|
||||
'yyyy': timezone.now().strftime('%Y'),
|
||||
'mm': timezone.now().strftime('%m'),
|
||||
'dd': timezone.now().strftime('%d'),
|
||||
'source': self.slugname,
|
||||
'source_full': self.name,
|
||||
'uploader': 'Some Channel Name',
|
||||
'title': 'some-media-title-name',
|
||||
'title_full': 'Some Media Title Name',
|
||||
'key': 'SoMeUnIqUiD',
|
||||
'format': '-'.join(fmt),
|
||||
'playlist_index': 1,
|
||||
'playlist_title': 'Some Playlist Title',
|
||||
'ext': self.extension,
|
||||
'resolution': self.source_resolution if self.source_resolution else '',
|
||||
|
|
@ -547,11 +455,6 @@ class Source(models.Model):
|
|||
except Exception as e:
|
||||
return ''
|
||||
|
||||
def is_regex_match(self, media_item_title):
|
||||
if not self.filter_text:
|
||||
return True
|
||||
return bool(re.search(self.filter_text, media_item_title))
|
||||
|
||||
def index_media(self):
|
||||
'''
|
||||
Index the media source returning a list of media metadata as dicts.
|
||||
|
|
@ -562,11 +465,7 @@ class Source(models.Model):
|
|||
response = indexer(self.index_url)
|
||||
if not isinstance(response, dict):
|
||||
return []
|
||||
entries = response.get('entries', [])
|
||||
|
||||
if settings.MAX_ENTRIES_PROCESSING:
|
||||
entries = entries[:settings.MAX_ENTRIES_PROCESSING]
|
||||
return entries
|
||||
return response.get('entries', [])
|
||||
|
||||
|
||||
def get_media_thumb_path(instance, filename):
|
||||
|
|
@ -660,6 +559,11 @@ class Media(models.Model):
|
|||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count',
|
||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count',
|
||||
},
|
||||
'playlist_index': {
|
||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_index',
|
||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_index',
|
||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'playlist_index',
|
||||
},
|
||||
'playlist_title': {
|
||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title',
|
||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title',
|
||||
|
|
@ -754,7 +658,7 @@ class Media(models.Model):
|
|||
media_file = models.FileField(
|
||||
_('media file'),
|
||||
upload_to=get_media_file_path,
|
||||
max_length=255,
|
||||
max_length=200,
|
||||
blank=True,
|
||||
null=True,
|
||||
storage=media_file_storage,
|
||||
|
|
@ -764,13 +668,7 @@ class Media(models.Model):
|
|||
_('skip'),
|
||||
db_index=True,
|
||||
default=False,
|
||||
help_text=_('INTERNAL FLAG - Media will be skipped and not downloaded')
|
||||
)
|
||||
manual_skip = models.BooleanField(
|
||||
_('manual_skip'),
|
||||
db_index=True,
|
||||
default=False,
|
||||
help_text=_('Media marked as "skipped", won\' be downloaded')
|
||||
help_text=_('Media will be skipped and not downloaded')
|
||||
)
|
||||
downloaded = models.BooleanField(
|
||||
_('downloaded'),
|
||||
|
|
@ -790,7 +688,7 @@ class Media(models.Model):
|
|||
max_length=30,
|
||||
blank=True,
|
||||
null=True,
|
||||
help_text=_('Video format (resolution) of the downloaded media')
|
||||
help_text=_('Audio codec of the downloaded media')
|
||||
)
|
||||
downloaded_height = models.PositiveIntegerField(
|
||||
_('downloaded height'),
|
||||
|
|
@ -917,23 +815,6 @@ class Media(models.Model):
|
|||
hdr = ''
|
||||
# If the download has completed use existing values
|
||||
if self.downloaded:
|
||||
# Check if there's any stored meta data at all
|
||||
if (not self.downloaded_video_codec and \
|
||||
not self.downloaded_audio_codec):
|
||||
# Marked as downloaded but no metadata, imported?
|
||||
return {
|
||||
'resolution': resolution,
|
||||
'height': height,
|
||||
'width': width,
|
||||
'vcodec': vcodec,
|
||||
'acodec': acodec,
|
||||
'fps': fps,
|
||||
'hdr': hdr,
|
||||
'format': tuple(fmt),
|
||||
}
|
||||
if self.downloaded_format:
|
||||
resolution = self.downloaded_format.lower()
|
||||
elif self.downloaded_height:
|
||||
resolution = f'{self.downloaded_height}p'
|
||||
if self.downloaded_format != 'audio':
|
||||
vcodec = self.downloaded_video_codec.lower()
|
||||
|
|
@ -961,7 +842,7 @@ class Media(models.Model):
|
|||
# Otherwise, calculate from matched format codes
|
||||
vformat = None
|
||||
aformat = None
|
||||
if format_str and '+' in format_str:
|
||||
if '+' in format_str:
|
||||
# Seperate audio and video streams
|
||||
vformat_code, aformat_code = format_str.split('+')
|
||||
vformat = self.get_format_by_code(vformat_code)
|
||||
|
|
@ -970,7 +851,7 @@ class Media(models.Model):
|
|||
# Combined stream or audio only
|
||||
cformat = self.get_format_by_code(format_str)
|
||||
aformat = cformat
|
||||
if cformat and cformat['vcodec']:
|
||||
if cformat['vcodec']:
|
||||
# Combined
|
||||
vformat = cformat
|
||||
if vformat:
|
||||
|
|
@ -1031,6 +912,7 @@ class Media(models.Model):
|
|||
'title_full': clean_filename(self.title),
|
||||
'key': self.key,
|
||||
'format': '-'.join(display_format['format']),
|
||||
'playlist_index': self.playlist_index,
|
||||
'playlist_title': self.playlist_title,
|
||||
'ext': self.source.extension,
|
||||
'resolution': display_format['resolution'],
|
||||
|
|
@ -1040,7 +922,6 @@ class Media(models.Model):
|
|||
'acodec': display_format['acodec'],
|
||||
'fps': display_format['fps'],
|
||||
'hdr': display_format['hdr'],
|
||||
'uploader': self.uploader,
|
||||
}
|
||||
|
||||
@property
|
||||
|
|
@ -1050,10 +931,7 @@ class Media(models.Model):
|
|||
@property
|
||||
def loaded_metadata(self):
|
||||
try:
|
||||
data = json.loads(self.metadata)
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
return data
|
||||
return json.loads(self.metadata)
|
||||
except Exception as e:
|
||||
return {}
|
||||
|
||||
|
|
@ -1090,10 +968,7 @@ class Media(models.Model):
|
|||
@property
|
||||
def upload_date(self):
|
||||
field = self.get_metadata_field('upload_date')
|
||||
try:
|
||||
upload_date_str = self.loaded_metadata.get(field, '').strip()
|
||||
except (AttributeError, ValueError) as e:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(upload_date_str, '%Y%m%d')
|
||||
except (AttributeError, ValueError) as e:
|
||||
|
|
@ -1102,12 +977,7 @@ class Media(models.Model):
|
|||
@property
|
||||
def duration(self):
|
||||
field = self.get_metadata_field('duration')
|
||||
duration = self.loaded_metadata.get(field, 0)
|
||||
try:
|
||||
duration = int(duration)
|
||||
except (TypeError, ValueError):
|
||||
duration = 0
|
||||
return duration
|
||||
return int(self.loaded_metadata.get(field, 0))
|
||||
|
||||
@property
|
||||
def duration_formatted(self):
|
||||
|
|
@ -1153,6 +1023,11 @@ class Media(models.Model):
|
|||
field = self.get_metadata_field('formats')
|
||||
return self.loaded_metadata.get(field, [])
|
||||
|
||||
@property
|
||||
def playlist_index(self):
|
||||
field = self.get_metadata_field('playlist_index')
|
||||
return self.loaded_metadata.get(field, 0)
|
||||
|
||||
@property
|
||||
def playlist_title(self):
|
||||
field = self.get_metadata_field('playlist_title')
|
||||
|
|
@ -1167,9 +1042,6 @@ class Media(models.Model):
|
|||
|
||||
@property
|
||||
def thumbname(self):
|
||||
if self.downloaded and self.media_file:
|
||||
filename = os.path.basename(self.media_file.path)
|
||||
else:
|
||||
filename = self.filename
|
||||
prefix, ext = os.path.splitext(filename)
|
||||
return f'{prefix}.jpg'
|
||||
|
|
@ -1180,9 +1052,6 @@ class Media(models.Model):
|
|||
|
||||
@property
|
||||
def nfoname(self):
|
||||
if self.downloaded and self.media_file:
|
||||
filename = os.path.basename(self.media_file.path)
|
||||
else:
|
||||
filename = self.filename
|
||||
prefix, ext = os.path.splitext(filename)
|
||||
return f'{prefix}.nfo'
|
||||
|
|
@ -1191,19 +1060,6 @@ class Media(models.Model):
|
|||
def nfopath(self):
|
||||
return self.source.directory_path / self.nfoname
|
||||
|
||||
@property
|
||||
def jsonname(self):
|
||||
if self.downloaded and self.media_file:
|
||||
filename = os.path.basename(self.media_file.path)
|
||||
else:
|
||||
filename = self.filename
|
||||
prefix, ext = os.path.splitext(filename)
|
||||
return f'{prefix}.info.json'
|
||||
|
||||
@property
|
||||
def jsonpath(self):
|
||||
return self.source.directory_path / self.jsonname
|
||||
|
||||
@property
|
||||
def directory_path(self):
|
||||
# Otherwise, create a suitable filename from the source media_format
|
||||
|
|
@ -1228,29 +1084,6 @@ class Media(models.Model):
|
|||
return False
|
||||
return os.path.exists(self.media_file.path)
|
||||
|
||||
@property
|
||||
def content_type(self):
|
||||
if not self.downloaded:
|
||||
return 'video/mp4'
|
||||
vcodec = self.downloaded_video_codec
|
||||
if vcodec is None:
|
||||
acodec = self.downloaded_audio_codec
|
||||
if acodec is None:
|
||||
raise TypeError() # nothing here.
|
||||
acodec = acodec.lower()
|
||||
if acodec == "mp4a":
|
||||
return "audio/mp4"
|
||||
elif acodec == "opus":
|
||||
return "audio/opus"
|
||||
else:
|
||||
# fall-fall-back.
|
||||
return 'audio/ogg'
|
||||
vcodec = vcodec.lower()
|
||||
if vcodec == 'vp9':
|
||||
return 'video/webm'
|
||||
else:
|
||||
return 'video/mp4'
|
||||
|
||||
@property
|
||||
def nfoxml(self):
|
||||
'''
|
||||
|
|
@ -1268,22 +1101,6 @@ class Media(models.Model):
|
|||
showtitle.text = str(self.source.name).strip()
|
||||
showtitle.tail = '\n '
|
||||
nfo.append(showtitle)
|
||||
# season = upload date year
|
||||
season = nfo.makeelement('season', {})
|
||||
if self.source.source_type == Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
|
||||
# If it's a playlist, set season to 1
|
||||
season.text = '1'
|
||||
else:
|
||||
# If it's not a playlist, set season to upload date year
|
||||
season.text = str(self.upload_date.year) if self.upload_date else ''
|
||||
season.tail = '\n '
|
||||
nfo.append(season)
|
||||
# episode = number of video in the year
|
||||
episode = nfo.makeelement('episode', {})
|
||||
episode_number = self.calculate_episode_number()
|
||||
episode.text = str(episode_number) if episode_number else ''
|
||||
episode.tail = '\n '
|
||||
nfo.append(episode)
|
||||
# ratings = media metadata youtube rating
|
||||
value = nfo.makeelement('value', {})
|
||||
value.text = str(self.rating)
|
||||
|
|
@ -1391,10 +1208,7 @@ class Media(models.Model):
|
|||
f'no valid format available')
|
||||
# Download the media with youtube-dl
|
||||
download_youtube_media(self.url, format_str, self.source.extension,
|
||||
str(self.filepath), self.source.write_json,
|
||||
self.source.sponsorblock_categories.selected_choices, self.source.embed_thumbnail,
|
||||
self.source.embed_metadata, self.source.enable_sponsorblock,
|
||||
self.source.write_subtitles, self.source.auto_subtitles,self.source.sub_langs )
|
||||
str(self.filepath))
|
||||
# Return the download paramaters
|
||||
return format_str, self.source.extension
|
||||
|
||||
|
|
@ -1404,23 +1218,10 @@ class Media(models.Model):
|
|||
'''
|
||||
indexer = self.INDEXERS.get(self.source.source_type, None)
|
||||
if not callable(indexer):
|
||||
raise Exception(f'Media with source type f"{self.source.source_type}" '
|
||||
raise Exception(f'Meida with source type f"{self.source.source_type}" '
|
||||
f'has no indexer')
|
||||
return indexer(self.url)
|
||||
|
||||
def calculate_episode_number(self):
|
||||
if self.source.source_type == Source.SOURCE_TYPE_YOUTUBE_PLAYLIST:
|
||||
sorted_media = Media.objects.filter(source=self.source)
|
||||
else:
|
||||
self_year = self.upload_date.year if self.upload_date else self.created.year
|
||||
filtered_media = Media.objects.filter(source=self.source, published__year=self_year)
|
||||
sorted_media = sorted(filtered_media, key=lambda x: (x.upload_date, x.key))
|
||||
position_counter = 1
|
||||
for media in sorted_media:
|
||||
if media == self:
|
||||
return position_counter
|
||||
position_counter += 1
|
||||
|
||||
|
||||
class MediaServer(models.Model):
|
||||
'''
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import os
|
||||
import glob
|
||||
from django.conf import settings
|
||||
from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
|
||||
from django.dispatch import receiver
|
||||
|
|
@ -48,7 +47,6 @@ def source_post_save(sender, instance, created, **kwargs):
|
|||
priority=0,
|
||||
verbose_name=verbose_name.format(instance.name)
|
||||
)
|
||||
if instance.index_schedule > 0:
|
||||
delete_task_by_source('sync.tasks.index_source_task', instance.pk)
|
||||
log.info(f'Scheduling media indexing for source: {instance.name}')
|
||||
verbose_name = _('Index media from source "{}"')
|
||||
|
|
@ -75,7 +73,6 @@ def source_pre_delete(sender, instance, **kwargs):
|
|||
media.delete()
|
||||
|
||||
|
||||
|
||||
@receiver(post_delete, sender=Source)
|
||||
def source_post_delete(sender, instance, **kwargs):
|
||||
# Triggered after a source is deleted
|
||||
|
|
@ -95,80 +92,17 @@ def task_task_failed(sender, task_id, completed_task, **kwargs):
|
|||
|
||||
@receiver(post_save, sender=Media)
|
||||
def media_post_save(sender, instance, created, **kwargs):
|
||||
# If the media is skipped manually, bail.
|
||||
if instance.manual_skip:
|
||||
return
|
||||
# Triggered after media is saved
|
||||
cap_changed = False
|
||||
can_download_changed = False
|
||||
# Reset the skip flag if the download cap has changed if the media has not
|
||||
# already been downloaded
|
||||
if not instance.downloaded and instance.metadata:
|
||||
max_cap_age = instance.source.download_cap_date
|
||||
filter_text = instance.source.filter_text.strip()
|
||||
published = instance.published
|
||||
if not published:
|
||||
if not instance.skip:
|
||||
log.warn(f'Media: {instance.source} / {instance} has no published date '
|
||||
f'set, marking to be skipped')
|
||||
instance.skip = True
|
||||
cap_changed = True
|
||||
else:
|
||||
log.debug(f'Media: {instance.source} / {instance} has no published date '
|
||||
f'set but is already marked to be skipped')
|
||||
else:
|
||||
if max_cap_age:
|
||||
if published > max_cap_age and instance.skip:
|
||||
if filter_text:
|
||||
if instance.source.is_regex_match(instance.title):
|
||||
log.info(f'Media: {instance.source} / {instance} has a valid '
|
||||
f'publishing date and title filter, marking to be unskipped')
|
||||
instance.skip = False
|
||||
cap_changed = True
|
||||
else:
|
||||
log.debug(f'Media: {instance.source} / {instance} has a valid publishing date '
|
||||
f'but failed the title filter match, already marked skipped')
|
||||
else:
|
||||
log.info(f'Media: {instance.source} / {instance} has a valid '
|
||||
f'publishing date, marking to be unskipped')
|
||||
instance.skip = False
|
||||
cap_changed = True
|
||||
elif published <= max_cap_age and not instance.skip:
|
||||
log.info(f'Media: {instance.source} / {instance} is too old for '
|
||||
f'the download cap date, marking to be skipped')
|
||||
instance.skip = True
|
||||
cap_changed = True
|
||||
else:
|
||||
if instance.skip:
|
||||
# Media marked to be skipped but source download cap removed
|
||||
if filter_text:
|
||||
if instance.source.is_regex_match(instance.title):
|
||||
log.info(f'Media: {instance.source} / {instance} has a valid '
|
||||
f'publishing date and title filter, marking to be unskipped')
|
||||
instance.skip = False
|
||||
cap_changed = True
|
||||
else:
|
||||
log.info(f'Media: {instance.source} / {instance} has a valid publishing date '
|
||||
f'but failed the title filter match, already marked skipped')
|
||||
else:
|
||||
log.debug(f'Media: {instance.source} / {instance} has a valid publishing date and '
|
||||
f'is already marked as not to be skipped')
|
||||
|
||||
cap_changed = False
|
||||
# Recalculate the "can_download" flag, this may
|
||||
# Triggered after media is saved, Recalculate the "can_download" flag, this may
|
||||
# need to change if the source specifications have been changed
|
||||
if instance.metadata:
|
||||
post_save.disconnect(media_post_save, sender=Media)
|
||||
if instance.get_format_str():
|
||||
if not instance.can_download:
|
||||
instance.can_download = True
|
||||
can_download_changed = True
|
||||
instance.save()
|
||||
else:
|
||||
if instance.can_download:
|
||||
instance.can_download = False
|
||||
can_download_changed = True
|
||||
# Save the instance if any changes were required
|
||||
if cap_changed or can_download_changed:
|
||||
post_save.disconnect(media_post_save, sender=Media)
|
||||
instance.save()
|
||||
post_save.connect(media_post_save, sender=Media)
|
||||
# If the media is missing metadata schedule it to be downloaded
|
||||
|
|
@ -177,7 +111,7 @@ def media_post_save(sender, instance, created, **kwargs):
|
|||
verbose_name = _('Downloading metadata for "{}"')
|
||||
download_media_metadata(
|
||||
str(instance.pk),
|
||||
priority=5,
|
||||
priority=10,
|
||||
verbose_name=verbose_name.format(instance.pk),
|
||||
remove_existing_tasks=True
|
||||
)
|
||||
|
|
@ -224,16 +158,6 @@ def media_pre_delete(sender, instance, **kwargs):
|
|||
if thumbnail_url:
|
||||
delete_task_by_media('sync.tasks.download_media_thumbnail',
|
||||
(str(instance.pk), thumbnail_url))
|
||||
if instance.source.delete_files_on_disk and (instance.media_file or instance.thumb):
|
||||
# Delete all media files if it contains filename
|
||||
filepath = instance.media_file.path if instance.media_file else instance.thumb.path
|
||||
barefilepath, fileext = os.path.splitext(filepath)
|
||||
# Get all files that start with the bare file path
|
||||
all_related_files = glob.glob(f'{barefilepath}.*')
|
||||
for file in all_related_files:
|
||||
log.info(f'Deleting file for: {instance} path: {file}')
|
||||
delete_file(file)
|
||||
|
||||
|
||||
|
||||
@receiver(post_delete, sender=Media)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import math
|
|||
import uuid
|
||||
from io import BytesIO
|
||||
from hashlib import sha1
|
||||
from datetime import timedelta, datetime
|
||||
from datetime import timedelta
|
||||
from shutil import copyfile
|
||||
from PIL import Image
|
||||
from django.conf import settings
|
||||
|
|
@ -22,7 +22,6 @@ from background_task import background
|
|||
from background_task.models import Task, CompletedTask
|
||||
from common.logger import log
|
||||
from common.errors import NoMediaException, DownloadFailedException
|
||||
from common.utils import json_serial
|
||||
from .models import Source, Media, MediaServer
|
||||
from .utils import (get_remote_image, resize_image_to_height, delete_file,
|
||||
write_text_file)
|
||||
|
|
@ -132,25 +131,18 @@ def cleanup_completed_tasks():
|
|||
|
||||
|
||||
def cleanup_old_media():
|
||||
for source in Source.objects.filter(delete_old_media=True, days_to_keep__gt=0):
|
||||
delta = timezone.now() - timedelta(days=source.days_to_keep)
|
||||
for media in source.media_source.filter(downloaded=True, download_date__lt=delta):
|
||||
log.info(f'Deleting expired media: {source} / {media} '
|
||||
f'(now older than {source.days_to_keep} days / '
|
||||
for media in Media.objects.filter(download_date__isnull=False):
|
||||
if media.source.delete_old_media and media.source.days_to_keep > 0:
|
||||
delta = timezone.now() - timedelta(days=media.source.days_to_keep)
|
||||
if media.downloaded and media.download_date < delta:
|
||||
# Media was downloaded after the cutoff date, delete it
|
||||
log.info(f'Deleting expired media: {media.source} / {media} '
|
||||
f'(now older than {media.source.days_to_keep} days / '
|
||||
f'download_date before {delta})')
|
||||
# .delete() also triggers a pre_delete signal that removes the files
|
||||
media.delete()
|
||||
|
||||
|
||||
def cleanup_removed_media(source, videos):
|
||||
media_objects = Media.objects.filter(source=source, downloaded=True)
|
||||
for item in media_objects:
|
||||
matching_source_item = [video['id'] for video in videos if video['id'] == item.key]
|
||||
if not matching_source_item:
|
||||
log.info(f'{item.title} is no longer in source, removing')
|
||||
item.delete()
|
||||
|
||||
|
||||
@background(schedule=0)
|
||||
def index_source_task(source_id):
|
||||
'''
|
||||
|
|
@ -160,6 +152,7 @@ def index_source_task(source_id):
|
|||
source = Source.objects.get(pk=source_id)
|
||||
except Source.DoesNotExist:
|
||||
# Task triggered but the Source has been deleted, delete the task
|
||||
delete_index_source_task(source_id)
|
||||
return
|
||||
# Reset any errors
|
||||
source.has_failed = False
|
||||
|
|
@ -182,7 +175,7 @@ def index_source_task(source_id):
|
|||
# Video has no unique key (ID), it can't be indexed
|
||||
continue
|
||||
try:
|
||||
media = Media.objects.get(key=key, source=source)
|
||||
media = Media.objects.get(key=key)
|
||||
except Media.DoesNotExist:
|
||||
media = Media(key=key)
|
||||
media.source = source
|
||||
|
|
@ -195,9 +188,6 @@ def index_source_task(source_id):
|
|||
cleanup_completed_tasks()
|
||||
# Tack on a cleanup of old media
|
||||
cleanup_old_media()
|
||||
if source.delete_removed_media:
|
||||
log.info(f'Cleaning up media no longer in source {source}')
|
||||
cleanup_removed_media(source, videos)
|
||||
|
||||
|
||||
@background(schedule=0)
|
||||
|
|
@ -211,6 +201,7 @@ def check_source_directory_exists(source_id):
|
|||
source = Source.objects.get(pk=source_id)
|
||||
except Source.DoesNotExist:
|
||||
# Task triggered but the Source has been deleted, delete the task
|
||||
delete_index_source_task(source_id)
|
||||
return
|
||||
# Check the source output directory exists
|
||||
if not source.directory_exists():
|
||||
|
|
@ -231,12 +222,9 @@ def download_media_metadata(media_id):
|
|||
log.error(f'Task download_media_metadata(pk={media_id}) called but no '
|
||||
f'media exists with ID: {media_id}')
|
||||
return
|
||||
if media.manual_skip:
|
||||
log.info(f'Task for ID: {media_id} skipped, due to task being manually skipped.')
|
||||
return
|
||||
source = media.source
|
||||
metadata = media.index_metadata()
|
||||
media.metadata = json.dumps(metadata, default=json_serial)
|
||||
media.metadata = json.dumps(metadata)
|
||||
upload_date = media.upload_date
|
||||
# Media must have a valid upload date
|
||||
if upload_date:
|
||||
|
|
@ -246,24 +234,14 @@ def download_media_metadata(media_id):
|
|||
media.skip = True
|
||||
# If the source has a download cap date check the upload date is allowed
|
||||
max_cap_age = source.download_cap_date
|
||||
if media.published and max_cap_age:
|
||||
if max_cap_age:
|
||||
if media.published < max_cap_age:
|
||||
# Media was published after the cap date, skip it
|
||||
log.warn(f'Media: {source} / {media} is older than cap age '
|
||||
f'{max_cap_age}, skipping')
|
||||
media.skip = True
|
||||
# If the source has a search filter, check the video title matches the filter
|
||||
if source.filter_text and not source.is_regex_match(media.title):
|
||||
# Filter text not found in the media title. Accepts regex string, blank search filter results in this returning false
|
||||
log.warn(f'Media: {source} / {media} does not match {source.filter_text}, skipping')
|
||||
media.skip = True
|
||||
# If the source has a cut-off check the upload date is within the allowed delta
|
||||
if source.delete_old_media and source.days_to_keep > 0:
|
||||
if not isinstance(media.published, datetime):
|
||||
# Media has no known published date or incomplete metadata
|
||||
log.warn(f'Media: {source} / {media} has no published date, skipping')
|
||||
media.skip = True
|
||||
else:
|
||||
delta = timezone.now() - timedelta(days=source.days_to_keep)
|
||||
if media.published < delta:
|
||||
# Media was published after the cutoff date, skip it
|
||||
|
|
@ -327,26 +305,18 @@ def download_media(media_id):
|
|||
return
|
||||
if media.skip:
|
||||
# Media was toggled to be skipped after the task was scheduled
|
||||
log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
|
||||
f'it is now marked to be skipped, not downloading')
|
||||
log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but it '
|
||||
f'is now marked to be skipped, not downloading')
|
||||
return
|
||||
if media.downloaded and media.media_file:
|
||||
# Media has been marked as downloaded before the download_media task was fired,
|
||||
# skip it
|
||||
log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
|
||||
f'it has already been marked as downloaded, not downloading again')
|
||||
log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but it '
|
||||
f'has already been marked as downloaded, not downloading again')
|
||||
return
|
||||
if not media.source.download_media:
|
||||
log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
|
||||
f'the source {media.source} has since been marked to not download, '
|
||||
f'not downloading')
|
||||
return
|
||||
max_cap_age = media.source.download_cap_date
|
||||
published = media.published
|
||||
if max_cap_age and published:
|
||||
if published <= max_cap_age:
|
||||
log.warn(f'Download task triggered media: {media} (UUID: {media.pk}) but '
|
||||
f'the source has a download cap and the media is now too old, '
|
||||
log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but the '
|
||||
f'source {media.source} has since been marked to not download media, '
|
||||
f'not downloading')
|
||||
return
|
||||
filepath = media.filepath
|
||||
|
|
@ -357,7 +327,7 @@ def download_media(media_id):
|
|||
log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: '
|
||||
f'"{filepath}"')
|
||||
# Link the media file to the object and update info about the download
|
||||
media.media_file.name = str(media.source.type_directory_path / media.filename)
|
||||
media.media_file.name = str(filepath)
|
||||
media.downloaded = True
|
||||
media.download_date = timezone.now()
|
||||
media.downloaded_filesize = os.path.getsize(filepath)
|
||||
|
|
|
|||
|
|
@ -25,12 +25,12 @@
|
|||
</tr>
|
||||
<tr>
|
||||
<td>{mm}</td>
|
||||
<td>Media publish month in MM</td>
|
||||
<td>Media publish year in MM</td>
|
||||
<td>01</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>{dd}</td>
|
||||
<td>Media publish day in DD</td>
|
||||
<td>Media publish year in DD</td>
|
||||
<td>31</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -43,11 +43,6 @@
|
|||
<td>Full source name</td>
|
||||
<td>My Source</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>{uploader}</td>
|
||||
<td>Uploader name</td>
|
||||
<td>Some Channel Name</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>{title}</td>
|
||||
<td>Lower case media title, max 80 chars</td>
|
||||
|
|
@ -68,6 +63,11 @@
|
|||
<td>Media format string</td>
|
||||
<td>720p-avc1-mp4a</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>{playlist_index}</td>
|
||||
<td>Playlist index of media, if it's in a playlist</td>
|
||||
<td>12</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>{playlist_title}</td>
|
||||
<td>Playlist title of media, if it's in a playlist</td>
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@
|
|||
</div>
|
||||
<div class="row">
|
||||
<div class="col s12">
|
||||
<h2 class="truncate">Runtime information</h2>
|
||||
<h2 class="truncate">Runtime infomation</h2>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
|
|
@ -123,10 +123,6 @@
|
|||
<td class="hide-on-small-only">Downloads directory</td>
|
||||
<td><span class="hide-on-med-and-up">Downloads directory<br></span><strong>{{ downloads_dir }}</strong></td>
|
||||
</tr>
|
||||
<tr title="Database connection used by TubeSync">
|
||||
<td class="hide-on-small-only">Database</td>
|
||||
<td><span class="hide-on-med-and-up">Database<br></span><strong>{{ database_connection }}</strong></td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -9,24 +9,10 @@
|
|||
{% if media.title %}<h2 class="truncate"><strong>{{ media.title }}</strong></h2>{% endif %}
|
||||
<p class="truncate"><strong><a href="{{ media.url }}" target="_blank"><i class="fas fa-link"></i> {{ media.url }}</a></strong></p>
|
||||
<p class="truncate">Downloading to: <strong>{{ media.source.directory_path }}</strong></p>
|
||||
{% if download_state == 'downloaded' %}
|
||||
{% if media.source.is_audio %}
|
||||
<audio controls src="{% url 'sync:media-content' pk=media.pk %}"></audio>
|
||||
{% else %}
|
||||
<video controls style="width: 100%">
|
||||
<source src="{% url 'sync:media-content' pk=media.pk %}">
|
||||
</video>
|
||||
{% endif %}
|
||||
|
||||
<p class="truncate"><a href="{% url 'sync:media-content' pk=media.pk %}" download="{{ media.filename }}"><strong><i class="fas fa-download"></i> Download</strong></a></p>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% if media.manual_skip %}{% include 'errorbox.html' with message='Media is marked to be skipped and will not be downloaded.' %}
|
||||
{% else %}
|
||||
{% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
|
||||
{% if media.skip %}{% include 'errorbox.html' with message='This media may be skipped due to error(s).' %}{% endif %}
|
||||
{% endif %}
|
||||
{% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %}
|
||||
{% if media.skip %}{% include 'errorbox.html' with message='Media is marked to be skipped and will not be downloaded.' %}{% endif %}
|
||||
{% include 'infobox.html' with message=message %}
|
||||
<div class="row">
|
||||
<div class="col s12 m7">
|
||||
|
|
@ -170,10 +156,10 @@
|
|||
{% else %}
|
||||
<div class="row">
|
||||
<div class="col s12">
|
||||
{% if media.manual_skip %}
|
||||
<a href="{% url 'sync:enable-media' pk=media.pk %}" class="btn">Unskip media (manually) <i class="fas fa-cloud-download-alt"></i></a>
|
||||
{% if media.skip %}
|
||||
<a href="{% url 'sync:enable-media' pk=media.pk %}" class="btn">Enable (unskip) media <i class="fas fa-cloud-download-alt"></i></a>
|
||||
{% else %}
|
||||
<a href="{% url 'sync:skip-media' pk=media.pk %}" class="btn delete-button">Manually mark media to be skipped <i class="fas fa-times-circle"></i></a>
|
||||
<a href="{% url 'sync:skip-media' pk=media.pk %}" class="btn delete-button">Skip media <i class="fas fa-times-circle"></i></a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -4,23 +4,9 @@
|
|||
|
||||
{% block content %}
|
||||
<div class="row">
|
||||
<div class="col s12 m6">
|
||||
<div class="col s12">
|
||||
<h1 class="truncate">Media</h1>
|
||||
</div>
|
||||
<div class="col s12 m3">
|
||||
{% if show_skipped %}
|
||||
<a href="{% url 'sync:media' %}{% if source %}?filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye-slash"></i> Hide skipped media</a>
|
||||
{% else %}
|
||||
<a href="{% url 'sync:media' %}?show_skipped=yes{% if source %}&filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye"></i> Show skipped media</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="col s12 m3">
|
||||
{% if only_skipped %}
|
||||
<a href="{% url 'sync:media' %}{% if source %}?filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye-slash"></i> Only skipped media</a>
|
||||
{% else %}
|
||||
<a href="{% url 'sync:media' %}?only_skipped=yes{% if source %}&filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye"></i> Only skipped media</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% include 'infobox.html' with message=message %}
|
||||
<div class="row no-margin-bottom">
|
||||
|
|
@ -36,10 +22,8 @@
|
|||
{% if m.downloaded %}
|
||||
<i class="fas fa-check-circle" title="Downloaded"></i> {{ m.download_date|date:'Y-m-d' }}
|
||||
{% else %}
|
||||
{% if m.manual_skip %}
|
||||
<span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Manually skipped</span>
|
||||
{% elif m.skip %}
|
||||
<span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped by system</span>
|
||||
{% if m.skip %}
|
||||
<span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped</span>
|
||||
{% elif not m.source.download_media %}
|
||||
<span class="error-text"><i class="fas fa-times" title="Not downloading media for this source"></i> Disabled at source</span>
|
||||
{% elif not m.has_metadata %}
|
||||
|
|
@ -64,5 +48,5 @@
|
|||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% include 'pagination.html' with pagination=sources.paginator filter=source.pk show_skipped=show_skipped only_skipped=only_skipped%}
|
||||
{% include 'pagination.html' with pagination=sources.paginator filter=source.pk %}
|
||||
{% endblock %}
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@
|
|||
<p>
|
||||
Are you sure you want to delete this source? Deleting a source is permanent.
|
||||
By default, deleting a source does not delete any saved media files. You can
|
||||
<strong>tick the "also delete downloaded media" checkbox to also remove directory {{ source.directory_path }}
|
||||
</strong>when you delete the source. Deleting a source cannot be undone.
|
||||
tick the "also delete downloaded media" checkbox to also remove save
|
||||
media when you delete the source. Deleting a source cannot be undone.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -43,10 +43,6 @@
|
|||
<td class="hide-on-small-only">Directory</td>
|
||||
<td><span class="hide-on-med-and-up">Directory<br></span><strong>{{ source.directory }}</strong></td>
|
||||
</tr>
|
||||
<tr title="Filter text">
|
||||
<td class="hide-on-small-only">Filter text</td>
|
||||
<td><span class="hide-on-med-and-up">Filter text<br></span><strong>{{ source.filter_text }}</strong></td>
|
||||
</tr>
|
||||
<tr title="Media file name format to use for saving files">
|
||||
<td class="hide-on-small-only">Media format</td>
|
||||
<td><span class="hide-on-med-and-up">Media format<br></span><strong>{{ source.media_format }}</strong></td>
|
||||
|
|
@ -115,18 +111,6 @@
|
|||
<td class="hide-on-small-only">Write NFO?</td>
|
||||
<td><span class="hide-on-med-and-up">Write NFO?<br></span><strong>{% if source.write_nfo %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
||||
</tr>
|
||||
<tr title="Should a JSON file be written with the media?">
|
||||
<td class="hide-on-small-only">Write JSON?</td>
|
||||
<td><span class="hide-on-med-and-up">Write JSON?<br></span><strong>{% if source.write_json %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
||||
</tr>
|
||||
<tr title="Delete media that is no longer on this playlist?">
|
||||
<td class="hide-on-small-only">Delete removed media</td>
|
||||
<td><span class="hide-on-med-and-up">Delete removed media<br></span><strong>{% if source.delete_removed_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
||||
</tr>
|
||||
<tr title="Delete files on disk when they are removed from TubeSync?">
|
||||
<td class="hide-on-small-only">Delete files on disk</td>
|
||||
<td><span class="hide-on-med-and-up">Delete files on disk<br></span><strong>{% if source.delete_files_on_disk %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
|
||||
</tr>
|
||||
{% if source.delete_old_media and source.days_to_keep > 0 %}
|
||||
<tr title="Days after which your media from this source will be locally deleted">
|
||||
<td class="hide-on-small-only">Delete old media</td>
|
||||
|
|
@ -142,55 +126,6 @@
|
|||
<td class="hide-on-small-only">UUID</td>
|
||||
<td><span class="hide-on-med-and-up">UUID<br></span><strong>{{ source.uuid }}</strong></td>
|
||||
</tr>
|
||||
<tr title="{{ _('Embedding thumbnail?') }}">
|
||||
<td class="hide-on-small-only">{{ _("Embed thumbnail?") }}:</td>
|
||||
<td><span class="hide-on-med-and-up">{{ _("Embed thumbnail?") }}<br></span><strong><i class="fas {% if source.embed_thumbnail %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
|
||||
</tr>
|
||||
<tr title="{{ _('Embedding metadata?') }}">
|
||||
<td class="hide-on-small-only">{{ _("Embed metadata?") }}:</td>
|
||||
<td><span class="hide-on-med-and-up">{{ _("Embed metadata?") }}<br></span><strong><i class="fas {% if source.embed_metadata %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
|
||||
</tr>
|
||||
|
||||
<tr title="{{ _('Is sponsorblock enabled?') }}">
|
||||
<td class="hide-on-small-only">{{ _("SponsorBlock?") }}:</td>
|
||||
<td><span class="hide-on-med-and-up">{{ _("Sponsorblock enabled?") }}<br></span><strong><i class="fas {% if source.enable_sponsorblock %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
|
||||
</tr>
|
||||
|
||||
{% if source.enable_sponsorblock %}
|
||||
<tr title="{{ _('SponsorBlock: What to block?') }}">
|
||||
<td class="hide-on-small-only">{{ _("What blocked?") }}:</td>
|
||||
<td><span class="hide-on-med-and-up">{{ _("What blocked?") }}<br></span><strong>
|
||||
{% if source.sponsorblock_categories.all_choice in source.sponsorblock_categories.selected_choices %}
|
||||
{% for k,v in source.sponsorblock_categories.possible_choices %}
|
||||
{{ v }}: <i class="fas fa-check"></i><BR>
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for c in source.sponsorblock_categories.selected_choices %}
|
||||
{% for k,v in source.sponsorblock_categories.possible_choices %}
|
||||
{% if k == c %} {{ v }}: <i class="fas fa-check"></i><BR>{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
</strong></td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
|
||||
<tr title="{{ _('Are Subtitles downloaded?') }}">
|
||||
<td class="hide-on-small-only">{{ _("Download subtitles?") }}:</td>
|
||||
<td><span class="hide-on-med-and-up">{{ _("Download subtitles?") }}:</span><strong><i class="fas {% if source.write_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
|
||||
</tr>
|
||||
|
||||
{% if source.write_subtitles %}
|
||||
<tr title="{{ _('Are auto subs accepted?') }}">
|
||||
<td class="hide-on-small-only">{{ _("Auto-generated subtitles?") }}:</td>
|
||||
<td><span class="hide-on-med-and-up">{{ _("Auto-generated subtitles?") }}:</span><strong><i class="fas {% if source.auto_subtitles %}fa-check{% else %}fa-times{% endif %}"></i></strong></td>
|
||||
</tr>
|
||||
<tr title="{{ _('Subs langs?') }}">
|
||||
<td class="hide-on-small-only">{{ _("Subs langs?") }}:</td>
|
||||
<td><span class="hide-on-med-and-up">{{ _("Subs langs?") }}:</span><strong>{{source.sub_langs}}</strong></td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -24,18 +24,15 @@
|
|||
<div class="col s12">
|
||||
<div class="collection">
|
||||
{% for source in sources %}
|
||||
<span class="collection-item flex-collection-container">
|
||||
<a href="{% url 'sync:source' pk=source.pk %}" class="flex-grow">
|
||||
<a href="{% url 'sync:source' pk=source.pk %}" class="collection-item">
|
||||
{{ source.icon|safe }} <strong>{{ source.name }}</strong> ({{ source.get_source_type_display }} "{{ source.key }}")<br>
|
||||
{{ source.format_summary }}<br>
|
||||
{% if source.has_failed %}
|
||||
<span class="error-text"><i class="fas fa-exclamation-triangle"></i> <strong>Source has permanent failures</strong></span>
|
||||
{% else %}
|
||||
<strong>{{ source.media_count }}</strong> media items, <strong>{{ source.downloaded_count }}</strong> downloaded{% if source.delete_old_media and source.days_to_keep > 0 %}, keeping {{ source.days_to_keep }} days of media{% endif %}
|
||||
<strong>{{ source.media_count }}</strong> media items{% if source.delete_old_media and source.days_to_keep > 0 %}, keep {{ source.days_to_keep }} days of media{% endif %}
|
||||
{% endif %}
|
||||
</a>
|
||||
<a href="{% url 'sync:source-sync-now' pk=source.pk %}" class="collection-item"><i class="fas fa-arrow-rotate-right"></i></a>
|
||||
</span>
|
||||
{% empty %}
|
||||
<span class="collection-item no-items"><i class="fas fa-info-circle"></i> You haven't added any sources.</span>
|
||||
{% endfor %}
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@
|
|||
{% for task in scheduled %}
|
||||
<a href="{% url task.url pk=task.instance.pk %}" class="collection-item">
|
||||
<i class="fas fa-stopwatch"></i> <strong>{{ task }}</strong><br>
|
||||
{% if task.instance.index_schedule and task.repeat > 0 %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
|
||||
{% if task.instance.index_schedule %}Scheduled to run {{ task.instance.get_index_schedule_display|lower }}.<br>{% endif %}
|
||||
<i class="fas fa-redo"></i> Task will run {% if task.run_now %}<strong>immediately</strong>{% else %}at <strong>{{ task.run_at|date:'Y-m-d H:i:s' }}</strong>{% endif %}
|
||||
</a>
|
||||
{% empty %}
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
<!--<input type="{{ option.type }}" name="{{ option.name }}" value="{{ option.value }}" id="{{ option.value }}"><BR>
|
||||
<label for="{{ option.value }}">{{option.label}}</label>-->
|
||||
|
||||
<label>
|
||||
<input type="{{ option.type }}" name="{{ option.name }}" value="{{ option.value }}" id="{{ option.value }}" {% if option.checked %}checked{% endif %}>
|
||||
<span>{{option.label}}</span>
|
||||
</label>
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
</label>
|
||||
{% for option in widget.multipleChoiceProperties %}
|
||||
{% include option.template_name with option=option %}
|
||||
{% endfor %}
|
||||
<label>
|
||||
|
|
@ -9,6 +9,7 @@
|
|||
"average_rating": 1.2345,
|
||||
"dislike_count": 123,
|
||||
"like_count": 456,
|
||||
"playlist_index": 789,
|
||||
"playlist_title": "test playlist",
|
||||
"uploader": "test uploader",
|
||||
"categories":[
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlsplit
|
||||
from xml.etree import ElementTree
|
||||
from django.conf import settings
|
||||
|
|
@ -14,7 +14,6 @@ from django.test import TestCase, Client
|
|||
from django.utils import timezone
|
||||
from background_task.models import Task
|
||||
from .models import Source, Media
|
||||
from .tasks import cleanup_old_media
|
||||
|
||||
|
||||
class FrontEndTestCase(TestCase):
|
||||
|
|
@ -37,9 +36,6 @@ class FrontEndTestCase(TestCase):
|
|||
test_sources = {
|
||||
'youtube-channel': {
|
||||
'valid': (
|
||||
'https://m.youtube.com/testchannel',
|
||||
'https://m.youtube.com/c/testchannel',
|
||||
'https://m.youtube.com/c/testchannel/videos',
|
||||
'https://www.youtube.com/testchannel',
|
||||
'https://www.youtube.com/c/testchannel',
|
||||
'https://www.youtube.com/c/testchannel/videos',
|
||||
|
|
@ -51,7 +47,6 @@ class FrontEndTestCase(TestCase):
|
|||
'invalid_domain': (
|
||||
'https://www.test.com/c/testchannel',
|
||||
'https://www.example.com/c/testchannel',
|
||||
'https://n.youtube.com/c/testchannel',
|
||||
),
|
||||
'invalid_path': (
|
||||
'https://www.youtube.com/test/invalid',
|
||||
|
|
@ -67,8 +62,6 @@ class FrontEndTestCase(TestCase):
|
|||
},
|
||||
'youtube-channel-id': {
|
||||
'valid': (
|
||||
'https://m.youtube.com/channel/channelid',
|
||||
'https://m.youtube.com/channel/channelid/videos',
|
||||
'https://www.youtube.com/channel/channelid',
|
||||
'https://www.youtube.com/channel/channelid/videos',
|
||||
),
|
||||
|
|
@ -79,7 +72,6 @@ class FrontEndTestCase(TestCase):
|
|||
'invalid_domain': (
|
||||
'https://www.test.com/channel/channelid',
|
||||
'https://www.example.com/channel/channelid',
|
||||
'https://n.youtube.com/channel/channelid',
|
||||
),
|
||||
'invalid_path': (
|
||||
'https://www.youtube.com/test/invalid',
|
||||
|
|
@ -91,8 +83,6 @@ class FrontEndTestCase(TestCase):
|
|||
},
|
||||
'youtube-playlist': {
|
||||
'valid': (
|
||||
'https://m.youtube.com/playlist?list=testplaylist',
|
||||
'https://m.youtube.com/watch?v=testvideo&list=testplaylist',
|
||||
'https://www.youtube.com/playlist?list=testplaylist',
|
||||
'https://www.youtube.com/watch?v=testvideo&list=testplaylist',
|
||||
),
|
||||
|
|
@ -103,7 +93,6 @@ class FrontEndTestCase(TestCase):
|
|||
'invalid_domain': (
|
||||
'https://www.test.com/playlist?list=testplaylist',
|
||||
'https://www.example.com/playlist?list=testplaylist',
|
||||
'https://n.youtube.com/playlist?list=testplaylist',
|
||||
),
|
||||
'invalid_path': (
|
||||
'https://www.youtube.com/notplaylist?list=testplaylist',
|
||||
|
|
@ -175,7 +164,6 @@ class FrontEndTestCase(TestCase):
|
|||
'directory': 'testdirectory',
|
||||
'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
|
||||
'download_cap': 0,
|
||||
'filter_text':'.*',
|
||||
'index_schedule': 3600,
|
||||
'delete_old_media': False,
|
||||
'days_to_keep': 14,
|
||||
|
|
@ -184,8 +172,7 @@ class FrontEndTestCase(TestCase):
|
|||
'source_acodec': 'OPUS',
|
||||
'prefer_60fps': False,
|
||||
'prefer_hdr': False,
|
||||
'fallback': 'f',
|
||||
'sub_langs': 'en',
|
||||
'fallback': 'f'
|
||||
}
|
||||
response = c.post('/source-add', data)
|
||||
self.assertEqual(response.status_code, 302)
|
||||
|
|
@ -218,7 +205,6 @@ class FrontEndTestCase(TestCase):
|
|||
'directory': 'testdirectory',
|
||||
'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
|
||||
'download_cap': 0,
|
||||
'filter_text':'.*',
|
||||
'index_schedule': Source.IndexSchedule.EVERY_HOUR,
|
||||
'delete_old_media': False,
|
||||
'days_to_keep': 14,
|
||||
|
|
@ -227,8 +213,7 @@ class FrontEndTestCase(TestCase):
|
|||
'source_acodec': Source.SOURCE_ACODEC_OPUS,
|
||||
'prefer_60fps': False,
|
||||
'prefer_hdr': False,
|
||||
'fallback': Source.FALLBACK_FAIL,
|
||||
'sub_langs': 'en',
|
||||
'fallback': Source.FALLBACK_FAIL
|
||||
}
|
||||
response = c.post(f'/source-update/{source_uuid}', data)
|
||||
self.assertEqual(response.status_code, 302)
|
||||
|
|
@ -249,7 +234,6 @@ class FrontEndTestCase(TestCase):
|
|||
'directory': 'testdirectory',
|
||||
'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
|
||||
'download_cap': 0,
|
||||
'filter_text':'.*',
|
||||
'index_schedule': Source.IndexSchedule.EVERY_2_HOURS, # changed
|
||||
'delete_old_media': False,
|
||||
'days_to_keep': 14,
|
||||
|
|
@ -258,8 +242,7 @@ class FrontEndTestCase(TestCase):
|
|||
'source_acodec': Source.SOURCE_ACODEC_OPUS,
|
||||
'prefer_60fps': False,
|
||||
'prefer_hdr': False,
|
||||
'fallback': Source.FALLBACK_FAIL,
|
||||
'sub_langs': 'en',
|
||||
'fallback': Source.FALLBACK_FAIL
|
||||
}
|
||||
response = c.post(f'/source-update/{source_uuid}', data)
|
||||
self.assertEqual(response.status_code, 302)
|
||||
|
|
@ -361,25 +344,21 @@ class FrontEndTestCase(TestCase):
|
|||
}]
|
||||
}
|
||||
'''
|
||||
past_date = timezone.make_aware(datetime(year=2000, month=1, day=1))
|
||||
test_media1 = Media.objects.create(
|
||||
key='mediakey1',
|
||||
source=test_source,
|
||||
published=past_date,
|
||||
metadata=test_minimal_metadata
|
||||
)
|
||||
test_media1_pk = str(test_media1.pk)
|
||||
test_media2 = Media.objects.create(
|
||||
key='mediakey2',
|
||||
source=test_source,
|
||||
published=past_date,
|
||||
metadata=test_minimal_metadata
|
||||
)
|
||||
test_media2_pk = str(test_media2.pk)
|
||||
test_media3 = Media.objects.create(
|
||||
key='mediakey3',
|
||||
source=test_source,
|
||||
published=past_date,
|
||||
metadata=test_minimal_metadata
|
||||
)
|
||||
test_media3_pk = str(test_media3.pk)
|
||||
|
|
@ -471,14 +450,11 @@ metadata_60fps_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_60
|
|||
metadata_60fps = open(metadata_60fps_filepath, 'rt').read()
|
||||
metadata_60fps_hdr_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_60fps_hdr.json'
|
||||
metadata_60fps_hdr = open(metadata_60fps_hdr_filepath, 'rt').read()
|
||||
metadata_20230629_filepath = settings.BASE_DIR / 'sync' / 'testdata' / 'metadata_2023-06-29.json'
|
||||
metadata_20230629 = open(metadata_20230629_filepath, 'rt').read()
|
||||
all_test_metadata = {
|
||||
'boring': metadata,
|
||||
'hdr': metadata_hdr,
|
||||
'60fps': metadata_60fps,
|
||||
'60fps+hdr': metadata_60fps_hdr,
|
||||
'20230629': metadata_20230629,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -511,7 +487,7 @@ class FilepathTestCase(TestCase):
|
|||
metadata=metadata,
|
||||
)
|
||||
|
||||
def test_source_media_format(self):
|
||||
def test_source_dirname(self):
|
||||
# Check media format validation is working
|
||||
# Empty
|
||||
self.source.media_format = ''
|
||||
|
|
@ -559,6 +535,9 @@ class FilepathTestCase(TestCase):
|
|||
self.source.media_format = 'test-{format}'
|
||||
self.assertEqual(self.source.get_example_media_format(),
|
||||
'test-1080p-vp9-opus')
|
||||
self.source.media_format = 'test-{playlist_index}'
|
||||
self.assertEqual(self.source.get_example_media_format(),
|
||||
'test-1')
|
||||
self.source.media_format = 'test-{playlist_title}'
|
||||
self.assertEqual(self.source.get_example_media_format(),
|
||||
'test-Some Playlist Title')
|
||||
|
|
@ -661,8 +640,6 @@ class MediaTestCase(TestCase):
|
|||
'<episodedetails>',
|
||||
' <title>no fancy stuff title</title>',
|
||||
' <showtitle>testname</showtitle>',
|
||||
' <season>2017</season>',
|
||||
' <episode></episode>',
|
||||
' <ratings>',
|
||||
' <rating default="True" max="5" name="youtube">',
|
||||
' <value>1.2345</value>',
|
||||
|
|
@ -1184,14 +1161,14 @@ class FormatMatchingTestCase(TestCase):
|
|||
('2160p', 'VP9', False, True): (True, '337'), # Exact match, hdr
|
||||
('2160p', 'VP9', True, False): (True, '315'), # Exact match, 60fps
|
||||
('2160p', 'VP9', True, True): (True, '337'), # Exact match, 60fps+hdr
|
||||
('4320p', 'AVC1', False, False): (False, False),
|
||||
('4320p', 'AVC1', False, True): (False, False),
|
||||
('4320p', 'AVC1', True, False): (False, False),
|
||||
('4320p', 'AVC1', True, True): (False, False),
|
||||
('4320p', 'VP9', False, False): (False, False),
|
||||
('4320p', 'VP9', False, True): (False, False),
|
||||
('4320p', 'VP9', True, False): (True, '272'), # Exact match, 60fps
|
||||
('4320p', 'VP9', True, True): (False, False),
|
||||
('4320P', 'AVC1', False, False): (False, False),
|
||||
('4320P', 'AVC1', False, True): (False, False),
|
||||
('4320P', 'AVC1', True, False): (False, False),
|
||||
('4320P', 'AVC1', True, True): (False, False),
|
||||
('4320P', 'VP9', False, False): (False, False),
|
||||
('4320P', 'VP9', False, True): (False, False),
|
||||
('4320P', 'VP9', True, False): (True, '272'), # Exact match, 60fps
|
||||
('4320P', 'VP9', True, True): (False, False),
|
||||
}
|
||||
for params, expected in expected_matches.items():
|
||||
resolution, vcodec, prefer_60fps, prefer_hdr = params
|
||||
|
|
@ -1390,14 +1367,14 @@ class FormatMatchingTestCase(TestCase):
|
|||
('2160p', 'VP9', False, True): (True, '337'), # Exact match, hdr
|
||||
('2160p', 'VP9', True, False): (True, '315'), # Exact match, 60fps
|
||||
('2160p', 'VP9', True, True): (True, '337'), # Exact match, 60fps+hdr
|
||||
('4320p', 'AVC1', False, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
|
||||
('4320p', 'AVC1', False, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
|
||||
('4320p', 'AVC1', True, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
|
||||
('4320p', 'AVC1', True, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
|
||||
('4320p', 'VP9', False, False): (False, '272'), # Fallback, 60fps (no other 8k streams)
|
||||
('4320p', 'VP9', False, True): (False, '272'), # Fallback, 60fps (no other 8k streams)
|
||||
('4320p', 'VP9', True, False): (True, '272'), # Exact match, 60fps
|
||||
('4320p', 'VP9', True, True): (False, '272'), # Fallback, 60fps (no other 8k streams)
|
||||
('4320P', 'AVC1', False, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
|
||||
('4320P', 'AVC1', False, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
|
||||
('4320P', 'AVC1', True, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
|
||||
('4320P', 'AVC1', True, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320P AVC1, no other 8k streams)
|
||||
('4320P', 'VP9', False, False): (False, '272'), # Fallback, 60fps (no other 8k streams)
|
||||
('4320P', 'VP9', False, True): (False, '272'), # Fallback, 60fps (no other 8k streams)
|
||||
('4320P', 'VP9', True, False): (True, '272'), # Exact match, 60fps
|
||||
('4320P', 'VP9', True, True): (False, '272'), # Fallback, 60fps (no other 8k streams)
|
||||
}
|
||||
for params, expected in expected_matches.items():
|
||||
resolution, vcodec, prefer_60fps, prefer_hdr = params
|
||||
|
|
@ -1409,118 +1386,3 @@ class FormatMatchingTestCase(TestCase):
|
|||
match_type, format_code = self.media.get_best_video_format()
|
||||
self.assertEqual(format_code, expected_format_code)
|
||||
self.assertEqual(match_type, expeceted_match_type)
|
||||
|
||||
def test_metadata_20230629(self):
|
||||
self.media.metadata = all_test_metadata['20230629']
|
||||
expected_matches = {
|
||||
# (format, vcodec, prefer_60fps, prefer_hdr): (match_type, code),
|
||||
('360p', 'AVC1', False, True): (False, '134'), # Fallback match, no hdr
|
||||
('360p', 'AVC1', True, False): (False, '134'), # Fallback match, no 60fps
|
||||
('360p', 'AVC1', True, True): (False, '332'), # Fallback match, 60fps+hdr, switched to VP9
|
||||
('360p', 'VP9', False, False): (True, '243'), # Exact match
|
||||
('360p', 'VP9', False, True): (True, '332'), # Exact match, hdr
|
||||
('360p', 'VP9', True, False): (False, '332'), # Fallback match, 60fps, extra hdr
|
||||
('360p', 'VP9', True, True): (True, '332'), # Exact match, 60fps+hdr
|
||||
('480p', 'AVC1', False, False): (True, '135'), # Exact match
|
||||
('480p', 'AVC1', False, True): (False, '135'), # Fallback match, no hdr
|
||||
('480p', 'AVC1', True, False): (False, '135'), # Fallback match, no 60fps
|
||||
('480p', 'AVC1', True, True): (False, '333'), # Fallback match, 60fps+hdr, switched to VP9
|
||||
('480p', 'VP9', False, False): (True, '244'), # Exact match
|
||||
('480p', 'VP9', False, True): (True, '333'), # Exact match, hdr
|
||||
('480p', 'VP9', True, False): (False, '333'), # Fallback match, 60fps, extra hdr
|
||||
('480p', 'VP9', True, True): (True, '333'), # Exact match, 60fps+hdr
|
||||
('720p', 'AVC1', False, False): (True, '136'), # Exact match
|
||||
('720p', 'AVC1', False, True): (False, '136'), # Fallback match, no hdr
|
||||
('720p', 'AVC1', True, False): (True, '298'), # Exact match, 60fps
|
||||
('720p', 'AVC1', True, True): (False, '334'), # Fallback match, 60fps+hdr, switched to VP9
|
||||
('720p', 'VP9', False, False): (True, '247'), # Exact match
|
||||
('720p', 'VP9', False, True): (True, '334'), # Exact match, hdr
|
||||
('720p', 'VP9', True, False): (True, '302'), # Exact match, 60fps
|
||||
('720p', 'VP9', True, True): (True, '334'), # Exact match, 60fps+hdr
|
||||
('1440p', 'AVC1', False, False): (False, '308'), # Fallback match, 60fps, switched to VP9 (no 1440p AVC1)
|
||||
('1440p', 'AVC1', False, True): (False, '336'), # Fallback match, 60fps+hdr, switched to VP9 (no 1440p AVC1)
|
||||
('1440p', 'AVC1', True, False): (False, '308'), # Fallback match, 60fps, switched to VP9 (no 1440p AVC1)
|
||||
('1440p', 'AVC1', True, True): (False, '336'), # Fallback match, 60fps+hdr, switched to VP9 (no 1440p AVC1)
|
||||
('1440p', 'VP9', False, False): (False, '308'), # Fallback, 60fps
|
||||
('1440p', 'VP9', False, True): (True, '336'), # Exact match, hdr
|
||||
('1440p', 'VP9', True, False): (True, '308'), # Exact match, 60fps
|
||||
('1440p', 'VP9', True, True): (True, '336'), # Exact match, 60fps+hdr
|
||||
('2160p', 'AVC1', False, False): (False, '315'), # Fallback, 60fps, switched to VP9 (no 2160p AVC1)
|
||||
('2160p', 'AVC1', False, True): (False, '337'), # Fallback match, 60fps+hdr, switched to VP9 (no 2160p AVC1)
|
||||
('2160p', 'AVC1', True, False): (False, '315'), # Fallback, switched to VP9 (no 2160p AVC1)
|
||||
('2160p', 'AVC1', True, True): (False, '337'), # Fallback match, 60fps+hdr, switched to VP9 (no 2160p AVC1)
|
||||
('2160p', 'VP9', False, False): (False, '315'), # Fallback, 60fps
|
||||
('2160p', 'VP9', False, True): (True, '337'), # Exact match, hdr
|
||||
('2160p', 'VP9', True, False): (True, '315'), # Exact match, 60fps
|
||||
('2160p', 'VP9', True, True): (True, '337'), # Exact match, 60fps+hdr
|
||||
('4320p', 'AVC1', False, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
|
||||
('4320p', 'AVC1', False, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
|
||||
('4320p', 'AVC1', True, False): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
|
||||
('4320p', 'AVC1', True, True): (False, '272'), # Fallback, 60fps, switched to VP9 (no 4320p AVC1, no other 8k streams)
|
||||
('4320p', 'VP9', False, False): (False, '272'), # Fallback, 60fps (no other 8k streams)
|
||||
('4320p', 'VP9', False, True): (False, '272'), # Fallback, 60fps (no other 8k streams)
|
||||
('4320p', 'VP9', True, False): (True, '272'), # Exact match, 60fps
|
||||
('4320p', 'VP9', True, True): (False, '272'), # Fallback, 60fps (no other 8k streams)
|
||||
}
|
||||
for params, expected in expected_matches.items():
|
||||
resolution, vcodec, prefer_60fps, prefer_hdr = params
|
||||
expeceted_match_type, expected_format_code = expected
|
||||
self.source.source_resolution = resolution
|
||||
self.source.source_vcodec = vcodec
|
||||
self.source.prefer_60fps = prefer_60fps
|
||||
self.source.prefer_hdr = prefer_hdr
|
||||
# The aim here is to execute the matching code to find error paths, specific testing isn't required
|
||||
self.media.get_best_video_format()
|
||||
self.media.get_best_audio_format()
|
||||
|
||||
def test_is_regex_match(self):
|
||||
|
||||
self.media.metadata = all_test_metadata['boring']
|
||||
expected_matches = {
|
||||
('.*'): (True),
|
||||
('no fancy stuff'): (True),
|
||||
('No fancy stuff'): (False),
|
||||
('(?i)No fancy stuff'): (True), #set case insensitive flag
|
||||
('no'): (True),
|
||||
('Foo'): (False),
|
||||
('^(?!.*fancy).*$'): (False),
|
||||
('^(?!.*funny).*$'): (True),
|
||||
('(?=.*f.*)(?=.{0,2}|.{4,})'): (True),
|
||||
('f{4,}'): (False),
|
||||
('^[^A-Z]*$'): (True),
|
||||
('^[^a-z]*$'): (False),
|
||||
('^[^\\s]*$'): (False)
|
||||
}
|
||||
|
||||
for params, expected in expected_matches.items():
|
||||
self.source.filter_text = params
|
||||
expected_match_result = expected
|
||||
self.assertEqual(self.source.is_regex_match(self.media.title), expected_match_result)
|
||||
|
||||
class TasksTestCase(TestCase):
|
||||
def setUp(self):
|
||||
# Disable general logging for test case
|
||||
logging.disable(logging.CRITICAL)
|
||||
|
||||
def test_delete_old_media(self):
|
||||
src1 = Source.objects.create(key='aaa', name='aaa', directory='/tmp/a', delete_old_media=False, days_to_keep=14)
|
||||
src2 = Source.objects.create(key='bbb', name='bbb', directory='/tmp/b', delete_old_media=True, days_to_keep=14)
|
||||
|
||||
now = timezone.now()
|
||||
|
||||
m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5))
|
||||
m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25))
|
||||
m13 = Media.objects.create(source=src1, downloaded=False, key='a13')
|
||||
|
||||
m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5))
|
||||
m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25))
|
||||
m23 = Media.objects.create(source=src2, downloaded=False, key='a23')
|
||||
|
||||
self.assertEquals(src1.media_source.all().count(), 3)
|
||||
self.assertEquals(src2.media_source.all().count(), 3)
|
||||
|
||||
cleanup_old_media()
|
||||
|
||||
self.assertEquals(src1.media_source.all().count(), 3)
|
||||
self.assertEquals(src2.media_source.all().count(), 2)
|
||||
self.assertEquals(Media.objects.filter(pk=m22.pk).exists(), False)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from django.urls import path
|
|||
from .views import (DashboardView, SourcesView, ValidateSourceView, AddSourceView,
|
||||
SourceView, UpdateSourceView, DeleteSourceView, MediaView,
|
||||
MediaThumbView, MediaItemView, MediaRedownloadView, MediaSkipView,
|
||||
MediaEnableView, MediaContent, TasksView, CompletedTasksView, ResetTasks,
|
||||
MediaEnableView, TasksView, CompletedTasksView, ResetTasks,
|
||||
MediaServersView, AddMediaServerView, MediaServerView,
|
||||
DeleteMediaServerView, UpdateMediaServerView)
|
||||
|
||||
|
|
@ -28,10 +28,6 @@ urlpatterns = [
|
|||
ValidateSourceView.as_view(),
|
||||
name='validate-source'),
|
||||
|
||||
path('source-sync-now/<uuid:pk>',
|
||||
SourcesView.as_view(),
|
||||
name='source-sync-now'),
|
||||
|
||||
path('source-add',
|
||||
AddSourceView.as_view(),
|
||||
name='add-source'),
|
||||
|
|
@ -74,10 +70,6 @@ urlpatterns = [
|
|||
MediaEnableView.as_view(),
|
||||
name='enable-media'),
|
||||
|
||||
path('media-content/<uuid:pk>',
|
||||
MediaContent.as_view(),
|
||||
name='media-content'),
|
||||
|
||||
# Task URLs
|
||||
|
||||
path('tasks',
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ def validate_url(url, validator):
|
|||
Validate a URL against a dict of validation requirements. Returns an extracted
|
||||
part of the URL if the URL is valid, if invalid raises a ValidationError.
|
||||
'''
|
||||
valid_scheme, valid_netlocs, valid_path, invalid_paths, valid_query, \
|
||||
valid_scheme, valid_netloc, valid_path, invalid_paths, valid_query, \
|
||||
extract_parts = (
|
||||
validator['scheme'], validator['domains'], validator['path_regex'],
|
||||
validator['scheme'], validator['domain'], validator['path_regex'],
|
||||
validator['path_must_not_match'], validator['qs_args'],
|
||||
validator['extract_key']
|
||||
)
|
||||
|
|
@ -25,8 +25,8 @@ def validate_url(url, validator):
|
|||
if url_scheme != valid_scheme:
|
||||
raise ValidationError(f'invalid scheme "{url_scheme}" must be "{valid_scheme}"')
|
||||
url_netloc = str(url_parts.netloc).strip().lower()
|
||||
if url_netloc not in valid_netlocs:
|
||||
raise ValidationError(f'invalid domain "{url_netloc}" must be one of "{valid_netlocs}"')
|
||||
if url_netloc != valid_netloc:
|
||||
raise ValidationError(f'invalid domain "{url_netloc}" must be "{valid_netloc}"')
|
||||
url_path = str(url_parts.path).strip()
|
||||
matches = re.findall(valid_path, url_path)
|
||||
if not matches:
|
||||
|
|
@ -78,7 +78,7 @@ def resize_image_to_height(image, width, height):
|
|||
if scaled_width < width:
|
||||
# Width too small, stretch it
|
||||
scaled_width = width
|
||||
image = image.resize((scaled_width, height), Image.LANCZOS)
|
||||
image = image.resize((scaled_width, height), Image.ANTIALIAS)
|
||||
if scaled_width > width:
|
||||
# Width too large, crop it
|
||||
delta = scaled_width - width
|
||||
|
|
|
|||
|
|
@ -1,24 +1,18 @@
|
|||
import glob
|
||||
import os
|
||||
import json
|
||||
from base64 import b64decode
|
||||
import pathlib
|
||||
import shutil
|
||||
import sys
|
||||
from django.conf import settings
|
||||
from django.http import FileResponse, Http404, HttpResponseNotFound, HttpResponseRedirect
|
||||
from django.http import Http404
|
||||
from django.views.generic import TemplateView, ListView, DetailView
|
||||
from django.views.generic.edit import (FormView, FormMixin, CreateView, UpdateView,
|
||||
DeleteView)
|
||||
from django.views.generic.detail import SingleObjectMixin
|
||||
from django.core.exceptions import SuspiciousFileOperation
|
||||
from django.http import HttpResponse
|
||||
from django.urls import reverse_lazy
|
||||
from django.db import IntegrityError
|
||||
from django.db.models import Q, Count, Sum, When, Case
|
||||
from django.forms import Form, ValidationError
|
||||
from django.db.models import Q, Count, Sum
|
||||
from django.forms import ValidationError
|
||||
from django.utils.text import slugify
|
||||
from django.utils._os import safe_join
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from common.utils import append_uri_params
|
||||
|
|
@ -61,7 +55,7 @@ class DashboardView(TemplateView):
|
|||
# Disk usage
|
||||
disk_usage = Media.objects.filter(
|
||||
downloaded=True, downloaded_filesize__isnull=False
|
||||
).defer('metadata').aggregate(Sum('downloaded_filesize'))
|
||||
).aggregate(Sum('downloaded_filesize'))
|
||||
data['disk_usage_bytes'] = disk_usage['downloaded_filesize__sum']
|
||||
if not data['disk_usage_bytes']:
|
||||
data['disk_usage_bytes'] = 0
|
||||
|
|
@ -72,19 +66,18 @@ class DashboardView(TemplateView):
|
|||
data['average_bytes_per_media'] = 0
|
||||
# Latest downloads
|
||||
data['latest_downloads'] = Media.objects.filter(
|
||||
downloaded=True, downloaded_filesize__isnull=False
|
||||
).defer('metadata').order_by('-download_date')[:10]
|
||||
downloaded=True
|
||||
).order_by('-download_date')[:10]
|
||||
# Largest downloads
|
||||
data['largest_downloads'] = Media.objects.filter(
|
||||
downloaded=True, downloaded_filesize__isnull=False
|
||||
).defer('metadata').order_by('-downloaded_filesize')[:10]
|
||||
).order_by('-downloaded_filesize')[:10]
|
||||
# UID and GID
|
||||
data['uid'] = os.getuid()
|
||||
data['gid'] = os.getgid()
|
||||
# Config and download locations
|
||||
data['config_dir'] = str(settings.CONFIG_BASE_DIR)
|
||||
data['downloads_dir'] = str(settings.DOWNLOAD_ROOT)
|
||||
data['database_connection'] = settings.DATABASE_CONNECTION_STR
|
||||
return data
|
||||
|
||||
|
||||
|
|
@ -98,27 +91,8 @@ class SourcesView(ListView):
|
|||
paginate_by = settings.SOURCES_PER_PAGE
|
||||
messages = {
|
||||
'source-deleted': _('Your selected source has been deleted.'),
|
||||
'source-refreshed': _('The source has been scheduled to be synced now.')
|
||||
}
|
||||
|
||||
def get(self, *args, **kwargs):
|
||||
if args[0].path.startswith("/source-sync-now/"):
|
||||
sobj = Source.objects.get(pk=kwargs["pk"])
|
||||
if sobj is None:
|
||||
return HttpResponseNotFound()
|
||||
|
||||
verbose_name = _('Index media from source "{}" once')
|
||||
index_source_task(
|
||||
str(sobj.pk),
|
||||
queue=str(sobj.pk),
|
||||
repeat=0,
|
||||
verbose_name=verbose_name.format(sobj.name))
|
||||
url = reverse_lazy('sync:sources')
|
||||
url = append_uri_params(url, {'message': 'source-refreshed'})
|
||||
return HttpResponseRedirect(url)
|
||||
else:
|
||||
return super().get(self, *args, **kwargs)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.message = None
|
||||
super().__init__(*args, **kwargs)
|
||||
|
|
@ -130,10 +104,7 @@ class SourcesView(ListView):
|
|||
|
||||
def get_queryset(self):
|
||||
all_sources = Source.objects.all().order_by('name')
|
||||
return all_sources.annotate(
|
||||
media_count=Count('media_source'),
|
||||
downloaded_count=Count(Case(When(media_source__downloaded=True, then=1)))
|
||||
)
|
||||
return all_sources.annotate(media_count=Count('media_source'))
|
||||
|
||||
def get_context_data(self, *args, **kwargs):
|
||||
data = super().get_context_data(*args, **kwargs)
|
||||
|
|
@ -196,7 +167,7 @@ class ValidateSourceView(FormView):
|
|||
validation_urls = {
|
||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: {
|
||||
'scheme': 'https',
|
||||
'domains': ('m.youtube.com', 'www.youtube.com'),
|
||||
'domain': 'www.youtube.com',
|
||||
'path_regex': '^\/(c\/)?([^\/]+)(\/videos)?$',
|
||||
'path_must_not_match': ('/playlist', '/c/playlist'),
|
||||
'qs_args': [],
|
||||
|
|
@ -205,7 +176,7 @@ class ValidateSourceView(FormView):
|
|||
},
|
||||
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: {
|
||||
'scheme': 'https',
|
||||
'domains': ('m.youtube.com', 'www.youtube.com'),
|
||||
'domain': 'www.youtube.com',
|
||||
'path_regex': '^\/channel\/([^\/]+)(\/videos)?$',
|
||||
'path_must_not_match': ('/playlist', '/c/playlist'),
|
||||
'qs_args': [],
|
||||
|
|
@ -214,7 +185,7 @@ class ValidateSourceView(FormView):
|
|||
},
|
||||
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: {
|
||||
'scheme': 'https',
|
||||
'domains': ('m.youtube.com', 'www.youtube.com'),
|
||||
'domain': 'www.youtube.com',
|
||||
'path_regex': '^\/(playlist|watch)$',
|
||||
'path_must_not_match': (),
|
||||
'qs_args': ('list',),
|
||||
|
|
@ -294,58 +265,23 @@ class ValidateSourceView(FormView):
|
|||
return append_uri_params(url, fields)
|
||||
|
||||
|
||||
class EditSourceMixin:
|
||||
model = Source
|
||||
fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'media_format',
|
||||
'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
|
||||
'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution',
|
||||
'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback',
|
||||
'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
|
||||
'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
|
||||
'auto_subtitles', 'sub_langs')
|
||||
errors = {
|
||||
'invalid_media_format': _('Invalid media format, the media format contains '
|
||||
'errors or is empty. Check the table at the end of '
|
||||
'this page for valid media name variables'),
|
||||
'dir_outside_dlroot': _('You cannot specify a directory outside of the '
|
||||
'base directory (%BASEDIR%)')
|
||||
}
|
||||
|
||||
def form_valid(self, form: Form):
|
||||
# Perform extra validation to make sure the media_format is valid
|
||||
obj = form.save(commit=False)
|
||||
source_type = form.cleaned_data['media_format']
|
||||
example_media_file = obj.get_example_media_format()
|
||||
|
||||
if example_media_file == '':
|
||||
form.add_error(
|
||||
'media_format',
|
||||
ValidationError(self.errors['invalid_media_format'])
|
||||
)
|
||||
|
||||
# Check for suspicious file path(s)
|
||||
try:
|
||||
targetCheck = form.cleaned_data['directory']+"/.virt"
|
||||
newdir = safe_join(settings.DOWNLOAD_ROOT,targetCheck)
|
||||
except SuspiciousFileOperation:
|
||||
form.add_error(
|
||||
'directory',
|
||||
ValidationError(self.errors['dir_outside_dlroot'].replace("%BASEDIR%",str(settings.DOWNLOAD_ROOT)))
|
||||
)
|
||||
|
||||
if form.errors:
|
||||
return super().form_invalid(form)
|
||||
|
||||
return super().form_valid(form)
|
||||
|
||||
|
||||
class AddSourceView(EditSourceMixin, CreateView):
|
||||
class AddSourceView(CreateView):
|
||||
'''
|
||||
Adds a new source, optionally takes some initial data querystring values to
|
||||
prepopulate some of the more unclear values.
|
||||
'''
|
||||
|
||||
template_name = 'sync/source-add.html'
|
||||
model = Source
|
||||
fields = ('source_type', 'key', 'name', 'directory', 'media_format',
|
||||
'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
|
||||
'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
|
||||
'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
|
||||
errors = {
|
||||
'invalid_media_format': _('Invalid media format, the media format contains '
|
||||
'errors or is empty. Check the table at the end of '
|
||||
'this page for valid media name variables'),
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.prepopulated_data = {}
|
||||
|
|
@ -372,6 +308,20 @@ class AddSourceView(EditSourceMixin, CreateView):
|
|||
initial[k] = v
|
||||
return initial
|
||||
|
||||
def form_valid(self, form):
|
||||
# Perform extra validation to make sure the media_format is valid
|
||||
obj = form.save(commit=False)
|
||||
source_type = form.cleaned_data['media_format']
|
||||
example_media_file = obj.get_example_media_format()
|
||||
if example_media_file == '':
|
||||
form.add_error(
|
||||
'media_format',
|
||||
ValidationError(self.errors['invalid_media_format'])
|
||||
)
|
||||
if form.errors:
|
||||
return super().form_invalid(form)
|
||||
return super().form_valid(form)
|
||||
|
||||
def get_success_url(self):
|
||||
url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
|
||||
return append_uri_params(url, {'message': 'source-created'})
|
||||
|
|
@ -406,13 +356,37 @@ class SourceView(DetailView):
|
|||
error_message = get_error_message(error)
|
||||
setattr(error, 'error_message', error_message)
|
||||
data['errors'].append(error)
|
||||
data['media'] = Media.objects.filter(source=self.object).order_by('-published').defer('metadata')
|
||||
data['media'] = Media.objects.filter(source=self.object).order_by('-published')
|
||||
return data
|
||||
|
||||
|
||||
class UpdateSourceView(EditSourceMixin, UpdateView):
|
||||
class UpdateSourceView(UpdateView):
|
||||
|
||||
template_name = 'sync/source-update.html'
|
||||
model = Source
|
||||
fields = ('source_type', 'key', 'name', 'directory', 'media_format',
|
||||
'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
|
||||
'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
|
||||
'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
|
||||
errors = {
|
||||
'invalid_media_format': _('Invalid media format, the media format contains '
|
||||
'errors or is empty. Check the table at the end of '
|
||||
'this page for valid media name variables'),
|
||||
}
|
||||
|
||||
def form_valid(self, form):
|
||||
# Perform extra validation to make sure the media_format is valid
|
||||
obj = form.save(commit=False)
|
||||
source_type = form.cleaned_data['media_format']
|
||||
example_media_file = obj.get_example_media_format()
|
||||
if example_media_file == '':
|
||||
form.add_error(
|
||||
'media_format',
|
||||
ValidationError(self.errors['invalid_media_format'])
|
||||
)
|
||||
if form.errors:
|
||||
return super().form_invalid(form)
|
||||
return super().form_valid(form)
|
||||
|
||||
def get_success_url(self):
|
||||
url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
|
||||
|
|
@ -437,13 +411,12 @@ class DeleteSourceView(DeleteView, FormMixin):
|
|||
source = self.get_object()
|
||||
for media in Media.objects.filter(source=source):
|
||||
if media.media_file:
|
||||
file_path = media.media_file.path
|
||||
matching_files = glob.glob(os.path.splitext(file_path)[0] + '.*')
|
||||
for file in matching_files:
|
||||
delete_file(file)
|
||||
directory_path = source.directory_path
|
||||
if os.path.exists(directory_path):
|
||||
shutil.rmtree(directory_path, True)
|
||||
# Delete the media file
|
||||
delete_file(media.media_file.name)
|
||||
# Delete thumbnail copy if it exists
|
||||
delete_file(media.thumbpath)
|
||||
# Delete NFO file if it exists
|
||||
delete_file(media.nfopath)
|
||||
return super().post(request, *args, **kwargs)
|
||||
|
||||
def get_success_url(self):
|
||||
|
|
@ -465,8 +438,6 @@ class MediaView(ListView):
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.filter_source = None
|
||||
self.show_skipped = False
|
||||
self.only_skipped = False
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def dispatch(self, request, *args, **kwargs):
|
||||
|
|
@ -476,30 +447,13 @@ class MediaView(ListView):
|
|||
self.filter_source = Source.objects.get(pk=filter_by)
|
||||
except Source.DoesNotExist:
|
||||
self.filter_source = None
|
||||
show_skipped = request.GET.get('show_skipped', '').strip()
|
||||
if show_skipped == 'yes':
|
||||
self.show_skipped = True
|
||||
if not self.show_skipped:
|
||||
only_skipped = request.GET.get('only_skipped', '').strip()
|
||||
if only_skipped == 'yes':
|
||||
self.only_skipped = True
|
||||
return super().dispatch(request, *args, **kwargs)
|
||||
|
||||
def get_queryset(self):
|
||||
if self.filter_source:
|
||||
if self.show_skipped:
|
||||
q = Media.objects.filter(source=self.filter_source)
|
||||
elif self.only_skipped:
|
||||
q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=True) | Q(manual_skip=True)))
|
||||
else:
|
||||
q = Media.objects.filter(Q(source=self.filter_source) & (Q(skip=False) & Q(manual_skip=False)))
|
||||
else:
|
||||
if self.show_skipped:
|
||||
q = Media.objects.all()
|
||||
elif self.only_skipped:
|
||||
q = Media.objects.filter(Q(skip=True)|Q(manual_skip=True))
|
||||
else:
|
||||
q = Media.objects.filter(Q(skip=False)&Q(manual_skip=False))
|
||||
return q.order_by('-published', '-created')
|
||||
|
||||
def get_context_data(self, *args, **kwargs):
|
||||
|
|
@ -510,8 +464,6 @@ class MediaView(ListView):
|
|||
message = str(self.messages.get('filter', ''))
|
||||
data['message'] = message.format(name=self.filter_source.name)
|
||||
data['source'] = self.filter_source
|
||||
data['show_skipped'] = self.show_skipped
|
||||
data['only_skipped'] = self.only_skipped
|
||||
return data
|
||||
|
||||
|
||||
|
|
@ -654,15 +606,13 @@ class MediaSkipView(FormView, SingleObjectMixin):
|
|||
delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),))
|
||||
# If the media file exists on disk, delete it
|
||||
if self.object.media_file_exists:
|
||||
# Delete all files which contains filename
|
||||
filepath = self.object.media_file.path
|
||||
barefilepath, fileext = os.path.splitext(filepath)
|
||||
# Get all files that start with the bare file path
|
||||
all_related_files = glob.glob(f'{barefilepath}.*')
|
||||
for file in all_related_files:
|
||||
delete_file(file)
|
||||
delete_file(self.object.media_file.path)
|
||||
self.object.media_file = None
|
||||
# If the media has an associated thumbnail copied, also delete it
|
||||
delete_file(self.object.thumbpath)
|
||||
# If the media has an associated NFO file with it, also delete it
|
||||
delete_file(self.object.nfopath)
|
||||
# Reset all download data
|
||||
self.object.metadata = None
|
||||
self.object.downloaded = False
|
||||
self.object.downloaded_audio_codec = None
|
||||
self.object.downloaded_video_codec = None
|
||||
|
|
@ -672,7 +622,6 @@ class MediaSkipView(FormView, SingleObjectMixin):
|
|||
self.object.downloaded_filesize = None
|
||||
# Mark it to be skipped
|
||||
self.object.skip = True
|
||||
self.object.manual_skip = True
|
||||
self.object.save()
|
||||
return super().form_valid(form)
|
||||
|
||||
|
|
@ -701,7 +650,6 @@ class MediaEnableView(FormView, SingleObjectMixin):
|
|||
def form_valid(self, form):
|
||||
# Mark it as not skipped
|
||||
self.object.skip = False
|
||||
self.object.manual_skip = False
|
||||
self.object.save()
|
||||
return super().form_valid(form)
|
||||
|
||||
|
|
@ -710,52 +658,6 @@ class MediaEnableView(FormView, SingleObjectMixin):
|
|||
return append_uri_params(url, {'message': 'enabled'})
|
||||
|
||||
|
||||
class MediaContent(DetailView):
|
||||
'''
|
||||
Redirect to nginx to download the file
|
||||
'''
|
||||
model = Media
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.object = None
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def dispatch(self, request, *args, **kwargs):
|
||||
self.object = self.get_object()
|
||||
# development direct file stream - DO NOT USE PRODUCTIVLY
|
||||
if settings.DEBUG and 'runserver' in sys.argv:
|
||||
# get media URL
|
||||
pth = self.object.media_file.url
|
||||
# remove "/media-data/"
|
||||
pth = pth.split("/media-data/",1)[1]
|
||||
# remove "/" (incase of absolute path)
|
||||
pth = pth.split(str(settings.DOWNLOAD_ROOT).lstrip("/"),1)
|
||||
|
||||
# if we do not have a "/" at the beginning, it is not a absolute path...
|
||||
if len(pth) > 1:
|
||||
pth = pth[1]
|
||||
else:
|
||||
pth = pth[0]
|
||||
|
||||
|
||||
# build final path
|
||||
filepth = pathlib.Path(str(settings.DOWNLOAD_ROOT) + pth)
|
||||
|
||||
if filepth.exists():
|
||||
# return file
|
||||
response = FileResponse(open(filepth,'rb'))
|
||||
return response
|
||||
else:
|
||||
return HttpResponseNotFound()
|
||||
|
||||
else:
|
||||
headers = {
|
||||
'Content-Type': self.object.content_type,
|
||||
'X-Accel-Redirect': self.object.media_file.url,
|
||||
}
|
||||
return HttpResponse(headers=headers)
|
||||
|
||||
|
||||
class TasksView(ListView):
|
||||
'''
|
||||
A list of tasks queued to be completed. This is, for example, scraping for new
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue