bump to 0.9.1

Added format descriptions.

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [meeb]

Pipfile

@@ -17,6 +17,7 @@ httptools = "*"
 youtube-dl = "*"
 django-background-tasks = "*"
 requests = "*"
+django-basicauth = "*"
 
 [requires]
 python_version = "3"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "a4bb556fc61ee4583f9588980450b071814298ee4d1a1023fad149c14d14aaba"
+            "sha256": "f698e2853dec2d325d2d7e752620fc81d911022d394a57f2f8a9349ac2682752"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -39,11 +39,11 @@
         },
         "django": {
             "hashes": [
-                "sha256:5c866205f15e7a7123f1eec6ab939d22d5bde1416635cab259684af66d8e48a2",
-                "sha256:edb10b5c45e7e9c0fb1dc00b76ec7449aca258a39ffd613dbd078c51d19c9f03"
+                "sha256:32ce792ee9b6a0cbbec340123e229ac9f765dff8c2a4ae9247a14b2ba3a365a7",
+                "sha256:baf099db36ad31f970775d0be5587cc58a6256a6771a44eb795b554d45f211b8"
             ],
             "index": "pypi",
-            "version": "==3.1.4"
+            "version": "==3.1.7"
         },
         "django-appconf": {
             "hashes": [
@@ -59,6 +59,14 @@
             "index": "pypi",
             "version": "==1.2.5"
         },
+        "django-basicauth": {
+            "hashes": [
+                "sha256:15e9e366f698f53c71b1e794dafea060f990a2ac556bae6b7330dd25324a091c",
+                "sha256:e5e47d1acdc1943bedcc1bf673059d6c15e257dfe9eef67a22fb824f79546c0d"
+            ],
+            "index": "pypi",
+            "version": "==0.5.3"
+        },
         "django-compat": {
             "hashes": [
                 "sha256:3ac9a3bedc56b9365d9eb241bc5157d0c193769bf995f9a78dc1bc24e7c2331b"
@@ -134,44 +142,49 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:006de60d7580d81f4a1a7e9f0173dc90a932e3905cc4d47ea909bc946302311a",
-                "sha256:0a2e8d03787ec7ad71dc18aec9367c946ef8ef50e1e78c71f743bc3a770f9fae",
-                "sha256:0eeeae397e5a79dc088d8297a4c2c6f901f8fb30db47795113a4a605d0f1e5ce",
-                "sha256:11c5c6e9b02c9dac08af04f093eb5a2f84857df70a7d4a6a6ad461aca803fb9e",
-                "sha256:2fb113757a369a6cdb189f8df3226e995acfed0a8919a72416626af1a0a71140",
-                "sha256:4b0ef2470c4979e345e4e0cc1bbac65fda11d0d7b789dbac035e4c6ce3f98adb",
-                "sha256:59e903ca800c8cfd1ebe482349ec7c35687b95e98cefae213e271c8c7fffa021",
-                "sha256:5abd653a23c35d980b332bc0431d39663b1709d64142e3652890df4c9b6970f6",
-                "sha256:5f9403af9c790cc18411ea398a6950ee2def2a830ad0cfe6dc9122e6d528b302",
-                "sha256:6b4a8fd632b4ebee28282a9fef4c341835a1aa8671e2770b6f89adc8e8c2703c",
-                "sha256:6c1aca8231625115104a06e4389fcd9ec88f0c9befbabd80dc206c35561be271",
-                "sha256:795e91a60f291e75de2e20e6bdd67770f793c8605b553cb6e4387ce0cb302e09",
-                "sha256:7ba0ba61252ab23052e642abdb17fd08fdcfdbbf3b74c969a30c58ac1ade7cd3",
-                "sha256:7c9401e68730d6c4245b8e361d3d13e1035cbc94db86b49dc7da8bec235d0015",
-                "sha256:81f812d8f5e8a09b246515fac141e9d10113229bc33ea073fec11403b016bcf3",
-                "sha256:895d54c0ddc78a478c80f9c438579ac15f3e27bf442c2a9aa74d41d0e4d12544",
-                "sha256:8de332053707c80963b589b22f8e0229f1be1f3ca862a932c1bcd48dafb18dd8",
-                "sha256:92c882b70a40c79de9f5294dc99390671e07fc0b0113d472cbea3fde15db1792",
-                "sha256:95edb1ed513e68bddc2aee3de66ceaf743590bf16c023fb9977adc4be15bd3f0",
-                "sha256:b63d4ff734263ae4ce6593798bcfee6dbfb00523c82753a3a03cbc05555a9cc3",
-                "sha256:bd7bf289e05470b1bc74889d1466d9ad4a56d201f24397557b6f65c24a6844b8",
-                "sha256:cc3ea6b23954da84dbee8025c616040d9aa5eaf34ea6895a0a762ee9d3e12e11",
-                "sha256:cc9ec588c6ef3a1325fa032ec14d97b7309db493782ea8c304666fb10c3bd9a7",
-                "sha256:d3d07c86d4efa1facdf32aa878bd508c0dc4f87c48125cc16b937baa4e5b5e11",
-                "sha256:d8a96747df78cda35980905bf26e72960cba6d355ace4780d4bdde3b217cdf1e",
-                "sha256:e38d58d9138ef972fceb7aeec4be02e3f01d383723965bfcef14d174c8ccd039",
-                "sha256:eb472586374dc66b31e36e14720747595c2b265ae962987261f044e5cce644b5",
-                "sha256:fbd922f702582cb0d71ef94442bfca57624352622d75e3be7a1e7e9360b07e72"
+                "sha256:01bb0a34f1a6689b138c0089d670ae2e8f886d2666a9b2f2019031abdea673c4",
+                "sha256:07872f1d8421db5a3fe770f7480835e5e90fddb58f36c216d4a2ac0d594de474",
+                "sha256:1022f8f6dc3c5b0dcf928f1c49ba2ac73051f576af100d57776e2b65c1f76a8d",
+                "sha256:14415e9e28410232370615dbde0cf0a00e526f522f665460344a5b96973a3086",
+                "sha256:172acfaf00434a28dddfe592d83f2980e22e63c769ff4a448ddf7b7a38ffd165",
+                "sha256:1c5e3c36f02c815766ae9dd91899b1c5b4652f2a37b7a51609f3bd467c0f11fb",
+                "sha256:292f2aa1ae5c5c1451cb4b558addb88c257411d3fd71c6cf45562911baffc979",
+                "sha256:2a40d7d4b17db87f5b9a1efc0aff56000e1d0d5ece415090c102aafa0ccbe858",
+                "sha256:2f0d7034d5faae9a8d1019d152ede924f653df2ce77d3bba4ce62cd21b5f94ae",
+                "sha256:33fdbd4f5608c852d97264f9d2e3b54e9e9959083d008145175b86100b275e5b",
+                "sha256:3b13d89d97b551e02549d1f0edf22bed6acfd6fd2e888cd1e9a953bf215f0e81",
+                "sha256:3e759bcc03d6f39bc751e56d86bc87252b9a21c689a27c5ed753717a87d53a5b",
+                "sha256:3ec87bd1248b23a2e4e19e774367fbe30fddc73913edc5f9b37470624f55dc1f",
+                "sha256:436b0a2dd9fe3f7aa6a444af6bdf53c1eb8f5ced9ea3ef104daa83f0ea18e7bc",
+                "sha256:43b3c859912e8bf754b3c5142df624794b18eb7ae07cfeddc917e1a9406a3ef2",
+                "sha256:4fe74636ee71c57a7f65d7b21a9f127d842b4fb75511e5d256ace258826eb352",
+                "sha256:59445af66b59cc39530b4f810776928d75e95f41e945f0c32a3de4aceb93c15d",
+                "sha256:69da5b1d7102a61ce9b45deb2920a2012d52fd8f4201495ea9411d0071b0ec22",
+                "sha256:7094bbdecb95ebe53166e4c12cf5e28310c2b550b08c07c5dc15433898e2238e",
+                "sha256:8211cac9bf10461f9e33fe9a3af6c5131f3fdd0d10672afc2abb2c70cf95c5ca",
+                "sha256:8cf77e458bd996dc85455f10fe443c0c946f5b13253773439bcbec08aa1aebc2",
+                "sha256:924fc33cb4acaf6267b8ca3b8f1922620d57a28470d5e4f49672cea9a841eb08",
+                "sha256:99ce3333b40b7a4435e0a18baad468d44ab118a4b1da0af0a888893d03253f1d",
+                "sha256:a7d690b2c5f7e4a932374615fedceb1e305d2dd5363c1de15961725fe10e7d16",
+                "sha256:b9af590adc1e46898a1276527f3cfe2da8048ae43fbbf9b1bf9395f6c99d9b47",
+                "sha256:bb18422ad00c1fecc731d06592e99c3be2c634da19e26942ba2f13d805005cf2",
+                "sha256:c10af40ee2f1a99e1ae755ab1f773916e8bca3364029a042cd9161c400416bd8",
+                "sha256:c143c409e7bc1db784471fe9d0bf95f37c4458e879ad84cfae640cb74ee11a26",
+                "sha256:c448d2b335e21951416a30cd48d35588d122a912d5fe9e41900afacecc7d21a1",
+                "sha256:d30f30c044bdc0ab8f3924e1eeaac87e0ff8a27e87369c5cac4064b6ec78fd83",
+                "sha256:df534e64d4f3e84e8f1e1a37da3f541555d947c1c1c09b32178537f0f243f69d",
+                "sha256:f6fc18f9c9c7959bf58e6faf801d14fafb6d4717faaf6f79a68c8bb2a13dcf20",
+                "sha256:ff83dfeb04c98bb3e7948f876c17513a34e9a19fd92e292288649164924c1b39"
             ],
             "index": "pypi",
-            "version": "==8.0.1"
+            "version": "==8.1.1"
         },
         "pytz": {
             "hashes": [
-                "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",
-                "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"
+                "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
+                "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
             ],
-            "version": "==2020.4"
+            "version": "==2021.1"
         },
         "rcssmin": {
             "hashes": [
@@ -221,10 +234,10 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
-                "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
+                "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80",
+                "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73"
             ],
-            "version": "==1.26.2"
+            "version": "==1.26.3"
         },
         "whitenoise": {
             "hashes": [
@@ -236,11 +249,11 @@
         },
         "youtube-dl": {
             "hashes": [
-                "sha256:65968065e66966955dc79fad9251565fcc982566118756da624bd21467f3a04c",
-                "sha256:eaa859f15b6897bec21474b7787dc958118c8088e1f24d4ef1d58eab13188958"
+                "sha256:02432aa2dd0e859e64d74fca2ad624abf3bead3dba811d594100e1cb7897dce7",
+                "sha256:28663ce51bb35d0a0fa764aed3492b38c570da0a5a62fef3c28f4431522a6d4a"
             ],
             "index": "pypi",
-            "version": "==2020.12.14"
+            "version": "==2021.3.3"
         }
     },
     "develop": {}

README.md

@@ -22,9 +22,12 @@ hopefully, quite reliable.
 # Latest container image
 
 ```yaml
-ghcr.io/meeb/tubesync:v0.6
+ghcr.io/meeb/tubesync:v0.9.1
 ```
 
+**NOTE: the `:latest` tag does exist, but will contain in-development commits and may
+be broken. Use at your own risk.**
+
 # Screenshots
 
 ### Dashboard
@@ -99,7 +102,7 @@ Finally, download and run the container:
 
 ```bash
 # Pull a versioned image
-$ docker pull ghcr.io/meeb/tubesync:v0.6
+$ docker pull ghcr.io/meeb/tubesync:v0.9.1
 # Start the container using your user ID and group ID
 $ docker run \
   -d \
@@ -110,7 +113,7 @@ $ docker run \
   -v /some/directory/tubesync-config:/config \
   -v /some/directory/tubesync-downloads:/downloads \
   -p 4848:4848 \
-  ghcr.io/meeb/tubesync:v0.6
+  ghcr.io/meeb/tubesync:v0.9.1
 ```
 
 Once running, open `http://localhost:4848` in your browser and you should see the
@@ -122,7 +125,7 @@ Alternatively, for Docker Compose, you can use something like:
 
 ```yaml
   tubesync:
-    image: ghcr.io/meeb/tubesync:v0.6
+    image: ghcr.io/meeb/tubesync:v0.9.1
     container_name: tubesync
     restart: unless-stopped
     ports:
@@ -136,6 +139,41 @@ Alternatively, for Docker Compose, you can use something like:
       - PGID=1000
 ```
 
+
+## Optional authentication
+
+Available in `v1.0` (or `:latest`)and later. If you want to enable a basic username and
+password to be required to access the TubeSync dashboard you can set them with the
+following environment variables:
+
+```bash
+HTTP_USER
+HTTP_PASS
+```
+
+For example in the `docker run ...` line add in:
+
+```bash
+...
+-e HTTP_USER=some-username \
+-e HTTP_PASS=some-secure-password \
+...
+```
+
+Or in your Docker Compose file you would add in:
+
+```yaml
+...
+    environment:
+      - HTTP_USER=some-username
+      - HTTP_PASS=some-secure-password
+...
+```
+
+When BOTH `HTTP_USER` and `HTTP_PASS` are set then basic HTTP authentication will be
+enabled.
+
+
 # Updating
 
 To update, you can just pull a new version of the container image as they are released.
@@ -193,6 +231,18 @@ $ docker logs --follow tubesync
 ```
 
 
+# Advanced usage guides
+
+Once you're happy using TubeSync there are some advanced usage guides for more complex
+and less common features:
+
+![Import existing media into TubeSync](https://github.com/meeb/tubesync/blob/main/docs/import-existing-media.md)
+
+![Sync or create missing metadata files](https://github.com/meeb/tubesync/blob/main/docs/create-missing-metadata.md)
+
+![Reset tasks from the command line](https://github.com/meeb/tubesync/blob/main/docs/reset-tasks.md)
+
+
 # Warnings
 
 ### 1. Index frequency
@@ -210,6 +260,14 @@ every hour" or similar short interval it's entirely possible your TubeSync insta
 spend its entire time just indexing the massive channel over and over again without
 downloading any media. Check your tasks for the status of your TubeSync install.
 
+If you add a significant amount of "work" due to adding many large channels you may
+need to increase the number of background workers by setting the `TUBESYNC_WORKERS`
+environment variable. Try around ~4 at most, although the absolute maximum allowed is 8.
+
+**Be nice.** it's likely entirely possible your IP address could get throttled by the
+source if you try and crawl extremely large amounts very quickly. **Try and be polite
+with the smallest amount of indexing and concurrent downloads possible for your needs.**
+
 
 # FAQ
 
@@ -279,7 +337,9 @@ can log in at http://localhost:4848/admin
 
 ### Are there user accounts or multi-user support?
 
-No not at the moment. This could be added later if there is demand for it.
+There is support for basic HTTP authentication by setting the `HTTP_USER` and
+`HTTP_PASS` environment variables. There is not support for multi-user or user
+management.
 
 ### Does TubeSync support HTTPS?
 
@@ -297,15 +357,18 @@ There are a number of other environment variables you can set. These are, mostly
 **NOT** required to be set in the default container installation, they are really only
 useful if you are manually installing TubeSync in some other environment. These are:
 
-| Name                     | What                                  | Example                            |
-| ------------------------ | ------------------------------------- | ---------------------------------- |
-| DJANGO_SECRET_KEY        | Django's SECRET_KEY                   | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l   |
-| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME            | /somepath                          |
-| TUBESYNC_DEBUG           | Enable debugging                      | True                               |
-| TUBESYNC_HOSTS           | Django's ALLOWED_HOSTS                | tubesync.example.com,otherhost.com |
-| GUNICORN_WORKERS         | Number of gunicorn workers to spawn   | 3                                  |
-| LISTEN_HOST              | IP address for gunicorn to listen on  | 127.0.0.1                          |
-| LISTEN_PORT              | Port number for gunicorn to listen on | 8080                               |
+| Name                     | What                                                         | Example                            |
+| ------------------------ | ------------------------------------------------------------ | ---------------------------------- |
+| DJANGO_SECRET_KEY        | Django's SECRET_KEY                                          | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l   |
+| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME                                   | /somepath                          |
+| TUBESYNC_DEBUG           | Enable debugging                                             | True                               |
+| TUBESYNC_WORKERS         | Number of background workers, default is 2, max allowed is 8 | 2                                  |
+| TUBESYNC_HOSTS           | Django's ALLOWED_HOSTS                                       | tubesync.example.com,otherhost.com |
+| GUNICORN_WORKERS         | Number of gunicorn workers to spawn                          | 3                                  |
+| LISTEN_HOST              | IP address for gunicorn to listen on                         | 127.0.0.1                          |
+| LISTEN_PORT              | Port number for gunicorn to listen on                        | 8080                               |
+| HTTP_USER                | Sets the username for HTTP basic authentication              | some-username                      |
+| HTTP_PASS                | Sets the password for HTTP basic authentication              | some-secure-password               |
 
 
 # Manual, non-containerised, installation
@@ -322,7 +385,7 @@ installing and running WSGI-based Python web applications before attempting this
    `tubesync/tubesync/local_settings.py` and edit it as appropriate
 5. Run migrations with `./manage.py migrate`
 6. Collect static files with `./manage.py collectstatic`
-6. Set up your prefered WSGI server, such as `gunicorn` poiting it to the application
+6. Set up your prefered WSGI server, such as `gunicorn` pointing it to the application
    in `tubesync/tubesync/wsgi.py`
 7. Set up your proxy server such as `nginx` and forward it to the WSGI server
 8. Check the web interface is working

config/root/etc/cont-init.d/10-tubesync

@@ -19,8 +19,8 @@ chown -R app:app /app/common/static && \
 chmod -R 0750 /app/common/static && \
 chown -R app:app /app/static && \
 chmod -R 0750 /app/static && \
-find /app -type f -exec chmod 640 {} \; && \
-chmod +x /app/healthcheck.py
+find /app -type f ! -iname healthcheck.py -exec chmod 640 {} \; && \
+chmod 0755 /app/healthcheck.py
 
 # Run migrations
 exec s6-setuidgid app \

docs/create-missing-metadata.md

@@ -0,0 +1,37 @@
+# TubeSync
+
+## Advanced usage guide - creating missing metadata
+
+This is a new feature in v0.9 of TubeSync and later. It allows you to create or
+re-create missing metadata in your TubeSync download directories for missing `nfo`
+files and thumbnails.
+
+If you add a source with "write NFO files" or "copy thumbnails" disabled, download
+some media and then update the source to write NFO files or copy thumbnails then
+TubeSync will not automatically retroactively attempt to copy or create your missing
+metadata files. You can use a special one-off command to manually write missing
+metadata files to the correct locations.
+
+## Requirements
+
+You have added a source without metadata writing enabled, downloaded some media, then
+updated the source to enable metadata writing.
+
+## Steps
+
+### 1. Run the batch metadata sync command
+
+Execute the following Django command:
+
+`./manage.py sync-missing-metadata`
+
+When deploying TubeSync inside a container, you can execute this with:
+
+`docker exec -ti tubesync python3 /app/manage.py sync-missing-metadata`
+
+This command will log what its doing to the terminal when you run it.
+
+Internally, this command loops over all your sources which have been saved with
+"write NFO files" or "copy thumbnails" enabled. Then, loops over all media saved to
+that source and confirms that the appropriate thumbnail files have been copied over and
+the NFO file has been written if enabled.

docs/import-existing-media.md

@@ -0,0 +1,81 @@
+# TubeSync
+
+## Advanced usage guide - importing existing media
+
+This is a new feature in v0.9 of TubeSync and later. It allows you to mark existing
+downloaded media as "downloaded" in TubeSync. You can use this feature if, for example,
+you already have an extensive catalogue of downloaded media which you want to mark
+as downloaded into TubeSync so TubeSync doesn't re-download media you already have.
+
+## Requirements
+
+Your existing downloaded media MUST contain the unique ID. For YouTube videos, this is
+means the YouTube video ID MUST be in the filename.
+
+Supported extensions to be imported are .m4a, .ogg, .mkv, .mp3, .mp4 and .avi. Your
+media you want to import must end in one of these file extensions.
+
+## Caveats
+
+As TubeSync does not probe media and your existing media may be re-encoded or in
+different formats to what is available in the current media metadata there is no way
+for TubeSync to know what codecs, resolution, bitrate etc. your imported media is in.
+Any manually imported existing local media will display blank boxes for this
+information on the TubeSync interface as it's unavailable.
+
+## Steps
+
+### 1. Add your source to TubeSync
+
+Add your source to TubeSync, such as a YouTube channel. **Make sure you untick the
+"download media" checkbox.**
+
+This will allow TubeSync to index all the available media on your source, but won't
+start downloading any media.
+
+### 2. Wait
+
+Wait for all the media on your source to be indexed. This may take some time.
+
+### 3. Move your existing media into TubeSync
+
+You now need to move your existing media into TubeSync. You need to move the media
+files into the correct download directories created by TubeSync. For example, if you
+have downloaded videos for a YouTube channel "TestChannel", you would have added this
+as a source called TestChannel and in a directory called test-channel in Tubesync. It
+would have a download directory created on disk at:
+
+`/path/to/downloads/test-channel`
+
+You would move all of your pre-existing videos you downloaded outside of TubeSync for
+this channel into this directory.
+
+In short, your existing media needs to be moved into the correct TubeSync source
+directory to be detected.
+
+This is required so TubeSync can known which Source to link the media to.
+
+### 4. Run the batch import command
+
+Execute the following Django command:
+
+`./manage.py import-existing-media`
+
+When deploying TubeSync inside a container, you can execute this with:
+
+`docker exec -ti tubesync python3 /app/manage.py import-existing-media`
+
+This command will log what its doing to the terminal when you run it.
+
+Internally, `import-existing-media` looks for the unique media key (for YouTube, this
+is the YouTube video ID) in the filename and detects the source to link it to based
+on the directory the media file is inside.
+
+
+### 5. Re-enable downloading at the source
+
+Edit your source and re-enable / tick the "download media" option. This will allow
+TubeSync to download any missing media you did not manually import.
+
+Note that TubeSync will still get screenshots write `nfo` files etc. for files you
+manually import if enabled at the source level.

docs/reset-tasks.md

@@ -0,0 +1,33 @@
+# TubeSync
+
+## Advanced usage guide - reset tasks from the command line
+
+This is a new feature in v1.0 of TubeSync and later. It allows you to reset all
+scheduled tasks from the command line as well as the "reset tasks" button in the
+"tasks" tab of the dashboard.
+
+This is useful for TubeSync installations where you may have a lot of media and
+sources added and the "reset tasks" button may take too long to the extent where
+the page times out (with a 502 error or similar issue).
+
+## Requirements
+
+You have added some sources and media
+
+## Steps
+
+### 1. Run the reset tasks command
+
+Execute the following Django command:
+
+`./manage.py reset-tasks`
+
+When deploying TubeSync inside a container, you can execute this with:
+
+`docker exec -ti tubesync python3 /app/manage.py reset-tasks`
+
+This command will log what its doing to the terminal when you run it.
+
+When this is run, new tasks will be immediately created so all your sources will be
+indexed again straight away, any missing information such as thumbnails will be
+redownloaded, etc.

tubesync/common/middleware.py

@@ -1,4 +1,6 @@
+from django.conf import settings
 from django.forms import BaseForm
+from basicauth.middleware import BasicAuthMiddleware as BaseBasicAuthMiddleware
 
 
 class MaterializeDefaultFieldsMiddleware:
@@ -19,3 +21,12 @@ class MaterializeDefaultFieldsMiddleware:
                 for _, field in v.fields.items():
                     field.widget.attrs.update({'class':'browser-default'})
         return response
+
+
+class BasicAuthMiddleware(BaseBasicAuthMiddleware):
+
+    def process_request(self, request):
+        bypass_uris = getattr(settings, 'BASICAUTH_ALWAYS_ALLOW_URIS', [])
+        if request.path in bypass_uris:
+            return None
+        return super().process_request(request)

tubesync/common/static/styles/_forms.scss

@@ -4,7 +4,7 @@
     }
     .help-text {
         color: $form-help-text-colour;
-        padding: 1rem 0 1rem 0;
+        padding-bottom: 1rem;
     }
     label {
         text-transform: uppercase;

tubesync/common/static/styles/_template.scss

@@ -5,6 +5,13 @@ html {
     color: $text-colour;
 }
 
+body {
+    display: flex;
+    min-height: 100vh;
+    flex-direction: column;
+    justify-content: space-between;
+}
+
 header {
 
     background-color: $header-background-colour;
@@ -174,8 +181,10 @@ main {
             display: inline-block;
             font-weight: bold;
             text-decoration: none;
-            padding: 5px 10px 5px 10px;
-            margin: 0 3px 0 3px;
+            padding: 5px 8px 4px 8px;
+            margin: 0 3px 6px 3px;
+            min-width: 40px;
+            min-height: 40px;
             background-color: $pagination-background-colour;
             color: $pagination-text-colour;
             border: 2px $pagination-border-colour solid;

tubesync/common/templates/base.html

@@ -16,32 +16,36 @@
 
   <body>
 
-    <header>
-      <div class="container">
-        <a href="{% url 'sync:dashboard' %}">
-          {% include 'tubesync.svg' with width='3rem' height='3rem' %}
-          <h1>TubeSync</h1>
-        </a>
-      </div>
-    </header>
+    <div class="app">
 
-    <nav>
-      <div class="container">
-        <ul>
-          <li><a href="{% url 'sync:dashboard' %}"><i class="fas fa-fw fa-th-large"></i><span class="hide-on-med-and-down"> Dashboard</span></a></li>
-          <li><a href="{% url 'sync:sources' %}"><i class="fas fa-fw fa-play"></i><span class="hide-on-med-and-down"> Sources</span></a></li>
-          <li><a href="{% url 'sync:media' %}"><i class="fas fa-fw fa-film"></i><span class="hide-on-med-and-down"> Media</span></a></li>
-          <li><a href="{% url 'sync:tasks' %}"><i class="far fa-fw fa-clock"></i><span class="hide-on-med-and-down"> Tasks</span></a></li>
-          <li><a href="{% url 'sync:mediaservers' %}"><i class="fas fa-fw fa-stream"></i><span class="hide-on-med-and-down"> Media Servers</span></a></li>
-        </ul>
-      </div>
-    </nav>
+      <header>
+        <div class="container">
+          <a href="{% url 'sync:dashboard' %}">
+            {% include 'tubesync.svg' with width='3rem' height='3rem' %}
+            <h1>TubeSync</h1>
+          </a>
+        </div>
+      </header>
 
-    <main>
-      <div class="container">
-        {% block content %}{% endblock %}
-      </div>
-    </main>
+      <nav>
+        <div class="container">
+          <ul>
+            <li><a href="{% url 'sync:dashboard' %}"><i class="fas fa-fw fa-th-large"></i><span class="hide-on-med-and-down"> Dashboard</span></a></li>
+            <li><a href="{% url 'sync:sources' %}"><i class="fas fa-fw fa-play"></i><span class="hide-on-med-and-down"> Sources</span></a></li>
+            <li><a href="{% url 'sync:media' %}"><i class="fas fa-fw fa-film"></i><span class="hide-on-med-and-down"> Media</span></a></li>
+            <li><a href="{% url 'sync:tasks' %}"><i class="far fa-fw fa-clock"></i><span class="hide-on-med-and-down"> Tasks</span></a></li>
+            <li><a href="{% url 'sync:mediaservers' %}"><i class="fas fa-fw fa-stream"></i><span class="hide-on-med-and-down"> Media Servers</span></a></li>
+          </ul>
+        </div>
+      </nav>
+
+      <main>
+        <div class="container">
+          {% block content %}{% endblock %}
+        </div>
+      </main>
+
+    </div>
 
     <footer>
       <div class="container">

tubesync/common/templates/pagination.html

@@ -3,7 +3,7 @@
   <div class="col s12">
     <div class="pagination">
       {% for i in paginator.page_range %}
-        <a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}">{{ i }}</a>
+        <a class="pagenum{% if i == page_obj.number %} currentpage{% endif %}" href="?{% if filter %}filter={{ filter }}&{% endif %}page={{ i }}{% if show_skipped %}&show_skipped=yes{% endif %}">{{ i }}</a>
       {% endfor %}
     </div>
   </div>

tubesync/common/utils.py

@@ -14,3 +14,13 @@ def append_uri_params(uri, params):
     uri = str(uri)
     qs = urlencode(params)
     return urlunsplit(('', '', uri, qs, ''))
+
+
+def clean_filename(filename):
+    if not isinstance(filename, str):
+        raise ValueError(f'filename must be a str, got {type(filename)}')
+    to_scrub = '<>\/:*?"|%'
+    for char in to_scrub:
+        filename = filename.replace(char, '')
+    filename = ''.join([c for c in filename if ord(c) > 30])
+    return ' '.join(filename.split())

tubesync/sync/admin.py

@@ -7,7 +7,7 @@ class SourceAdmin(admin.ModelAdmin):
 
     ordering = ('-created',)
     list_display = ('uuid', 'name', 'source_type', 'last_crawl',
-                    'has_failed')
+                    'download_media', 'has_failed')
     readonly_fields = ('uuid', 'created')
     search_fields = ('uuid', 'key', 'name')
 

tubesync/sync/management/commands/import-existing-media.py

@@ -0,0 +1,55 @@
+import os
+from pathlib import Path
+from django.core.management.base import BaseCommand, CommandError
+from common.logger import log
+from sync.models import Source, Media
+
+
+class Command(BaseCommand):
+
+    help = ('Scans download media directories for media not yet downloaded and ',
+            'marks them as downloaded')
+    extra_extensions = ['mp3', 'mp4', 'avi']
+
+    def handle(self, *args, **options):
+        log.info('Building directory to Source map...')
+        dirmap = {}
+        for s in Source.objects.all():
+            dirmap[s.directory_path] = s
+        log.info(f'Scanning sources...')
+        file_extensions = list(Source.EXTENSIONS) + self.extra_extensions
+        for sourceroot, source in dirmap.items():
+            media = list(Media.objects.filter(source=source, downloaded=False,
+                                              skip=False))
+            if not media:
+                log.info(f'Source "{source}" has no missing media')
+                continue
+            log.info(f'Scanning Source "{source}" directory for media to '
+                     f'import: {sourceroot}, looking for {len(media)} '
+                     f'undownloaded and unskipped items')
+            on_disk = []
+            for (root, dirs, files) in os.walk(sourceroot):
+                rootpath = Path(root)
+                for filename in files:
+                    filepart, ext = os.path.splitext(filename)
+                    if ext.startswith('.'):
+                        ext = ext[1:]
+                    ext = ext.strip().lower()
+                    if ext not in file_extensions:
+                        continue
+                    on_disk.append(str(rootpath / filename))
+            filemap = {}
+            for item in media:
+                for filepath in on_disk:
+                    if item.key in filepath:
+                        # The unique item key is in the file name on disk, map it to
+                        # the undownloaded media item
+                        filemap[filepath] = item
+                        continue
+            for filepath, item in filemap.items():
+                log.info(f'Matched on-disk file: {filepath} '
+                         f'to media item: {item.source} / {item}')
+                item.media_file.name = filepath
+                item.downloaded = True
+                item.save()
+        log.info('Done')

tubesync/sync/management/commands/reset-tasks.py

@@ -0,0 +1,32 @@
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import gettext_lazy as _
+from background_task.models import Task
+from sync.models import Source
+from sync.tasks import index_source_task
+
+
+from common.logger import log
+
+
+class Command(BaseCommand):
+
+    help = 'Resets all tasks'
+
+    def handle(self, *args, **options):
+        log.info('Resettings all tasks...')
+        # Delete all tasks
+        Task.objects.all().delete()
+        # Iter all tasks
+        for source in Source.objects.all():
+            # Recreate the initial indexing task
+            verbose_name = _('Index media from source "{}"')
+            index_source_task(
+                str(source.pk),
+                repeat=source.index_schedule,
+                queue=str(source.pk),
+                priority=5,
+                verbose_name=verbose_name.format(source.name)
+            )
+            # This also chains down to call each Media objects .save() as well
+            source.save()
+        log.info('Done')

tubesync/sync/management/commands/sync-missing-metadata.py

@@ -0,0 +1,34 @@
+import os
+from shutil import copyfile
+from django.core.management.base import BaseCommand, CommandError
+from django.db.models import Q
+from common.logger import log
+from sync.models import Source, Media
+from sync.utils import write_text_file
+
+
+class Command(BaseCommand):
+
+    help = 'Syncs missing metadata (such as nfo files) if source settings are updated'
+
+    def handle(self, *args, **options):
+        log.info('Syncing missing metadata...')
+        sources = Source.objects.filter(Q(copy_thumbnails=True) | Q(write_nfo=True))
+        for source in sources.order_by('name'):
+            log.info(f'Finding media for source: {source}')
+            for item in Media.objects.filter(source=source, downloaded=True):
+                log.info(f'Checking media for missing metadata: {source} / {item}')
+                thumbpath = item.thumbpath
+                if not thumbpath.is_file():
+                    if item.thumb:
+                        log.info(f'Copying missing thumbnail from: {item.thumb.path} '
+                                 f'to: {thumbpath}')
+                        copyfile(item.thumb.path, thumbpath)
+                    else:
+                        log.error(f'Tried to copy missing thumbnail for {item} but '
+                                  f'the thumbnail has not been downloaded')
+                nfopath = item.nfopath
+                if not nfopath.is_file():
+                    log.info(f'Writing missing NFO file: {nfopath}')
+                    write_text_file(nfopath, item.nfoxml)
+        log.info('Done')

tubesync/sync/matching.py

@@ -66,7 +66,7 @@ def get_best_audio_format(media):
     # No codecs matched
     if media.source.can_fallback:
         # Can fallback, find the next highest bitrate non-matching codec
-        return False, audio_formats[0]
+        return False, audio_formats[0]['id']
     else:
         # Can't fallback
         return False, False

tubesync/sync/migrations/0009_auto_20210218_0442.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.1.6 on 2021-02-18 04:42
+
+import django.core.files.storage
+from django.db import migrations, models
+import sync.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0008_source_download_cap'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='download_media',
+            field=models.BooleanField(default=True, help_text='Download media from this source, if not selected the source will only be indexed', verbose_name='download media'),
+        ),
+        migrations.AlterField(
+            model_name='media',
+            name='media_file',
+            field=models.FileField(blank=True, help_text='Media file', max_length=200, null=True, storage=django.core.files.storage.FileSystemStorage(location='/home/meeb/Repos/github.com/meeb/tubesync/tubesync/downloads'), upload_to=sync.models.get_media_file_path, verbose_name='media file'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='media_format',
+            field=models.CharField(default='{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files, detailed options at bottom of page.', max_length=200, verbose_name='media format'),
+        ),
+    ]

tubesync/sync/models.py

@@ -12,6 +12,7 @@ from django.utils.text import slugify
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from common.errors import NoFormatException
+from common.utils import clean_filename
 from .youtube import (get_media_info as get_youtube_media_info,
                       download_media as download_youtube_media)
 from .utils import seconds_to_timestr, parse_media_format
@@ -100,6 +101,11 @@ class Source(models.Model):
         (FALLBACK_NEXT_BEST_HD, _('Get next best resolution but at least HD'))
     )
 
+    EXTENSION_M4A = 'm4a'
+    EXTENSION_OGG = 'ogg'
+    EXTENSION_MKV = 'mkv'
+    EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV)
+
     # Fontawesome icons used for the source on the front end
     ICONS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
@@ -112,6 +118,12 @@ class Source(models.Model):
         SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}',
         SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
     }
+    # Format used to create indexable URLs
+    INDEX_URLS = {
+        SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos',
+        SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos',
+        SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
+    }
     # Callback functions to get a list of media from the source
     INDEXERS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
@@ -200,7 +212,7 @@ class Source(models.Model):
         _('media format'),
         max_length=200,
         default=settings.MEDIA_FORMATSTR_DEFAULT,
-        help_text=_('File format to use for saving files')
+        help_text=_('File format to use for saving files, detailed options at bottom of page.')
     )
     index_schedule = models.IntegerField(
         _('index schedule'),
@@ -209,6 +221,11 @@ class Source(models.Model):
         default=IndexSchedule.EVERY_6_HOURS,
         help_text=_('Schedule of how often to index the source for new media')
     )
+    download_media = models.BooleanField(
+        _('download media'),
+        default=True,
+        help_text=_('Download media from this source, if not selected the source will only be indexed')
+    )
     download_cap = models.IntegerField(
         _('download cap'),
         choices=CapChoices.choices,
@@ -327,23 +344,32 @@ class Source(models.Model):
         '''
         if self.is_audio:
             if self.source_acodec == self.SOURCE_ACODEC_MP4A:
-                return 'm4a'
+                return self.EXTENSION_M4A
             elif self.source_acodec == self.SOURCE_ACODEC_OPUS:
-                return 'ogg'
+                return self.EXTENSION_OGG
             else:
                 raise ValueError('Unable to choose audio extension, uknown acodec')
         else:
-            return 'mkv'
+            return self.EXTENSION_MKV
 
     @classmethod
     def create_url(obj, source_type, key):
         url = obj.URLS.get(source_type)
         return url.format(key=key)
 
+    @classmethod
+    def create_index_url(obj, source_type, key):
+        url = obj.INDEX_URLS.get(source_type)
+        return url.format(key=key)
+
     @property
     def url(self):
         return Source.create_url(self.source_type, self.key)
 
+    @property
+    def index_url(self):
+        return Source.create_index_url(self.source_type, self.key)
+
     @property
     def format_summary(self):
         if self.source_resolution == Source.SOURCE_RESOLUTION_AUDIO:
@@ -436,25 +462,10 @@ class Source(models.Model):
         indexer = self.INDEXERS.get(self.source_type, None)
         if not callable(indexer):
             raise Exception(f'Source type f"{self.source_type}" has no indexer')
-        response = indexer(self.url)
-
-        # Account for nested playlists, such as a channel of playlists of playlists
-        def _recurse_playlists(playlist):
-            videos = []
-            if not playlist:
-                return videos
-            entries = playlist.get('entries', [])
-            for entry in entries:
-                if not entry:
-                    continue
-                subentries = entry.get('entries', [])
-                if subentries:
-                    videos = videos + _recurse_playlists(entry)
-                else:
-                    videos.append(entry)
-            return videos
-
-        return _recurse_playlists(response)
+        response = indexer(self.index_url)
+        if not isinstance(response, dict):
+            return []
+        return response.get('entries', [])
 
 
 def get_media_thumb_path(instance, filename):
@@ -480,6 +491,12 @@ class Media(models.Model):
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/watch?v={key}',
         Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}',
     }
+    # Callback functions to get a list of media from the source
+    INDEXERS = {
+        Source.SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
+        Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
+        Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
+    }
     # Maps standardised names to names used in source metdata
     METADATA_FIELDS = {
         'upload_date': {
@@ -557,14 +574,18 @@ class Media(models.Model):
     STATE_SCHEDULED = 'scheduled'
     STATE_DOWNLOADING = 'downloading'
     STATE_DOWNLOADED = 'downloaded'
+    STATE_SKIPPED = 'skipped'
+    STATE_DISABLED_AT_SOURCE = 'source-disabled'
     STATE_ERROR = 'error'
     STATES = (STATE_UNKNOWN, STATE_SCHEDULED, STATE_DOWNLOADING, STATE_DOWNLOADED,
-              STATE_ERROR)
+              STATE_SKIPPED, STATE_DISABLED_AT_SOURCE, STATE_ERROR)
     STATE_ICONS = {
         STATE_UNKNOWN: '<i class="far fa-question-circle" title="Unknown download state"></i>',
         STATE_SCHEDULED: '<i class="far fa-clock" title="Scheduled to download"></i>',
         STATE_DOWNLOADING: '<i class="fas fa-download" title="Downloading now"></i>',
         STATE_DOWNLOADED: '<i class="far fa-check-circle" title="Downloaded"></i>',
+        STATE_SKIPPED: '<i class="fas fa-exclamation-circle" title="Skipped"></i>',
+        STATE_DISABLED_AT_SOURCE: '<i class="fas fa-stop-circle" title="Media downloading disabled at source"></i>',
         STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>',
     }
 
@@ -838,8 +859,9 @@ class Media(models.Model):
             fmt.append(resolution)
             vcodec = vformat['vcodec'].lower()
             fmt.append(vcodec)
-        acodec = aformat['acodec'].lower()
-        fmt.append(acodec)
+        if aformat:
+            acodec = aformat['acodec'].lower()
+            fmt.append(acodec)
         if vformat:
             if vformat['is_60fps']:
                 fps = '60fps'
@@ -887,7 +909,7 @@ class Media(models.Model):
             'source': self.source.slugname,
             'source_full': self.source.name,
             'title': self.slugtitle,
-            'title_full': self.title,
+            'title_full': clean_filename(self.title),
             'key': self.key,
             'format': '-'.join(display_format['format']),
             'playlist_index': self.playlist_index,
@@ -902,10 +924,17 @@ class Media(models.Model):
             'hdr': display_format['hdr'],
         }
 
+    @property
+    def has_metadata(self):
+        return self.metadata is not None
+
     @property
     def loaded_metadata(self):
         try:
-            return json.loads(self.metadata)
+            data = json.loads(self.metadata)
+            if not isinstance(data, dict):
+                return {}
+            return data
         except Exception as e:
             return {}
 
@@ -942,7 +971,10 @@ class Media(models.Model):
     @property
     def upload_date(self):
         field = self.get_metadata_field('upload_date')
-        upload_date_str = self.loaded_metadata.get(field, '').strip()
+        try:
+            upload_date_str = self.loaded_metadata.get(field, '').strip()
+        except (AttributeError, ValueError) as e:
+            return None
         try:
             return datetime.strptime(upload_date_str, '%Y%m%d')
         except (AttributeError, ValueError) as e:
@@ -974,8 +1006,12 @@ class Media(models.Model):
     def votes(self):
         field = self.get_metadata_field('upvotes')
         upvotes = self.loaded_metadata.get(field, 0)
+        if not isinstance(upvotes, int):
+            upvotes = 0
         field = self.get_metadata_field('downvotes')
         downvotes = self.loaded_metadata.get(field, 0)
+        if not isinstance(downvotes, int):
+            downvotes = 0
         return upvotes + downvotes
 
     @property
@@ -1005,7 +1041,7 @@ class Media(models.Model):
 
     @property
     def filename(self):
-        # Otherwise, create a suitable filename from the source media_format
+        # Create a suitable filename from the source media_format
         media_format = str(self.source.media_format)
         media_details = self.format_dict
         return media_format.format(**media_details)
@@ -1161,6 +1197,10 @@ class Media(models.Model):
                 return self.STATE_ERROR
             else:
                 return self.STATE_SCHEDULED
+        if self.skip:
+            return self.STATE_SKIPPED
+        if not self.source.download_media:
+            return self.STATE_DISABLED_AT_SOURCE
         return self.STATE_UNKNOWN
 
     def get_download_state_icon(self, task=None):
@@ -1178,6 +1218,16 @@ class Media(models.Model):
         # Return the download paramaters
         return format_str, self.source.extension
 
+    def index_metadata(self):
+        '''
+            Index the media metadata returning a dict of info.
+        '''
+        indexer = self.INDEXERS.get(self.source.source_type, None)
+        if not callable(indexer):
+            raise Exception(f'Meida with source type f"{self.source.source_type}" '
+                            f'has no indexer')
+        return indexer(self.url)
+
 
 class MediaServer(models.Model):
     '''

tubesync/sync/signals.py

@@ -8,8 +8,9 @@ from background_task.models import Task
 from common.logger import log
 from .models import Source, Media, MediaServer
 from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task,
-                    download_media_thumbnail, map_task_to_instance,
-                    check_source_directory_exists, download_media, rescan_media_server)
+                    download_media_thumbnail, download_media_metadata,
+                    map_task_to_instance, check_source_directory_exists,
+                    download_media, rescan_media_server)
 from .utils import delete_file
 
 
@@ -93,16 +94,27 @@ def task_task_failed(sender, task_id, completed_task, **kwargs):
 def media_post_save(sender, instance, created, **kwargs):
     # Triggered after media is saved, Recalculate the "can_download" flag, this may
     # need to change if the source specifications have been changed
-    post_save.disconnect(media_post_save, sender=Media)
-    if instance.get_format_str():
-        if not instance.can_download:
-            instance.can_download = True
-            instance.save()
-    else:
-        if instance.can_download:
-            instance.can_download = False
-            instance.save()
-    post_save.connect(media_post_save, sender=Media)
+    if instance.metadata:
+        post_save.disconnect(media_post_save, sender=Media)
+        if instance.get_format_str():
+            if not instance.can_download:
+                instance.can_download = True
+                instance.save()
+        else:
+            if instance.can_download:
+                instance.can_download = False
+                instance.save()
+        post_save.connect(media_post_save, sender=Media)
+    # If the media is missing metadata schedule it to be downloaded
+    if not instance.metadata:
+        log.info(f'Scheduling task to download metadata for: {instance.url}')
+        verbose_name = _('Downloading metadata for "{}"')
+        download_media_metadata(
+            str(instance.pk),
+            priority=10,
+            verbose_name=verbose_name.format(instance.pk),
+            remove_existing_tasks=True
+        )
     # If the media is missing a thumbnail schedule it to be downloaded
     if not instance.thumb_file_exists:
         instance.thumb = None
@@ -124,7 +136,8 @@ def media_post_save(sender, instance, created, **kwargs):
     if not instance.media_file_exists:
         instance.downloaded = False
         instance.media_file = None
-    if not instance.downloaded and instance.can_download and not instance.skip:
+    if (not instance.downloaded and instance.can_download and not instance.skip
+        and instance.source.download_media):
         delete_task_by_media('sync.tasks.download_media', (str(instance.pk),))
         verbose_name = _('Downloading media for "{}"')
         download_media(

tubesync/sync/tasks.py

@@ -10,7 +10,7 @@ import math
 import uuid
 from io import BytesIO
 from hashlib import sha1
-from datetime import timedelta
+from datetime import timedelta, datetime
 from shutil import copyfile
 from PIL import Image
 from django.conf import settings
@@ -179,30 +179,6 @@ def index_source_task(source_id):
         except Media.DoesNotExist:
             media = Media(key=key)
         media.source = source
-        media.metadata = json.dumps(video)
-        upload_date = media.upload_date
-        # Media must have a valid upload date
-        if upload_date:
-            media.published = timezone.make_aware(upload_date)
-        else:
-            log.error(f'Media has no upload date, skipping: {source} / {media}')
-            continue
-        # If the source has a download cap date check the upload date is allowed
-        max_cap_age = source.download_cap_date
-        if max_cap_age:
-            if media.published < max_cap_age:
-                # Media was published after the cap date, skip it
-                log.warn(f'Media: {source} / {media} is older than cap age '
-                         f'{max_cap_age}, skipping')
-                continue
-        # If the source has a cut-off check the upload date is within the allowed delta
-        if source.delete_old_media and source.days_to_keep > 0:
-            delta = timezone.now() - timedelta(days=source.days_to_keep)
-            if media.published < delta:
-                # Media was published after the cutoff date, skip it
-                log.warn(f'Media: {source} / {media} is older than '
-                         f'{source.days_to_keep} days, skipping')
-                continue
         try:
             media.save()
             log.info(f'Indexed media: {source} / {media}')
@@ -234,6 +210,61 @@ def check_source_directory_exists(source_id):
         source.make_directory()
 
 
+@background(schedule=0)
+def download_media_metadata(media_id):
+    '''
+        Downloads the metadata for a media item.
+    '''
+    try:
+        media = Media.objects.get(pk=media_id)
+    except Media.DoesNotExist:
+        # Task triggered but the media no longer exists, do nothing
+        log.error(f'Task download_media_metadata(pk={media_id}) called but no '
+                  f'media exists with ID: {media_id}')
+        return
+    source = media.source
+    metadata = media.index_metadata()
+    media.metadata = json.dumps(metadata)
+    upload_date = media.upload_date
+    # Media must have a valid upload date
+    if upload_date:
+        media.published = timezone.make_aware(upload_date)
+    else:
+        log.error(f'Media has no upload date, skipping: {source} / {media}')
+        media.skip = True
+    # If the source has a download cap date check the upload date is allowed
+    max_cap_age = source.download_cap_date
+    if max_cap_age:
+        if media.published < max_cap_age:
+            # Media was published after the cap date, skip it
+            log.warn(f'Media: {source} / {media} is older than cap age '
+                     f'{max_cap_age}, skipping')
+            media.skip = True
+    # If the source has a cut-off check the upload date is within the allowed delta
+    if source.delete_old_media and source.days_to_keep > 0:
+        if not isinstance(media.published, datetime):
+            # Media has no known published date or incomplete metadata
+            log.warn(f'Media: {source} / {media} has no published date, skipping')
+            media.skip = True
+        else:
+            delta = timezone.now() - timedelta(days=source.days_to_keep)
+            if media.published < delta:
+                # Media was published after the cutoff date, skip it
+                log.warn(f'Media: {source} / {media} is older than '
+                         f'{source.days_to_keep} days, skipping')
+                media.skip = True
+    # Check we can download the media item
+    if not media.skip:
+        if media.get_format_str():
+            media.can_download = True
+        else:
+            media.can_download = False
+    # Save the media
+    media.save()
+    log.info(f'Saved {len(media.metadata)} bytes of metadata for: '
+             f'{source} / {media_id}')
+
+
 @background(schedule=0)
 def download_media_thumbnail(media_id, url):
     '''
@@ -282,6 +313,17 @@ def download_media(media_id):
         log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but it '
                  f'is now marked to be skipped, not downloading')
         return
+    if media.downloaded and media.media_file:
+        # Media has been marked as downloaded before the download_media task was fired,
+        # skip it
+        log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but it '
+                 f'has already been marked as downloaded, not downloading again')
+        return
+    if not media.source.download_media:
+        log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but the '
+                 f'source {media.source} has since been marked to not download media, '
+                 f'not downloading')
+        return
     filepath = media.filepath
     log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"')
     format_str, container = media.download_media()
@@ -315,7 +357,7 @@ def download_media(media_id):
             media.downloaded_audio_codec = cformat['acodec']
             if cformat['vcodec']:
                 # Combined
-                media.downloaded_format = vformat['format']
+                media.downloaded_format = cformat['format']
                 media.downloaded_height = cformat['height']
                 media.downloaded_width = cformat['width']
                 media.downloaded_video_codec = cformat['vcodec']

tubesync/sync/templates/sync/media-item.html

@@ -64,8 +64,14 @@
         <td class="hide-on-small-only">Fallback</td>
         <td><span class="hide-on-med-and-up">Fallback<br></span><strong>{{ media.source.get_fallback_display }}</strong></td>
       </tr>
+      {% if not media.source.download_media %}
+      <tr title="Is media marked to be downloaded at the source?">
+        <td class="hide-on-small-only">Source download?</td>
+        <td><span class="hide-on-med-and-up">Source download?<br></span><strong>{% if media.source.download_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
+      {% endif %}
       {% if media.skip %}
-      <tr title="Has the media been downloaded?">
+      <tr title="Is the media marked to be skipped?">
         <td class="hide-on-small-only">Skipping?</td>
         <td><span class="hide-on-med-and-up">Skipping?<br></span><strong>{% if media.skip %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
@@ -109,7 +115,7 @@
       {% else %}
       <tr title="Can the media be downloaded?">
         <td class="hide-on-small-only">Can download?</td>
-        <td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if youtube_dl_format %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+        <td><span class="hide-on-med-and-up">Can download?<br></span><strong>{% if media.can_download %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
       </tr>
       {% endif %}
       <tr title="The available media formats">

tubesync/sync/templates/sync/media.html

@@ -4,9 +4,16 @@
 
 {% block content %}
 <div class="row">
-  <div class="col s12">
+  <div class="col s12 m9">
     <h1 class="truncate">Media</h1>
   </div>
+  <div class="col s12 m3">
+    {% if show_skipped %}
+    <a href="{% url 'sync:media' %}{% if source %}?filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye-slash"></i> Hide skipped media</a>
+    {% else %}
+    <a href="{% url 'sync:media' %}?show_skipped=yes{% if source %}&filter={{ source.pk }}{% endif %}" class="btn"><i class="far fa-eye"></i> Show skipped media</a>
+    {% endif %}
+  </div>
 </div>
 {% include 'infobox.html' with message=message %}
 <div class="row no-margin-bottom">
@@ -24,8 +31,12 @@
             {% else %}
               {% if m.skip %}
               <span class="error-text"><i class="fas fa-times" title="Skipping media"></i> Skipped</span>
+              {% elif not m.source.download_media %}
+              <span class="error-text"><i class="fas fa-times" title="Not downloading media for this source"></i> Disabled at source</span>
+              {% elif not m.has_metadata %}
+              <i class="far fa-clock" title="Waiting for metadata"></i> Fetching metadata
               {% elif m.can_download %}
-              <i class="far fa-clock" title="Waiting to download or downloading"></i> {{ m.published|date:'Y-m-d' }}
+              <i class="far fa-clock" title="Waiting to download or downloading"></i> Downloading
               {% else %}
               <span class="error-text"><i class="fas fa-exclamation-triangle" title="No matching formats to download"></i> No matching formats</span>
               {% endif %}
@@ -44,5 +55,5 @@
   </div>
   {% endfor %}
 </div>
-{% include 'pagination.html' with pagination=sources.paginator filter=source.pk %}
+{% include 'pagination.html' with pagination=sources.paginator filter=source.pk show_skipped=show_skipped %}
 {% endblock %}

tubesync/sync/templates/sync/source.html

@@ -61,6 +61,10 @@
         <td class="hide-on-small-only">Index schedule</td>
         <td><span class="hide-on-med-and-up">Index schedule<br></span><strong>{{ source.get_index_schedule_display }}</strong></td>
       </tr>
+      <tr title="Download media from this source">
+        <td class="hide-on-small-only">Download media?</td>
+        <td><span class="hide-on-med-and-up">Download media?<br></span><strong>{% if source.download_media %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
+      </tr>
       <tr title="When then source was created locally in TubeSync">
         <td class="hide-on-small-only">Created</td>
         <td><span class="hide-on-med-and-up">Created<br></span><strong>{{ source.created|date:'Y-m-d H:i:s' }}</strong></td>

tubesync/sync/templates/sync/sources.html

@@ -30,7 +30,7 @@
         {% if source.has_failed %}
         <span class="error-text"><i class="fas fa-exclamation-triangle"></i> <strong>Source has permanent failures</strong></span>
         {% else %}
-        <strong>{{ source.media_count }}</strong> media items{% if source.delete_old_media and source.days_to_keep > 0 %}, keep {{ source.days_to_keep }} days of media{% endif %}
+        <strong>{{ source.media_count }}</strong> media items, <strong>{{ source.downloaded_count }}</strong> downloaded{% if source.delete_old_media and source.days_to_keep > 0 %}, keeping {{ source.days_to_keep }} days of media{% endif %}
         {% endif %}
       </a>
     {% empty %}

tubesync/sync/views.py

@@ -10,7 +10,7 @@ from django.views.generic.detail import SingleObjectMixin
 from django.http import HttpResponse
 from django.urls import reverse_lazy
 from django.db import IntegrityError
-from django.db.models import Q, Count, Sum
+from django.db.models import Q, Count, Sum, When, Case
 from django.forms import ValidationError
 from django.utils.text import slugify
 from django.utils import timezone
@@ -104,7 +104,10 @@ class SourcesView(ListView):
 
     def get_queryset(self):
         all_sources = Source.objects.all().order_by('name')
-        return all_sources.annotate(media_count=Count('media_source'))
+        return all_sources.annotate(
+            media_count=Count('media_source'),
+            downloaded_count=Count(Case(When(media_source__downloaded=True, then=1)))
+        )
 
     def get_context_data(self, *args, **kwargs):
         data = super().get_context_data(*args, **kwargs)
@@ -274,9 +277,9 @@ class AddSourceView(CreateView):
     template_name = 'sync/source-add.html'
     model = Source
     fields = ('source_type', 'key', 'name', 'directory', 'media_format',
-              'index_schedule', 'download_cap', 'delete_old_media', 'days_to_keep',
-              'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
-              'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
+              'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
+              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
+              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
     errors = {
         'invalid_media_format': _('Invalid media format, the media format contains '
                                   'errors or is empty. Check the table at the end of '
@@ -365,9 +368,9 @@ class UpdateSourceView(UpdateView):
     template_name = 'sync/source-update.html'
     model = Source
     fields = ('source_type', 'key', 'name', 'directory', 'media_format',
-              'index_schedule', 'download_cap', 'delete_old_media', 'days_to_keep',
-              'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
-              'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
+              'index_schedule', 'download_media', 'download_cap', 'delete_old_media',
+              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
+              'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_thumbnails', 'write_nfo')
     errors = {
         'invalid_media_format': _('Invalid media format, the media format contains '
                                   'errors or is empty. Check the table at the end of '
@@ -438,6 +441,7 @@ class MediaView(ListView):
 
     def __init__(self, *args, **kwargs):
         self.filter_source = None
+        self.show_skipped = False
         super().__init__(*args, **kwargs)
 
     def dispatch(self, request, *args, **kwargs):
@@ -447,13 +451,22 @@ class MediaView(ListView):
                 self.filter_source = Source.objects.get(pk=filter_by)
             except Source.DoesNotExist:
                 self.filter_source = None
+        show_skipped = request.GET.get('show_skipped', '').strip()
+        if show_skipped == 'yes':
+            self.show_skipped = True
         return super().dispatch(request, *args, **kwargs)
 
     def get_queryset(self):
         if self.filter_source:
-            q = Media.objects.filter(source=self.filter_source)
+            if self.show_skipped:
+                q = Media.objects.filter(source=self.filter_source)
+            else:
+                q = Media.objects.filter(source=self.filter_source, skip=False)
         else:
-            q = Media.objects.all()
+            if self.show_skipped:
+                q = Media.objects.all()
+            else:
+                q = Media.objects.filter(skip=False)
         return q.order_by('-published', '-created')
 
     def get_context_data(self, *args, **kwargs):
@@ -464,6 +477,7 @@ class MediaView(ListView):
             message = str(self.messages.get('filter', ''))
             data['message'] = message.format(name=self.filter_source.name)
             data['source'] = self.filter_source
+        data['show_skipped'] = self.show_skipped
         return data
 
 
@@ -1011,7 +1025,7 @@ class UpdateMediaServerView(FormView, SingleObjectMixin):
 
     def get_context_data(self, *args, **kwargs):
         data = super().get_context_data(*args, **kwargs)
-        data['server_help'] = self.object.help_html
+        data['server_help'] = self.object.get_help_html
         return data
 
     def get_success_url(self):

tubesync/sync/youtube.py

@@ -37,7 +37,8 @@ def get_media_info(url):
         'skip_download': True,
         'forcejson': True,
         'simulate': True,
-        'logger': log
+        'logger': log,
+        'extract_flat': True,
     })
     response = {}
     with youtube_dl.YoutubeDL(opts) as y:
@@ -45,6 +46,11 @@ def get_media_info(url):
             response = y.extract_info(url, download=False)
         except youtube_dl.utils.DownloadError as e:
             raise YouTubeError(f'Failed to extract_info for "{url}": {e}') from e
+    if not response:
+        raise YouTubeError(f'Failed to extract_info for "{url}": No metadata was '
+                           f'returned by youtube-dl, check for error messages in the '
+                           f'logs above. This task will be retried later with an '
+                           f'exponential backoff.')
     return response
 
 

tubesync/tubesync/local_settings.py.container

@@ -28,7 +28,25 @@ DATABASES = {
     }
 }
 
+DEFAULT_THREADS = 1
+MAX_BACKGROUND_TASK_ASYNC_THREADS = 8
+BACKGROUND_TASK_ASYNC_THREADS = int(os.getenv('TUBESYNC_WORKERS', DEFAULT_THREADS))
+if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS:
+    BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS
+
 
 MEDIA_ROOT = CONFIG_BASE_DIR / 'media'
 DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR
 YOUTUBE_DL_CACHEDIR = CONFIG_BASE_DIR / 'cache'
+
+
+BASICAUTH_USERNAME = os.getenv('HTTP_USER', '').strip()
+BASICAUTH_PASSWORD = os.getenv('HTTP_PASS', '').strip()
+if BASICAUTH_USERNAME and BASICAUTH_PASSWORD:
+    BASICAUTH_DISABLE = False
+    BASICAUTH_USERS = {
+        BASICAUTH_USERNAME: BASICAUTH_PASSWORD,
+    }
+else:
+    BASICAUTH_DISABLE = True
+    BASICAUTH_USERS = {}

tubesync/tubesync/settings.py

@@ -6,7 +6,7 @@ CONFIG_BASE_DIR = BASE_DIR
 DOWNLOADS_BASE_DIR = BASE_DIR
 
 
-VERSION = 0.6
+VERSION = '0.9.1'
 SECRET_KEY = ''
 DEBUG = False
 ALLOWED_HOSTS = []
@@ -37,6 +37,7 @@ MIDDLEWARE = [
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'whitenoise.middleware.WhiteNoiseMiddleware',
     'common.middleware.MaterializeDefaultFieldsMiddleware',
+    'common.middleware.BasicAuthMiddleware',
 ]
 
 
@@ -114,20 +115,30 @@ Disallow: /
 '''.strip()
 
 
+X_FRAME_OPTIONS = 'SAMEORIGIN'
+
+
+BASICAUTH_DISABLE = True
+BASICAUTH_REALM = 'Authenticate to TubeSync'
+BASICAUTH_ALWAYS_ALLOW_URIS = ('/healthcheck',)
+BASICAUTH_USERS = {}
+
+
 HEALTHCHECK_FIREWALL = True
 HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',)
 
 
-MAX_ATTEMPTS = 10                           # Number of times tasks will be retried
+MAX_ATTEMPTS = 15                           # Number of times tasks will be retried
 MAX_RUN_TIME = 1800                         # Maximum amount of time in seconds a task can run
-BACKGROUND_TASK_RUN_ASYNC = False           # Run tasks async in the background
+BACKGROUND_TASK_RUN_ASYNC = True            # Run tasks async in the background
 BACKGROUND_TASK_ASYNC_THREADS = 1           # Number of async tasks to run at once
+MAX_BACKGROUND_TASK_ASYNC_THREADS = 8       # For sanity reasons
 BACKGROUND_TASK_PRIORITY_ORDERING = 'ASC'   # Use 'niceness' task priority ordering
 COMPLETED_TASKS_DAYS_TO_KEEP = 7            # Number of days to keep completed tasks
 
 
 SOURCES_PER_PAGE = 100
-MEDIA_PER_PAGE = 72
+MEDIA_PER_PAGE = 144
 TASKS_PER_PAGE = 100
 
 
@@ -148,7 +159,7 @@ YOUTUBE_DEFAULTS = {
 }
 
 
-MEDIA_FORMATSTR_DEFAULT = '{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}'
+MEDIA_FORMATSTR_DEFAULT = '{yyyy_mm_dd}_{source}_{title}_{key}_{format}.{ext}'
 
 
 try: