bump to v0.8

fix form label padding from overlapping field, resolves #37
fix bug handling audio fallback detection, resolves #31
2021-01-20 18:13:28 +11:00 · 2021-01-20 18:03:24 +11:00 · 2021-01-20 18:00:28 +11:00 · 2021-01-20 17:42:16 +11:00 · 2021-01-20 17:34:50 +11:00 · 2021-01-20 17:34:19 +11:00
13 changed files with 132 additions and 1269 deletions
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -39,11 +39,11 @@
        },
        "django": {
            "hashes": [
-                "sha256:5c866205f15e7a7123f1eec6ab939d22d5bde1416635cab259684af66d8e48a2",
-                "sha256:edb10b5c45e7e9c0fb1dc00b76ec7449aca258a39ffd613dbd078c51d19c9f03"
+                "sha256:2d78425ba74c7a1a74b196058b261b9733a8570782f4e2828974777ccca7edf7",
+                "sha256:efa2ab96b33b20c2182db93147a0c3cd7769d418926f9e9f140a60dca7c64ca9"
            ],
            "index": "pypi",
-            "version": "==3.1.4"
+            "version": "==3.1.5"
        },
        "django-appconf": {
            "hashes": [
@@ -134,44 +134,48 @@
        },
        "pillow": {
            "hashes": [
-                "sha256:006de60d7580d81f4a1a7e9f0173dc90a932e3905cc4d47ea909bc946302311a",
-                "sha256:0a2e8d03787ec7ad71dc18aec9367c946ef8ef50e1e78c71f743bc3a770f9fae",
-                "sha256:0eeeae397e5a79dc088d8297a4c2c6f901f8fb30db47795113a4a605d0f1e5ce",
-                "sha256:11c5c6e9b02c9dac08af04f093eb5a2f84857df70a7d4a6a6ad461aca803fb9e",
-                "sha256:2fb113757a369a6cdb189f8df3226e995acfed0a8919a72416626af1a0a71140",
-                "sha256:4b0ef2470c4979e345e4e0cc1bbac65fda11d0d7b789dbac035e4c6ce3f98adb",
-                "sha256:59e903ca800c8cfd1ebe482349ec7c35687b95e98cefae213e271c8c7fffa021",
-                "sha256:5abd653a23c35d980b332bc0431d39663b1709d64142e3652890df4c9b6970f6",
-                "sha256:5f9403af9c790cc18411ea398a6950ee2def2a830ad0cfe6dc9122e6d528b302",
-                "sha256:6b4a8fd632b4ebee28282a9fef4c341835a1aa8671e2770b6f89adc8e8c2703c",
-                "sha256:6c1aca8231625115104a06e4389fcd9ec88f0c9befbabd80dc206c35561be271",
-                "sha256:795e91a60f291e75de2e20e6bdd67770f793c8605b553cb6e4387ce0cb302e09",
-                "sha256:7ba0ba61252ab23052e642abdb17fd08fdcfdbbf3b74c969a30c58ac1ade7cd3",
-                "sha256:7c9401e68730d6c4245b8e361d3d13e1035cbc94db86b49dc7da8bec235d0015",
-                "sha256:81f812d8f5e8a09b246515fac141e9d10113229bc33ea073fec11403b016bcf3",
-                "sha256:895d54c0ddc78a478c80f9c438579ac15f3e27bf442c2a9aa74d41d0e4d12544",
-                "sha256:8de332053707c80963b589b22f8e0229f1be1f3ca862a932c1bcd48dafb18dd8",
-                "sha256:92c882b70a40c79de9f5294dc99390671e07fc0b0113d472cbea3fde15db1792",
-                "sha256:95edb1ed513e68bddc2aee3de66ceaf743590bf16c023fb9977adc4be15bd3f0",
-                "sha256:b63d4ff734263ae4ce6593798bcfee6dbfb00523c82753a3a03cbc05555a9cc3",
-                "sha256:bd7bf289e05470b1bc74889d1466d9ad4a56d201f24397557b6f65c24a6844b8",
-                "sha256:cc3ea6b23954da84dbee8025c616040d9aa5eaf34ea6895a0a762ee9d3e12e11",
-                "sha256:cc9ec588c6ef3a1325fa032ec14d97b7309db493782ea8c304666fb10c3bd9a7",
-                "sha256:d3d07c86d4efa1facdf32aa878bd508c0dc4f87c48125cc16b937baa4e5b5e11",
-                "sha256:d8a96747df78cda35980905bf26e72960cba6d355ace4780d4bdde3b217cdf1e",
-                "sha256:e38d58d9138ef972fceb7aeec4be02e3f01d383723965bfcef14d174c8ccd039",
-                "sha256:eb472586374dc66b31e36e14720747595c2b265ae962987261f044e5cce644b5",
-                "sha256:fbd922f702582cb0d71ef94442bfca57624352622d75e3be7a1e7e9360b07e72"
+                "sha256:165c88bc9d8dba670110c689e3cc5c71dbe4bfb984ffa7cbebf1fac9554071d6",
+                "sha256:1d208e670abfeb41b6143537a681299ef86e92d2a3dac299d3cd6830d5c7bded",
+                "sha256:22d070ca2e60c99929ef274cfced04294d2368193e935c5d6febfd8b601bf865",
+                "sha256:2353834b2c49b95e1313fb34edf18fca4d57446675d05298bb694bca4b194174",
+                "sha256:39725acf2d2e9c17356e6835dccebe7a697db55f25a09207e38b835d5e1bc032",
+                "sha256:3de6b2ee4f78c6b3d89d184ade5d8fa68af0848f9b6b6da2b9ab7943ec46971a",
+                "sha256:47c0d93ee9c8b181f353dbead6530b26980fe4f5485aa18be8f1fd3c3cbc685e",
+                "sha256:5e2fe3bb2363b862671eba632537cd3a823847db4d98be95690b7e382f3d6378",
+                "sha256:604815c55fd92e735f9738f65dabf4edc3e79f88541c221d292faec1904a4b17",
+                "sha256:6c5275bd82711cd3dcd0af8ce0bb99113ae8911fc2952805f1d012de7d600a4c",
+                "sha256:731ca5aabe9085160cf68b2dbef95fc1991015bc0a3a6ea46a371ab88f3d0913",
+                "sha256:7612520e5e1a371d77e1d1ca3a3ee6227eef00d0a9cddb4ef7ecb0b7396eddf7",
+                "sha256:7916cbc94f1c6b1301ac04510d0881b9e9feb20ae34094d3615a8a7c3db0dcc0",
+                "sha256:81c3fa9a75d9f1afafdb916d5995633f319db09bd773cb56b8e39f1e98d90820",
+                "sha256:887668e792b7edbfb1d3c9d8b5d8c859269a0f0eba4dda562adb95500f60dbba",
+                "sha256:93a473b53cc6e0b3ce6bf51b1b95b7b1e7e6084be3a07e40f79b42e83503fbf2",
+                "sha256:96d4dc103d1a0fa6d47c6c55a47de5f5dafd5ef0114fa10c85a1fd8e0216284b",
+                "sha256:a3d3e086474ef12ef13d42e5f9b7bbf09d39cf6bd4940f982263d6954b13f6a9",
+                "sha256:b02a0b9f332086657852b1f7cb380f6a42403a6d9c42a4c34a561aa4530d5234",
+                "sha256:b09e10ec453de97f9a23a5aa5e30b334195e8d2ddd1ce76cc32e52ba63c8b31d",
+                "sha256:b6f00ad5ebe846cc91763b1d0c6d30a8042e02b2316e27b05de04fa6ec831ec5",
+                "sha256:bba80df38cfc17f490ec651c73bb37cd896bc2400cfba27d078c2135223c1206",
+                "sha256:c3d911614b008e8a576b8e5303e3db29224b455d3d66d1b2848ba6ca83f9ece9",
+                "sha256:ca20739e303254287138234485579b28cb0d524401f83d5129b5ff9d606cb0a8",
+                "sha256:cb192176b477d49b0a327b2a5a4979552b7a58cd42037034316b8018ac3ebb59",
+                "sha256:cdbbe7dff4a677fb555a54f9bc0450f2a21a93c5ba2b44e09e54fcb72d2bd13d",
+                "sha256:cf6e33d92b1526190a1de904df21663c46a456758c0424e4f947ae9aa6088bf7",
+                "sha256:d355502dce85ade85a2511b40b4c61a128902f246504f7de29bbeec1ae27933a",
+                "sha256:d673c4990acd016229a5c1c4ee8a9e6d8f481b27ade5fc3d95938697fa443ce0",
+                "sha256:dc577f4cfdda354db3ae37a572428a90ffdbe4e51eda7849bf442fb803f09c9b",
+                "sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d",
+                "sha256:f50e7a98b0453f39000619d845be8b06e611e56ee6e8186f7f60c3b1e2f0feae"
            ],
            "index": "pypi",
-            "version": "==8.0.1"
+            "version": "==8.1.0"
        },
        "pytz": {
            "hashes": [
-                "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",
-                "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"
+                "sha256:16962c5fb8db4a8f63a26646d8886e9d769b6c511543557bc84e9569fb9a9cb4",
+                "sha256:180befebb1927b16f6b57101720075a984c019ac16b1b7575673bea42c6c3da5"
            ],
-            "version": "==2020.4"
+            "version": "==2020.5"
        },
        "rcssmin": {
            "hashes": [
@@ -236,11 +240,11 @@
        },
        "youtube-dl": {
            "hashes": [
-                "sha256:65968065e66966955dc79fad9251565fcc982566118756da624bd21467f3a04c",
-                "sha256:eaa859f15b6897bec21474b7787dc958118c8088e1f24d4ef1d58eab13188958"
+                "sha256:8f421ca8394d2529e06225e44ec66538d2a28f6f340c03065776894bf3d24ea6",
+                "sha256:acf74701a31b6c3d06f9d4245a46ba8fb6c378931681177412043c6e8276fee7"
            ],
            "index": "pypi",
-            "version": "==2020.12.14"
+            "version": "==2021.1.16"
        }
    },
    "develop": {}
--- a/README.md
+++ b/README.md
@@ -22,9 +22,12 @@ hopefully, quite reliable.
 # Latest container image

 ```yaml
-ghcr.io/meeb/tubesync:v0.6
+ghcr.io/meeb/tubesync:v0.8
 ```

+**NOTE: the `:latest` tag does exist, but will contain in-development commits and may
+be broken. Use at your own risk.**
+
 # Screenshots

 ### Dashboard
@@ -99,7 +102,7 @@ Finally, download and run the container:

 ```bash
 # Pull a versioned image
-$ docker pull ghcr.io/meeb/tubesync:v0.6
+$ docker pull ghcr.io/meeb/tubesync:v0.8
 # Start the container using your user ID and group ID
 $ docker run \
  -d \
@@ -110,7 +113,7 @@ $ docker run \
  -v /some/directory/tubesync-config:/config \
  -v /some/directory/tubesync-downloads:/downloads \
  -p 4848:4848 \
-  ghcr.io/meeb/tubesync:v0.6
+  ghcr.io/meeb/tubesync:v0.8
 ```

 Once running, open `http://localhost:4848` in your browser and you should see the
@@ -122,7 +125,7 @@ Alternatively, for Docker Compose, you can use something like:

 ```yaml
  tubesync:
-    image: ghcr.io/meeb/tubesync:v0.6
+    image: ghcr.io/meeb/tubesync:v0.8
    container_name: tubesync
    restart: unless-stopped
    ports:
@@ -210,6 +213,14 @@ every hour" or similar short interval it's entirely possible your TubeSync insta
 spend its entire time just indexing the massive channel over and over again without
 downloading any media. Check your tasks for the status of your TubeSync install.

+If you add a significant amount of "work" due to adding many large channels you may
+need to increase the number of background workers by setting the `TUBESYNC_WORKERS`
+environment variable. Try around ~4 at most, although the absolute maximum allowed is 8.
+
+**Be nice.** it's likely entirely possible your IP address could get throttled by the
+source if you try and crawl extremely large amounts very quickly. **Try and be polite
+with the smallest amount of indexing and concurrent downloads possible for your needs.**
+

 # FAQ

@@ -298,10 +309,11 @@ There are a number of other environment variables you can set. These are, mostly
 useful if you are manually installing TubeSync in some other environment. These are:

 | Name                     | What                                                         | Example                            |
-| ------------------------ | ------------------------------------- | ---------------------------------- |
+| ------------------------ | ------------------------------------------------------------ | ---------------------------------- |
 | DJANGO_SECRET_KEY        | Django's SECRET_KEY                                          | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l   |
 | DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME                                   | /somepath                          |
 | TUBESYNC_DEBUG           | Enable debugging                                             | True                               |
+| TUBESYNC_WORKERS         | Number of background workers, default is 2, max allowed is 8 | 2                                  |
 | TUBESYNC_HOSTS           | Django's ALLOWED_HOSTS                                       | tubesync.example.com,otherhost.com |
 | GUNICORN_WORKERS         | Number of gunicorn workers to spawn                          | 3                                  |
 | LISTEN_HOST              | IP address for gunicorn to listen on                         | 127.0.0.1                          |
--- a/tubesync/common/static/styles/_forms.scss
+++ b/tubesync/common/static/styles/_forms.scss
@@ -4,7 +4,7 @@
    }
    .help-text {
        color: $form-help-text-colour;
-        padding: 1rem 0 1rem 0;
+        padding-bottom: 1rem;
    }
    label {
        text-transform: uppercase;
--- a/tubesync/common/static/styles/_template.scss
+++ b/tubesync/common/static/styles/_template.scss
@@ -5,6 +5,13 @@ html {
    color: $text-colour;
 }

+body {
+    display: flex;
+    min-height: 100vh;
+    flex-direction: column;
+    justify-content: space-between;
+}
+
 header {

    background-color: $header-background-colour;
--- a/tubesync/common/templates/base.html
+++ b/tubesync/common/templates/base.html
@@ -16,6 +16,8 @@

  <body>

+    <div class="app">
+
      <header>
        <div class="container">
          <a href="{% url 'sync:dashboard' %}">
@@ -43,6 +45,8 @@
        </div>
      </main>

+    </div>
+
    <footer>
      <div class="container">
        <p>
--- a/tubesync/common/utils.py
+++ b/tubesync/common/utils.py
@@ -14,3 +14,13 @@ def append_uri_params(uri, params):
    uri = str(uri)
    qs = urlencode(params)
    return urlunsplit(('', '', uri, qs, ''))
+
+
+def clean_filename(filename):
+    if not isinstance(filename, str):
+        raise ValueError(f'filename must be a str, got {type(filename)}')
+    to_scrub = '<>\/:*?"|'
+    for char in to_scrub:
+        filename = filename.replace(char, '')
+    filename = ''.join([c for c in filename if ord(c) > 30])
+    return ' '.join(filename.split())
--- a/tubesync/spam
+++ b/tubesync/spam
--- a/tubesync/sync/matching.py
+++ b/tubesync/sync/matching.py
@@ -66,7 +66,7 @@ def get_best_audio_format(media):
    # No codecs matched
    if media.source.can_fallback:
        # Can fallback, find the next highest bitrate non-matching codec
-        return False, audio_formats[0]
+        return False, audio_formats[0]['id']
    else:
        # Can't fallback
        return False, False
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -12,6 +12,7 @@ from django.utils.text import slugify
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from common.errors import NoFormatException
+from common.utils import clean_filename
 from .youtube import (get_media_info as get_youtube_media_info,
                      download_media as download_youtube_media)
 from .utils import seconds_to_timestr, parse_media_format
@@ -200,7 +201,7 @@ class Source(models.Model):
        _('media format'),
        max_length=200,
        default=settings.MEDIA_FORMATSTR_DEFAULT,
-        help_text=_('File format to use for saving files')
+        help_text=_('File format to use for saving files, detailed options at bottom of page.')
    )
    index_schedule = models.IntegerField(
        _('index schedule'),
@@ -838,6 +839,7 @@ class Media(models.Model):
            fmt.append(resolution)
            vcodec = vformat['vcodec'].lower()
            fmt.append(vcodec)
+        if aformat:
            acodec = aformat['acodec'].lower()
            fmt.append(acodec)
        if vformat:
@@ -887,7 +889,7 @@ class Media(models.Model):
            'source': self.source.slugname,
            'source_full': self.source.name,
            'title': self.slugtitle,
-            'title_full': self.title,
+            'title_full': clean_filename(self.title),
            'key': self.key,
            'format': '-'.join(display_format['format']),
            'playlist_index': self.playlist_index,
@@ -1005,7 +1007,7 @@ class Media(models.Model):

    @property
    def filename(self):
-        # Otherwise, create a suitable filename from the source media_format
+        # Create a suitable filename from the source media_format
        media_format = str(self.source.media_format)
        media_details = self.format_dict
        return media_format.format(**media_details)
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -315,7 +315,7 @@ def download_media(media_id):
            media.downloaded_audio_codec = cformat['acodec']
            if cformat['vcodec']:
                # Combined
-                media.downloaded_format = vformat['format']
+                media.downloaded_format = cformat['format']
                media.downloaded_height = cformat['height']
                media.downloaded_width = cformat['width']
                media.downloaded_video_codec = cformat['vcodec']
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -1011,7 +1011,7 @@ class UpdateMediaServerView(FormView, SingleObjectMixin):

    def get_context_data(self, *args, **kwargs):
        data = super().get_context_data(*args, **kwargs)
-        data['server_help'] = self.object.help_html
+        data['server_help'] = self.object.get_help_html
        return data

    def get_success_url(self):
--- a/tubesync/tubesync/local_settings.py.container
+++ b/tubesync/tubesync/local_settings.py.container
@@ -28,6 +28,12 @@ DATABASES = {
    }
 }

+DEFAULT_THREADS = 1
+MAX_BACKGROUND_TASK_ASYNC_THREADS = 8
+BACKGROUND_TASK_ASYNC_THREADS = int(os.getenv('TUBESYNC_WORKERS', DEFAULT_THREADS))
+if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS:
+    BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS
+

 MEDIA_ROOT = CONFIG_BASE_DIR / 'media'
 DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR
--- a/tubesync/tubesync/settings.py
+++ b/tubesync/tubesync/settings.py
@@ -6,7 +6,7 @@ CONFIG_BASE_DIR = BASE_DIR
 DOWNLOADS_BASE_DIR = BASE_DIR


-VERSION = 0.6
+VERSION = 0.8
 SECRET_KEY = ''
 DEBUG = False
 ALLOWED_HOSTS = []
@@ -120,8 +120,9 @@ HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',)

 MAX_ATTEMPTS = 10                           # Number of times tasks will be retried
 MAX_RUN_TIME = 1800                         # Maximum amount of time in seconds a task can run
-BACKGROUND_TASK_RUN_ASYNC = False           # Run tasks async in the background
+BACKGROUND_TASK_RUN_ASYNC = True            # Run tasks async in the background
 BACKGROUND_TASK_ASYNC_THREADS = 1           # Number of async tasks to run at once
+MAX_BACKGROUND_TASK_ASYNC_THREADS = 8       # For sanity reasons
 BACKGROUND_TASK_PRIORITY_ORDERING = 'ASC'   # Use 'niceness' task priority ordering
 COMPLETED_TASKS_DAYS_TO_KEEP = 7            # Number of days to keep completed tasks
Author	SHA1	Message	Date
meeb	ffd69e8d40	bump to v0.8	2021-01-20 18:13:28 +11:00
meeb	eebef3371f	fix form label padding from overlapping field, resolves #37	2021-01-20 18:03:24 +11:00
meeb	4cd6701c8a	fix bug handling audio fallback detection, resolves #31	2021-01-20 18:00:28 +11:00
meeb	4ebe6f2a37	pin footer at the bottom of the viewport, resolves #26	2021-01-20 17:42:16 +11:00
meeb	d553d58fde	update youtube-dl	2021-01-20 17:34:50 +11:00
meeb	df40a1367a	sanitise youtube video titles for use in sane filenames, resolves #35	2021-01-20 17:34:19 +11:00
meeb	607ee77e70	update deps	2021-01-16 03:50:16 +11:00
meeb	9af493aa8a	update upstream library versions	2021-01-03 15:18:31 +11:00
meeb	f0c94ff789	Merge pull request #29 from ltomes/main Added format descriptions.	2020-12-30 15:40:41 +11:00
Levi Tomes	39c7799831	Changed PR intent Changed PR intent	2020-12-29 22:36:43 -06:00
Levi Tomes	da7371f830	Added format descriptions Added format descriptions to the media format help_text.	2020-12-29 21:50:04 -06:00
meeb	387cfefc8f	issues with dupe background async worker threads, drop defaults back down to 1 worker	2020-12-21 03:04:54 +11:00
meeb	d92dbde781	bump to 0.7	2020-12-20 18:24:04 +11:00
meeb	e36658e1a1	missing default setting for container build	2020-12-20 17:57:04 +11:00
meeb	51cd942717	use async workers, spawn two workers by default, add TUBESYNC_WORKERS env var to adjust the number of workers, resolves #19	2020-12-20 17:52:45 +11:00
meeb	001554db1a	fix format container name, resolves #22	2020-12-20 13:01:14 +11:00
meeb	7cf86bb98d	fix attribute name, resolves #21	2020-12-20 04:15:43 +11:00
meeb	c28c095f48	add :latest tag warning	2020-12-19 19:29:26 +11:00