diff --git a/README.md b/README.md index 2ab0005..47bfeb8 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,14 @@ every hour" or similar short interval it's entirely possible your TubeSync insta spend its entire time just indexing the massive channel over and over again without downloading any media. Check your tasks for the status of your TubeSync install. +If you add a significant amount of "work" due to adding many large channels you may +need to increase the number of background workers by setting the `TUBESYNC_WORKERS` +environment variable. Try around ~4 at most, although the absolute maximum allowed is 8. + +**Be nice.** it's likely entirely possible your IP address could get throttled by the +source if you try and crawl extremely large amounts very quickly. **Try and be polite +with the smallest amount of indexing and concurrent downloads possible for your needs.** + # FAQ @@ -300,15 +308,16 @@ There are a number of other environment variables you can set. These are, mostly **NOT** required to be set in the default container installation, they are really only useful if you are manually installing TubeSync in some other environment. These are: -| Name | What | Example | -| ------------------------ | ------------------------------------- | ---------------------------------- | -| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | -| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME | /somepath | -| TUBESYNC_DEBUG | Enable debugging | True | -| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS | tubesync.example.com,otherhost.com | -| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | -| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | -| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | +| Name | What | Example | +| ------------------------ | ------------------------------------------------------------ | ---------------------------------- | +| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | +| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME | /somepath | +| TUBESYNC_DEBUG | Enable debugging | True | +| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 | +| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS | tubesync.example.com,otherhost.com | +| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | +| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | +| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | # Manual, non-containerised, installation diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 9a61395..d1d36c5 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -28,6 +28,11 @@ DATABASES = { } } +DEFAULT_THREADS = 2 +BACKGROUND_TASK_ASYNC_THREADS = int(os.getenv('TUBESYNC_WORKERS', DEFAULT_THREADS)) +if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS: + BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS + MEDIA_ROOT = CONFIG_BASE_DIR / 'media' DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 37580fa..162d1d8 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -120,8 +120,9 @@ HEALTHCHECK_ALLOWED_IPS = ('127.0.0.1',) MAX_ATTEMPTS = 10 # Number of times tasks will be retried MAX_RUN_TIME = 1800 # Maximum amount of time in seconds a task can run -BACKGROUND_TASK_RUN_ASYNC = False # Run tasks async in the background +BACKGROUND_TASK_RUN_ASYNC = True # Run tasks async in the background BACKGROUND_TASK_ASYNC_THREADS = 1 # Number of async tasks to run at once +MAX_BACKGROUND_TASK_ASYNC_THREADS = 8 # For sanity reasons BACKGROUND_TASK_PRIORITY_ORDERING = 'ASC' # Use 'niceness' task priority ordering COMPLETED_TASKS_DAYS_TO_KEEP = 7 # Number of days to keep completed tasks