From 20df9f4044a3c66f641b2de145efc1b15e0ece56 Mon Sep 17 00:00:00 2001 From: meeb Date: Sun, 4 Apr 2021 23:01:15 +1000 Subject: [PATCH] support external postgresql, mysql and mariadb databases, resolves #72 --- Pipfile | 2 + Pipfile.lock | 54 +++++++++++- README.md | 27 +++--- docs/other-database-backends.md | 69 +++++++++++++++ tubesync/common/errors.py | 7 ++ tubesync/common/tests.py | 39 +++++++++ tubesync/common/utils.py | 83 ++++++++++++++++++- tubesync/sync/templates/sync/dashboard.html | 4 + tubesync/sync/views.py | 1 + tubesync/tubesync/local_settings.py.container | 31 +++++-- 10 files changed, 298 insertions(+), 19 deletions(-) create mode 100644 docs/other-database-backends.md diff --git a/Pipfile b/Pipfile index 7f93ee7..8de307f 100644 --- a/Pipfile +++ b/Pipfile @@ -18,6 +18,8 @@ youtube-dl = "*" django-background-tasks = "*" requests = "*" django-basicauth = "*" +psycopg2-binary = "*" +mysqlclient = "*" [requires] python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index 9d10765..260c217 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f698e2853dec2d325d2d7e752620fc81d911022d394a57f2f8a9349ac2682752" + "sha256": "bea753f24d773d83d202254027b78d83943fa90bf5649d014faf1c09d9eab8b0" }, "pipfile-spec": 6, "requires": { @@ -139,6 +139,17 @@ "index": "pypi", "version": "==0.20.1" }, + "mysqlclient": { + "hashes": [ + "sha256:0ac0dd759c4ca02c35a9fedc24bc982cf75171651e8187c2495ec957a87dfff7", + "sha256:3381ca1a4f37ff1155fcfde20836b46416d66531add8843f6aa6d968982731c3", + "sha256:71c4b330cf2313bbda0307fc858cc9055e64493ba9bf28454d25cf8b3ee8d7f5", + "sha256:f6ebea7c008f155baeefe16c56cd3ee6239f7a5a9ae42396c2f1860f08a7c432", + "sha256:fc575093cf81b6605bed84653e48b277318b880dc9becf42dd47fa11ffd3e2b6" + ], + "index": "pypi", + "version": "==2.0.3" + }, "pillow": { "hashes": [ "sha256:01425106e4e8cee195a411f729cff2a7d61813b0b11737c12bd5991f5f14bcd5", @@ -178,6 +189,47 @@ "index": "pypi", "version": "==8.2.0" }, + "psycopg2-binary": { + "hashes": [ + "sha256:0deac2af1a587ae12836aa07970f5cb91964f05a7c6cdb69d8425ff4c15d4e2c", + "sha256:0e4dc3d5996760104746e6cfcdb519d9d2cd27c738296525d5867ea695774e67", + "sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0", + "sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6", + "sha256:1fabed9ea2acc4efe4671b92c669a213db744d2af8a9fc5d69a8e9bc14b7a9db", + "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94", + "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52", + "sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056", + "sha256:6a32f3a4cb2f6e1a0b15215f448e8ce2da192fd4ff35084d80d5e39da683e79b", + "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd", + "sha256:7d92a09b788cbb1aec325af5fcba9fed7203897bbd9269d5691bb1e3bce29550", + "sha256:833709a5c66ca52f1d21d41865a637223b368c0ee76ea54ca5bad6f2526c7679", + "sha256:89705f45ce07b2dfa806ee84439ec67c5d9a0ef20154e0e475e2b2ed392a5b83", + "sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77", + "sha256:950bc22bb56ee6ff142a2cb9ee980b571dd0912b0334aa3fe0fe3788d860bea2", + "sha256:a0c50db33c32594305b0ef9abc0cb7db13de7621d2cadf8392a1d9b3c437ef77", + "sha256:a0eb43a07386c3f1f1ebb4dc7aafb13f67188eab896e7397aa1ee95a9c884eb2", + "sha256:aaa4213c862f0ef00022751161df35804127b78adf4a2755b9f991a507e425fd", + "sha256:ac0c682111fbf404525dfc0f18a8b5f11be52657d4f96e9fcb75daf4f3984859", + "sha256:ad20d2eb875aaa1ea6d0f2916949f5c08a19c74d05b16ce6ebf6d24f2c9f75d1", + "sha256:b4afc542c0ac0db720cf516dd20c0846f71c248d2b3d21013aa0d4ef9c71ca25", + "sha256:b8a3715b3c4e604bcc94c90a825cd7f5635417453b253499664f784fc4da0152", + "sha256:ba28584e6bca48c59eecbf7efb1576ca214b47f05194646b081717fa628dfddf", + "sha256:ba381aec3a5dc29634f20692349d73f2d21f17653bda1decf0b52b11d694541f", + "sha256:bd1be66dde2b82f80afb9459fc618216753f67109b859a361cf7def5c7968729", + "sha256:c2507d796fca339c8fb03216364cca68d87e037c1f774977c8fc377627d01c71", + "sha256:cec7e622ebc545dbb4564e483dd20e4e404da17ae07e06f3e780b2dacd5cee66", + "sha256:d14b140a4439d816e3b1229a4a525df917d6ea22a0771a2a78332273fd9528a4", + "sha256:d1b4ab59e02d9008efe10ceabd0b31e79519da6fb67f7d8e8977118832d0f449", + "sha256:d5227b229005a696cc67676e24c214740efd90b148de5733419ac9aaba3773da", + "sha256:e1f57aa70d3f7cc6947fd88636a481638263ba04a742b4a37dd25c373e41491a", + "sha256:e74a55f6bad0e7d3968399deb50f61f4db1926acf4a6d83beaaa7df986f48b1c", + "sha256:e82aba2188b9ba309fd8e271702bd0d0fc9148ae3150532bbb474f4590039ffb", + "sha256:ee69dad2c7155756ad114c02db06002f4cded41132cc51378e57aad79cc8e4f4", + "sha256:f5ab93a2cb2d8338b1674be43b442a7f544a0971da062a5da774ed40587f18f5" + ], + "index": "pypi", + "version": "==2.8.6" + }, "pytz": { "hashes": [ "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da", diff --git a/README.md b/README.md index ff4e50d..f959618 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,8 @@ and less common features: ![Reset tasks from the command line](https://github.com/meeb/tubesync/blob/main/docs/reset-tasks.md) +![Using PostgreSQL, MySQL or MariaDB as database backends](https://github.com/meeb/tubesync/blob/main/docs/other-database-backends.md) + # Warnings @@ -357,18 +359,19 @@ There are a number of other environment variables you can set. These are, mostly **NOT** required to be set in the default container installation, they are really only useful if you are manually installing TubeSync in some other environment. These are: -| Name | What | Example | -| ------------------------ | ------------------------------------------------------------ | ---------------------------------- | -| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | -| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME | /somepath | -| TUBESYNC_DEBUG | Enable debugging | True | -| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 | -| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS | tubesync.example.com,otherhost.com | -| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | -| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | -| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | -| HTTP_USER | Sets the username for HTTP basic authentication | some-username | -| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | +| Name | What | Example | +| ------------------------ | ------------------------------------------------------------ | ------------------------------------ | +| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | +| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME | /somepath | +| TUBESYNC_DEBUG | Enable debugging | True | +| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 | +| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS | tubesync.example.com,otherhost.com | +| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | +| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | +| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | +| HTTP_USER | Sets the username for HTTP basic authentication | some-username | +| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | +| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database | # Manual, non-containerised, installation diff --git a/docs/other-database-backends.md b/docs/other-database-backends.md new file mode 100644 index 0000000..41dd9be --- /dev/null +++ b/docs/other-database-backends.md @@ -0,0 +1,69 @@ +# TubeSync + +## Advanced usage guide - using other database backends + +This is a new feature in v1.0 of TubeSync and later. It allows you to use a custom +existing external database server instead of the default SQLite database. You may want +to use this if you encounter performance issues with adding very large or a large +number of channels and database write contention (as shown by errors in the log) +become an issue. + +## Requirements + +TubeSync supports SQLite (the automatic default) as well as PostgreSQL, MySQL and +MariaDB. For MariaDB just follow the MySQL instructions as the driver is the same. + +You should a blank install of TubeSync. Migrating to a new database will reset your +database. If you are comfortable with Django you can export and re-import existing +database data with: + +```bash +$ docker exec -ti tubesync python3 /app/manage.py dumpdata > some-file.json +``` + +Then change you database backend over, then use + +```bash +$ docker exec -ti tubesync python3 /app/manage.py loaddata some-file.json +``` + +As detailed in the Django documentation: + +https://docs.djangoproject.com/en/3.1/ref/django-admin/#dumpdata + +and: + +https://docs.djangoproject.com/en/3.1/ref/django-admin/#loaddata + +Further instructions are beyond the scope of TubeSync documenation and you should refer +to Django documentation for more details. + +If you are not comfortable with the above, then skip the `dumpdata` steps, however +remember you will start again with a completely new database. + +## Steps + +### 1. Create a database in your external database server + +You need to create a database and a user with permissions to access the database in +your chosen external database server. Steps vary between PostgreSQL, MySQL and MariaDB +so this is up to you to work out. + +### 2. Set the database connection string environment variable + +You need to provide the database connection details to TubeSync via an environment +variable. The environment variable name is `DATABASE_CONNECTION` and the format is the +standard URL-style string. Example are: + +`postgresql://tubesync:password@localhost:5432/tubesync` +`mysql://tubesync:password@localhost:3306/tubesync` + +### 3. Start TubeSync and check the logs + +Once you start TubeSync with the new database connection you should see the folling log +entry in the container or stdout logs: + +`2021-04-04 22:42:17,912 [tubesync/INFO] Using database connection: django.db.backends.postgresql://tubesync:[hidden]@localhost:5432/tubesync` + +If you see a line similar to the above and the web interface loads, congratulations, +you are now using an external database server for your TubeSync data! diff --git a/tubesync/common/errors.py b/tubesync/common/errors.py index 9f3eda4..130510a 100644 --- a/tubesync/common/errors.py +++ b/tubesync/common/errors.py @@ -20,3 +20,10 @@ class DownloadFailedException(Exception): exist. ''' pass + + +class DatabaseConnectionError(Exception): + ''' + Raised when parsing or initially connecting to a database. + ''' + pass diff --git a/tubesync/common/tests.py b/tubesync/common/tests.py index 3a12bf1..720324c 100644 --- a/tubesync/common/tests.py +++ b/tubesync/common/tests.py @@ -2,6 +2,8 @@ import os.path from django.conf import settings from django.test import TestCase, Client from .testutils import prevent_request_warnings +from .utils import parse_database_connection_string +from .errors import DatabaseConnectionError class ErrorPageTestCase(TestCase): @@ -61,3 +63,40 @@ class CommonStaticTestCase(TestCase): favicon_real_path = os.path.join(os.sep.join(root_parts), os.sep.join(url_parts)) self.assertTrue(os.path.exists(favicon_real_path)) + + +class DatabaseConnectionTestCase(TestCase): + + def test_parse_database_connection_string(self): + database_dict = parse_database_connection_string( + 'postgresql://tubesync:password@localhost:5432/tubesync') + database_dict = parse_database_connection_string( + 'mysql://tubesync:password@localhost:3306/tubesync') + # Invalid driver + with self.assertRaises(DatabaseConnectionError): + parse_database_connection_string( + 'test://tubesync:password@localhost:5432/tubesync') + # No username + with self.assertRaises(DatabaseConnectionError): + parse_database_connection_string( + 'postgresql://password@localhost:5432/tubesync') + # No database name + with self.assertRaises(DatabaseConnectionError): + parse_database_connection_string( + 'postgresql://tubesync:password@5432') + # Invalid port + with self.assertRaises(DatabaseConnectionError): + parse_database_connection_string( + 'postgresql://tubesync:password@localhost:test/tubesync') + # Invalid port + with self.assertRaises(DatabaseConnectionError): + parse_database_connection_string( + 'postgresql://tubesync:password@localhost:65537/tubesync') + # Invalid username or password + with self.assertRaises(DatabaseConnectionError): + parse_database_connection_string( + 'postgresql://tubesync:password:test@localhost:5432/tubesync') + # Invalid database name + with self.assertRaises(DatabaseConnectionError): + parse_database_connection_string( + 'postgresql://tubesync:password@localhost:5432/tubesync/test') diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 26aeef8..184b57b 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -1,4 +1,85 @@ -from urllib.parse import urlunsplit, urlencode +from urllib.parse import urlunsplit, urlencode, urlparse +from .errors import DatabaseConnectionError + + +def parse_database_connection_string(database_connection_string): + ''' + Parses a connection string in a URL style format, such as: + postgresql://tubesync:password@localhost:5432/tubesync + mysql://someuser:somepassword@localhost:3306/tubesync + into a Django-compatible settings.DATABASES dict format. + ''' + valid_drivers = ('postgresql', 'mysql') + default_ports = { + 'postgresql': 5432, + 'mysql': 3306, + } + django_backends = { + 'postgresql': 'django.db.backends.postgresql', + 'mysql': 'django.db.backends.mysql', + } + try: + parts = urlparse(str(database_connection_string)) + except Exception as e: + raise DatabaseConnectionError(f'Failed to parse "{database_connection_string}" ' + f'as a database connection string: {e}') from e + driver = parts.scheme + user_pass_host_port = parts.netloc + database = parts.path + if driver not in valid_drivers: + raise DatabaseConnectionError(f'Database connection string ' + f'"{database_connection_string}" specified an ' + f'invalid driver, must be one of {valid_drivers}') + django_driver = django_backends.get(driver) + host_parts = user_pass_host_port.split('@') + if len(host_parts) != 2: + raise DatabaseConnectionError(f'Database connection string netloc must be in ' + f'the format of user:pass@host') + user_pass, host_port = host_parts + user_pass_parts = user_pass.split(':') + if len(user_pass_parts) != 2: + raise DatabaseConnectionError(f'Database connection string netloc must be in ' + f'the format of user:pass@host') + username, password = user_pass_parts + host_port_parts = host_port.split(':') + if len(host_port_parts) == 1: + # No port number, assign a default port + hostname = host_port_parts[0] + port = default_ports.get(driver) + elif len(host_port_parts) == 2: + # Host name and port number + hostname, port = host_port_parts + try: + port = int(port) + except (ValueError, TypeError) as e: + raise DatabaseConnectionError(f'Database connection string contained an ' + f'invalid port, ports must be integers: ' + f'{e}') from e + if not 0 < port < 63336: + raise DatabaseConnectionError(f'Database connection string contained an ' + f'invalid port, ports must be between 1 and ' + f'65535, got {port}') + else: + # Malformed + raise DatabaseConnectionError(f'Database connection host must be a hostname or ' + f'a hostname:port combination') + if database.startswith('/'): + database = database[1:] + if not database: + raise DatabaseConnectionError(f'Database connection string path must be a ' + f'string in the format of /databasename') + if '/' in database: + raise DatabaseConnectionError(f'Database connection string path can only ' + f'contain a single string name, got: {database}') + return { + 'DRIVER': driver, + 'ENGINE': django_driver, + 'NAME': database, + 'USER': username, + 'PASSWORD': password, + 'HOST': hostname, + 'PORT': port, + } def get_client_ip(request): diff --git a/tubesync/sync/templates/sync/dashboard.html b/tubesync/sync/templates/sync/dashboard.html index de63493..7342316 100644 --- a/tubesync/sync/templates/sync/dashboard.html +++ b/tubesync/sync/templates/sync/dashboard.html @@ -123,6 +123,10 @@ Downloads directory Downloads directory
{{ downloads_dir }} + + Database + Database
{{ database_connection }} + diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 0aae807..a45ed70 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -78,6 +78,7 @@ class DashboardView(TemplateView): # Config and download locations data['config_dir'] = str(settings.CONFIG_BASE_DIR) data['downloads_dir'] = str(settings.DOWNLOAD_ROOT) + data['database_connection'] = settings.DATABASE_CONNECTION_STR return data diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 39840e4..b1fab19 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -1,5 +1,7 @@ import os from pathlib import Path +from common.logger import log +from common.utils import parse_database_connection_string BASE_DIR = Path(__file__).resolve().parent.parent @@ -21,12 +23,31 @@ FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', None) TIME_ZONE = os.getenv('TZ', 'UTC') -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': CONFIG_BASE_DIR / 'db.sqlite3', +database_dict = {} +database_connection_env = os.getenv('DATABASE_CONNECTION', '') +if database_connection_env: + database_dict = parse_database_connection_string(database_connection_env) + + +if database_dict: + log.info(f'Using database connection: {database_dict["ENGINE"]}://' + f'{database_dict["USER"]}:[hidden]@{database_dict["HOST"]}:' + f'{database_dict["PORT"]}/{database_dict["NAME"]}') + DATABASES = { + 'default': database_dict, } -} + DATABASE_CONNECTION_STR = (f'{database_dict["DRIVER"]} at "{database_dict["HOST"]}:' + f'{database_dict["PORT"]}" database ' + f'"{database_dict["NAME"]}"') +else: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': CONFIG_BASE_DIR / 'db.sqlite3', + } + } + DATABASE_CONNECTION_STR = f'sqlite at "{DATABASES["default"]["NAME"]}"' + DEFAULT_THREADS = 1 MAX_BACKGROUND_TASK_ASYNC_THREADS = 8