support external postgresql, mysql and mariadb databases, resolves #72

Pipfile

@@ -18,6 +18,8 @@ youtube-dl = "*"
 django-background-tasks = "*"
 requests = "*"
 django-basicauth = "*"
+psycopg2-binary = "*"
+mysqlclient = "*"
 
 [requires]
 python_version = "3"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "f698e2853dec2d325d2d7e752620fc81d911022d394a57f2f8a9349ac2682752"
+            "sha256": "bea753f24d773d83d202254027b78d83943fa90bf5649d014faf1c09d9eab8b0"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -139,6 +139,17 @@
             "index": "pypi",
             "version": "==0.20.1"
         },
+        "mysqlclient": {
+            "hashes": [
+                "sha256:0ac0dd759c4ca02c35a9fedc24bc982cf75171651e8187c2495ec957a87dfff7",
+                "sha256:3381ca1a4f37ff1155fcfde20836b46416d66531add8843f6aa6d968982731c3",
+                "sha256:71c4b330cf2313bbda0307fc858cc9055e64493ba9bf28454d25cf8b3ee8d7f5",
+                "sha256:f6ebea7c008f155baeefe16c56cd3ee6239f7a5a9ae42396c2f1860f08a7c432",
+                "sha256:fc575093cf81b6605bed84653e48b277318b880dc9becf42dd47fa11ffd3e2b6"
+            ],
+            "index": "pypi",
+            "version": "==2.0.3"
+        },
         "pillow": {
             "hashes": [
                 "sha256:01425106e4e8cee195a411f729cff2a7d61813b0b11737c12bd5991f5f14bcd5",
@@ -178,6 +189,47 @@
             "index": "pypi",
             "version": "==8.2.0"
         },
+        "psycopg2-binary": {
+            "hashes": [
+                "sha256:0deac2af1a587ae12836aa07970f5cb91964f05a7c6cdb69d8425ff4c15d4e2c",
+                "sha256:0e4dc3d5996760104746e6cfcdb519d9d2cd27c738296525d5867ea695774e67",
+                "sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0",
+                "sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6",
+                "sha256:1fabed9ea2acc4efe4671b92c669a213db744d2af8a9fc5d69a8e9bc14b7a9db",
+                "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94",
+                "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52",
+                "sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056",
+                "sha256:6a32f3a4cb2f6e1a0b15215f448e8ce2da192fd4ff35084d80d5e39da683e79b",
+                "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd",
+                "sha256:7d92a09b788cbb1aec325af5fcba9fed7203897bbd9269d5691bb1e3bce29550",
+                "sha256:833709a5c66ca52f1d21d41865a637223b368c0ee76ea54ca5bad6f2526c7679",
+                "sha256:89705f45ce07b2dfa806ee84439ec67c5d9a0ef20154e0e475e2b2ed392a5b83",
+                "sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77",
+                "sha256:950bc22bb56ee6ff142a2cb9ee980b571dd0912b0334aa3fe0fe3788d860bea2",
+                "sha256:a0c50db33c32594305b0ef9abc0cb7db13de7621d2cadf8392a1d9b3c437ef77",
+                "sha256:a0eb43a07386c3f1f1ebb4dc7aafb13f67188eab896e7397aa1ee95a9c884eb2",
+                "sha256:aaa4213c862f0ef00022751161df35804127b78adf4a2755b9f991a507e425fd",
+                "sha256:ac0c682111fbf404525dfc0f18a8b5f11be52657d4f96e9fcb75daf4f3984859",
+                "sha256:ad20d2eb875aaa1ea6d0f2916949f5c08a19c74d05b16ce6ebf6d24f2c9f75d1",
+                "sha256:b4afc542c0ac0db720cf516dd20c0846f71c248d2b3d21013aa0d4ef9c71ca25",
+                "sha256:b8a3715b3c4e604bcc94c90a825cd7f5635417453b253499664f784fc4da0152",
+                "sha256:ba28584e6bca48c59eecbf7efb1576ca214b47f05194646b081717fa628dfddf",
+                "sha256:ba381aec3a5dc29634f20692349d73f2d21f17653bda1decf0b52b11d694541f",
+                "sha256:bd1be66dde2b82f80afb9459fc618216753f67109b859a361cf7def5c7968729",
+                "sha256:c2507d796fca339c8fb03216364cca68d87e037c1f774977c8fc377627d01c71",
+                "sha256:cec7e622ebc545dbb4564e483dd20e4e404da17ae07e06f3e780b2dacd5cee66",
+                "sha256:d14b140a4439d816e3b1229a4a525df917d6ea22a0771a2a78332273fd9528a4",
+                "sha256:d1b4ab59e02d9008efe10ceabd0b31e79519da6fb67f7d8e8977118832d0f449",
+                "sha256:d5227b229005a696cc67676e24c214740efd90b148de5733419ac9aaba3773da",
+                "sha256:e1f57aa70d3f7cc6947fd88636a481638263ba04a742b4a37dd25c373e41491a",
+                "sha256:e74a55f6bad0e7d3968399deb50f61f4db1926acf4a6d83beaaa7df986f48b1c",
+                "sha256:e82aba2188b9ba309fd8e271702bd0d0fc9148ae3150532bbb474f4590039ffb",
+                "sha256:ee69dad2c7155756ad114c02db06002f4cded41132cc51378e57aad79cc8e4f4",
+                "sha256:f5ab93a2cb2d8338b1674be43b442a7f544a0971da062a5da774ed40587f18f5"
+            ],
+            "index": "pypi",
+            "version": "==2.8.6"
+        },
         "pytz": {
             "hashes": [
                 "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",

README.md

@@ -242,6 +242,8 @@ and less common features:
 
 ![Reset tasks from the command line](https://github.com/meeb/tubesync/blob/main/docs/reset-tasks.md)
 
+![Using PostgreSQL, MySQL or MariaDB as database backends](https://github.com/meeb/tubesync/blob/main/docs/other-database-backends.md)
+
 
 # Warnings
 
@@ -357,18 +359,19 @@ There are a number of other environment variables you can set. These are, mostly
 **NOT** required to be set in the default container installation, they are really only
 useful if you are manually installing TubeSync in some other environment. These are:
 
-| Name                     | What                                                         | Example                            |
+| Name                     | What                                                         | Example                              |
-| ------------------------ | ------------------------------------------------------------ | ---------------------------------- |
+| ------------------------ | ------------------------------------------------------------ | ------------------------------------ |
-| DJANGO_SECRET_KEY        | Django's SECRET_KEY                                          | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l   |
+| DJANGO_SECRET_KEY        | Django's SECRET_KEY                                          | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l     |
-| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME                                   | /somepath                          |
+| DJANGO_FORCE_SCRIPT_NAME | Django's FORCE_SCRIPT_NAME                                   | /somepath                            |
-| TUBESYNC_DEBUG           | Enable debugging                                             | True                               |
+| TUBESYNC_DEBUG           | Enable debugging                                             | True                                 |
-| TUBESYNC_WORKERS         | Number of background workers, default is 2, max allowed is 8 | 2                                  |
+| TUBESYNC_WORKERS         | Number of background workers, default is 2, max allowed is 8 | 2                                    |
-| TUBESYNC_HOSTS           | Django's ALLOWED_HOSTS                                       | tubesync.example.com,otherhost.com |
+| TUBESYNC_HOSTS           | Django's ALLOWED_HOSTS                                       | tubesync.example.com,otherhost.com   |
-| GUNICORN_WORKERS         | Number of gunicorn workers to spawn                          | 3                                  |
+| GUNICORN_WORKERS         | Number of gunicorn workers to spawn                          | 3                                    |
-| LISTEN_HOST              | IP address for gunicorn to listen on                         | 127.0.0.1                          |
+| LISTEN_HOST              | IP address for gunicorn to listen on                         | 127.0.0.1                            |
-| LISTEN_PORT              | Port number for gunicorn to listen on                        | 8080                               |
+| LISTEN_PORT              | Port number for gunicorn to listen on                        | 8080                                 |
-| HTTP_USER                | Sets the username for HTTP basic authentication              | some-username                      |
+| HTTP_USER                | Sets the username for HTTP basic authentication              | some-username                        |
-| HTTP_PASS                | Sets the password for HTTP basic authentication              | some-secure-password               |
+| HTTP_PASS                | Sets the password for HTTP basic authentication              | some-secure-password                 |
+| DATABASE_CONNECTION      | Optional external database connection details                | mysql://user:pass@host:port/database |
 
 
 # Manual, non-containerised, installation

docs/other-database-backends.md

@@ -0,0 +1,69 @@
+# TubeSync
+
+## Advanced usage guide - using other database backends
+
+This is a new feature in v1.0 of TubeSync and later. It allows you to use a custom
+existing external database server instead of the default SQLite database. You may want
+to use this if you encounter performance issues with adding very large or a large
+number of channels and database write contention (as shown by errors in the log)
+become an issue.
+
+## Requirements
+
+TubeSync supports SQLite (the automatic default) as well as PostgreSQL, MySQL and
+MariaDB. For MariaDB just follow the MySQL instructions as the driver is the same.
+
+You should a blank install of TubeSync. Migrating to a new database will reset your
+database. If you are comfortable with Django you can export and re-import existing
+database data with:
+
+```bash
+$ docker exec -ti tubesync python3 /app/manage.py dumpdata > some-file.json
+```
+
+Then change you database backend over, then use
+
+```bash
+$ docker exec -ti tubesync python3 /app/manage.py loaddata some-file.json
+```
+
+As detailed in the Django documentation:
+
+https://docs.djangoproject.com/en/3.1/ref/django-admin/#dumpdata
+
+and:
+
+https://docs.djangoproject.com/en/3.1/ref/django-admin/#loaddata
+
+Further instructions are beyond the scope of TubeSync documenation and you should refer
+to Django documentation for more details.
+
+If you are not comfortable with the above, then skip the `dumpdata` steps, however
+remember you will start again with a completely new database.
+
+## Steps
+
+### 1. Create a database in your external database server
+
+You need to create a database and a user with permissions to access the database in
+your chosen external database server. Steps vary between PostgreSQL, MySQL and MariaDB
+so this is up to you to work out.
+
+### 2. Set the database connection string environment variable
+
+You need to provide the database connection details to TubeSync via an environment
+variable. The environment variable name is `DATABASE_CONNECTION` and the format is the
+standard URL-style string. Example are:
+
+`postgresql://tubesync:password@localhost:5432/tubesync`
+`mysql://tubesync:password@localhost:3306/tubesync`
+
+### 3. Start TubeSync and check the logs
+
+Once you start TubeSync with the new database connection you should see the folling log
+entry in the container or stdout logs:
+
+`2021-04-04 22:42:17,912 [tubesync/INFO] Using database connection: django.db.backends.postgresql://tubesync:[hidden]@localhost:5432/tubesync`
+
+If you see a line similar to the above and the web interface loads, congratulations,
+you are now using an external database server for your TubeSync data!

tubesync/common/errors.py

@@ -20,3 +20,10 @@ class DownloadFailedException(Exception):
         exist.
     '''
     pass
+
+
+class DatabaseConnectionError(Exception):
+    '''
+        Raised when parsing or initially connecting to a database.
+    '''
+    pass

tubesync/common/tests.py

@@ -2,6 +2,8 @@ import os.path
 from django.conf import settings
 from django.test import TestCase, Client
 from .testutils import prevent_request_warnings
+from .utils import parse_database_connection_string
+from .errors import DatabaseConnectionError
 
 
 class ErrorPageTestCase(TestCase):
@@ -61,3 +63,40 @@ class CommonStaticTestCase(TestCase):
         favicon_real_path = os.path.join(os.sep.join(root_parts),
                                          os.sep.join(url_parts))
         self.assertTrue(os.path.exists(favicon_real_path))
+
+
+class DatabaseConnectionTestCase(TestCase):
+
+    def test_parse_database_connection_string(self):
+        database_dict = parse_database_connection_string(
+            'postgresql://tubesync:password@localhost:5432/tubesync')
+        database_dict = parse_database_connection_string(
+            'mysql://tubesync:password@localhost:3306/tubesync')
+        # Invalid driver
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'test://tubesync:password@localhost:5432/tubesync')
+        # No username
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'postgresql://password@localhost:5432/tubesync')
+        # No database name
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'postgresql://tubesync:password@5432')
+        # Invalid port
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'postgresql://tubesync:password@localhost:test/tubesync')
+        # Invalid port
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'postgresql://tubesync:password@localhost:65537/tubesync')
+        # Invalid username or password
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'postgresql://tubesync:password:test@localhost:5432/tubesync')
+        # Invalid database name
+        with self.assertRaises(DatabaseConnectionError):
+            parse_database_connection_string(
+                'postgresql://tubesync:password@localhost:5432/tubesync/test')

tubesync/common/utils.py

@@ -1,4 +1,85 @@
-from urllib.parse import urlunsplit, urlencode
+from urllib.parse import urlunsplit, urlencode, urlparse
+from .errors import DatabaseConnectionError
+
+
+def parse_database_connection_string(database_connection_string):
+    '''
+        Parses a connection string in a URL style format, such as:
+            postgresql://tubesync:password@localhost:5432/tubesync
+            mysql://someuser:somepassword@localhost:3306/tubesync
+        into a Django-compatible settings.DATABASES dict format. 
+    '''
+    valid_drivers = ('postgresql', 'mysql')
+    default_ports = {
+        'postgresql': 5432,
+        'mysql': 3306,
+    }
+    django_backends = {
+        'postgresql': 'django.db.backends.postgresql',
+        'mysql': 'django.db.backends.mysql',
+    }
+    try:
+        parts = urlparse(str(database_connection_string))
+    except Exception as e:
+        raise DatabaseConnectionError(f'Failed to parse "{database_connection_string}" '
+                                      f'as a database connection string: {e}') from e
+    driver = parts.scheme
+    user_pass_host_port = parts.netloc
+    database = parts.path
+    if driver not in valid_drivers:
+        raise DatabaseConnectionError(f'Database connection string '
+                                      f'"{database_connection_string}" specified an '
+                                      f'invalid driver, must be one of {valid_drivers}')
+    django_driver = django_backends.get(driver)
+    host_parts = user_pass_host_port.split('@')
+    if len(host_parts) != 2:
+        raise DatabaseConnectionError(f'Database connection string netloc must be in '
+                                      f'the format of user:pass@host')
+    user_pass, host_port = host_parts
+    user_pass_parts = user_pass.split(':')
+    if len(user_pass_parts) != 2:
+        raise DatabaseConnectionError(f'Database connection string netloc must be in '
+                                      f'the format of user:pass@host')
+    username, password = user_pass_parts
+    host_port_parts = host_port.split(':')
+    if len(host_port_parts) == 1:
+        # No port number, assign a default port
+        hostname = host_port_parts[0]
+        port = default_ports.get(driver)
+    elif len(host_port_parts) == 2:
+        # Host name and port number
+        hostname, port = host_port_parts
+        try:
+            port = int(port)
+        except (ValueError, TypeError) as e:
+            raise DatabaseConnectionError(f'Database connection string contained an '
+                                          f'invalid port, ports must be integers: '
+                                          f'{e}') from e
+        if not 0 < port < 63336:
+            raise DatabaseConnectionError(f'Database connection string contained an '
+                                          f'invalid port, ports must be between 1 and '
+                                          f'65535, got {port}')
+    else:
+        # Malformed
+        raise DatabaseConnectionError(f'Database connection host must be a hostname or '
+                                      f'a hostname:port combination')
+    if database.startswith('/'):
+        database = database[1:]
+    if not database:
+        raise DatabaseConnectionError(f'Database connection string path must be a '
+                                      f'string in the format of /databasename')    
+    if '/' in database:
+        raise DatabaseConnectionError(f'Database connection string path can only '
+                                      f'contain a single string name, got: {database}')
+    return {
+        'DRIVER': driver,
+        'ENGINE': django_driver,
+        'NAME': database,
+        'USER': username,
+        'PASSWORD': password,
+        'HOST': hostname,
+        'PORT': port,
+    }
 
 
 def get_client_ip(request):

tubesync/sync/templates/sync/dashboard.html

@@ -123,6 +123,10 @@
         <td class="hide-on-small-only">Downloads directory</td>
         <td><span class="hide-on-med-and-up">Downloads directory<br></span><strong>{{ downloads_dir }}</strong></td>
       </tr>
+      <tr title="Database connection used by TubeSync">
+        <td class="hide-on-small-only">Database</td>
+        <td><span class="hide-on-med-and-up">Database<br></span><strong>{{ database_connection }}</strong></td>
+      </tr>
     </table>
   </div>
 </div>

tubesync/sync/views.py

@@ -78,6 +78,7 @@ class DashboardView(TemplateView):
         # Config and download locations
         data['config_dir'] = str(settings.CONFIG_BASE_DIR)
         data['downloads_dir'] = str(settings.DOWNLOAD_ROOT)
+        data['database_connection'] = settings.DATABASE_CONNECTION_STR
         return data
 
 

tubesync/tubesync/local_settings.py.container

@@ -1,5 +1,7 @@
 import os
 from pathlib import Path
+from common.logger import log
+from common.utils import parse_database_connection_string
 
 
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -21,12 +23,31 @@ FORCE_SCRIPT_NAME = os.getenv('DJANGO_FORCE_SCRIPT_NAME', None)
 TIME_ZONE = os.getenv('TZ', 'UTC')
 
 
-DATABASES = {
+database_dict = {}
-    'default': {
+database_connection_env = os.getenv('DATABASE_CONNECTION', '')
-        'ENGINE': 'django.db.backends.sqlite3',
+if database_connection_env:
-        'NAME': CONFIG_BASE_DIR / 'db.sqlite3',
+    database_dict = parse_database_connection_string(database_connection_env)
+
+
+if database_dict:
+    log.info(f'Using database connection: {database_dict["ENGINE"]}://'
+             f'{database_dict["USER"]}:[hidden]@{database_dict["HOST"]}:'
+             f'{database_dict["PORT"]}/{database_dict["NAME"]}')
+    DATABASES = {
+        'default': database_dict,
     }
-}
+    DATABASE_CONNECTION_STR = (f'{database_dict["DRIVER"]} at "{database_dict["HOST"]}:'
+                               f'{database_dict["PORT"]}" database '
+                               f'"{database_dict["NAME"]}"')
+else:
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.sqlite3',
+            'NAME': CONFIG_BASE_DIR / 'db.sqlite3',
+        }
+    }
+    DATABASE_CONNECTION_STR = f'sqlite at "{DATABASES["default"]["NAME"]}"'
+
 
 DEFAULT_THREADS = 1
 MAX_BACKGROUND_TASK_ASYNC_THREADS = 8