From 18e506a703d2fb14919cfecbb30dcbd9e25ae6c5 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 27 May 2025 03:39:43 -0400 Subject: [PATCH 01/48] Add `django-huey` --- Pipfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Pipfile b/Pipfile index 49b5127b..b0b3c6f5 100644 --- a/Pipfile +++ b/Pipfile @@ -8,6 +8,7 @@ autopep8 = "*" [packages] django = "~=5.2.1" +django-huey = "*" django-sass-processor = {extras = ["management-command"], version = "*"} pillow = "*" whitenoise = "*" From 15d4a8f4e46fe86108a3a2cea94c7fe7304a8826 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 28 May 2025 21:13:15 -0400 Subject: [PATCH 02/48] Create huey.py --- tubesync/common/huey.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tubesync/common/huey.py diff --git a/tubesync/common/huey.py b/tubesync/common/huey.py new file mode 100644 index 00000000..3d145ca0 --- /dev/null +++ b/tubesync/common/huey.py @@ -0,0 +1,19 @@ + + +def sqlite_tasks(key, /, prefix=None): + name_fmt = 'huey_{}' + if prefix is None: + prefix = '' + if prefix: + name_fmt = f'huey_{prefix}_' + '{}' + name = name_fmt.format(key) + return dict( + huey_class='huey.SqliteHuey', + name=name, + connection=dict( + filename=f'/config/tasks/{name}.db', + fsync=True, + strict_fifo=True, + ), + ) + From 261e17954fec7a4f2d0f7da9b4a87ebf17dcb6d3 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 28 May 2025 21:17:27 -0400 Subject: [PATCH 03/48] Add `django_huey` queues to settings --- tubesync/tubesync/settings.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index cd6d1dca..a087da1a 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -1,5 +1,6 @@ from django import VERSION as DJANGO_VERSION from pathlib import Path +from common.huey import sqlite_tasks from common.utils import getenv @@ -8,7 +9,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.4' +VERSION = '0.15.7' SECRET_KEY = '' DEBUG = False ALLOWED_HOSTS = [] @@ -24,6 +25,7 @@ INSTALLED_APPS = [ 'django.contrib.humanize', 'sass_processor', 'background_task', + 'django_huey', 'common', 'sync', ] @@ -47,6 +49,17 @@ ROOT_URLCONF = 'tubesync.urls' FORCE_SCRIPT_NAME = None +DJANGO_HUEY = { + 'default': 'network', + 'queues': { + 'database': sqlite_tasks('database'), + 'filesystem': sqlite_tasks('filesystem'), + 'limited': sqlite_tasks('limited', prefix='net'), + 'network': sqlite_tasks('network'), + }, +} + + TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', From 8d64a966ff19f83d770624443493312b09ca98ee Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 28 May 2025 22:15:12 -0400 Subject: [PATCH 04/48] Add more configuration --- tubesync/common/huey.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tubesync/common/huey.py b/tubesync/common/huey.py index 3d145ca0..97bb6ee2 100644 --- a/tubesync/common/huey.py +++ b/tubesync/common/huey.py @@ -10,10 +10,21 @@ def sqlite_tasks(key, /, prefix=None): return dict( huey_class='huey.SqliteHuey', name=name, + immediate=False, + results=True, + store_none=False, + utc=True, + compression=True, connection=dict( filename=f'/config/tasks/{name}.db', fsync=True, strict_fifo=True, ), + consumer=dict( + workers=1, + worker_type='process', + max_delay=20.0, + scheduler_interval=10, + ), ) From 482d85768da17c36a2b6893ce9b18e117fc52d9b Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 28 May 2025 22:32:25 -0400 Subject: [PATCH 05/48] Create the `/config/tasks` directory --- config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run index baaf6e0c..7b085515 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run @@ -4,6 +4,9 @@ groupmod -o -g "${PGID:=911}" app usermod -o -u "${PUID:=911}" app +# Ensure /config directories exist +mkdir -v -p /config/tasks + # Reset permissions chown -R app:app /run/app chmod -R 0700 /run/app From 16ee589ab30d3faa9d10f25e5fd3f243f68a71a0 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 28 May 2025 23:46:36 -0400 Subject: [PATCH 06/48] Create huey-net-limited/dependencies.d/base --- .../etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/base | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/base diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/base new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/base @@ -0,0 +1 @@ + From 730e75e086513403674463e5686ed7feefcca0e2 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 28 May 2025 23:54:11 -0400 Subject: [PATCH 07/48] Create huey-network/dependencies.d/base --- .../root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/base | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/base diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/base new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/base @@ -0,0 +1 @@ + From 379d54873bfe557dfd9be386ccf92a415f06a998 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 28 May 2025 23:56:16 -0400 Subject: [PATCH 08/48] Create huey-database/dependencies.d/base --- .../etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/base | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/base diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/base new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/base @@ -0,0 +1 @@ + From f2afbc17b4fa1d0cc2b770c63ac96f511ff2077e Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 28 May 2025 23:58:21 -0400 Subject: [PATCH 09/48] Create huey-filesystem/dependencies.d/base --- .../etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/base | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/base diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/base new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/base @@ -0,0 +1 @@ + From 5c54909edd1329fb58afa840e72f40ada8e312da Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 00:01:50 -0400 Subject: [PATCH 10/48] Create huey-net-limited/down-signal --- config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/down-signal | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/down-signal diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/down-signal b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/down-signal new file mode 100644 index 00000000..d751378e --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/down-signal @@ -0,0 +1 @@ +SIGINT From 08d593d320958554b84bf4fa436ce15a40724ccb Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 00:03:04 -0400 Subject: [PATCH 11/48] Create huey-network/down-signal --- config/root/etc/s6-overlay/s6-rc.d/huey-network/down-signal | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-network/down-signal diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-network/down-signal b/config/root/etc/s6-overlay/s6-rc.d/huey-network/down-signal new file mode 100644 index 00000000..d751378e --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-network/down-signal @@ -0,0 +1 @@ +SIGINT From 61260fe2a5275a35ec182b93a4ef3c40ff54a83c Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 00:04:27 -0400 Subject: [PATCH 12/48] Create huey-database/down-signal --- config/root/etc/s6-overlay/s6-rc.d/huey-database/down-signal | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-database/down-signal diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-database/down-signal b/config/root/etc/s6-overlay/s6-rc.d/huey-database/down-signal new file mode 100644 index 00000000..d751378e --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-database/down-signal @@ -0,0 +1 @@ +SIGINT From c39491b644250e2701b8c167a449556746617a8a Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 00:05:32 -0400 Subject: [PATCH 13/48] Create huey-filesystem/down-signal --- config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/down-signal | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/down-signal diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/down-signal b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/down-signal new file mode 100644 index 00000000..d751378e --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/down-signal @@ -0,0 +1 @@ +SIGINT From 227624a854d2be551cd7d00958ee27638fb47668 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 00:07:52 -0400 Subject: [PATCH 14/48] Create huey-net-limited/type --- config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/type | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/type diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/type b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/type new file mode 100644 index 00000000..5883cff0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/type @@ -0,0 +1 @@ +longrun From c16d3fe18f0d718110738e868d309bd38a08e782 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 00:08:53 -0400 Subject: [PATCH 15/48] Create huey-network/type --- config/root/etc/s6-overlay/s6-rc.d/huey-network/type | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-network/type diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-network/type b/config/root/etc/s6-overlay/s6-rc.d/huey-network/type new file mode 100644 index 00000000..5883cff0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-network/type @@ -0,0 +1 @@ +longrun From 8b08177394d1c9dce4b1b772cfade238ea48d9eb Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 00:10:01 -0400 Subject: [PATCH 16/48] Create huey-database/type --- config/root/etc/s6-overlay/s6-rc.d/huey-database/type | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-database/type diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-database/type b/config/root/etc/s6-overlay/s6-rc.d/huey-database/type new file mode 100644 index 00000000..5883cff0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-database/type @@ -0,0 +1 @@ +longrun From 7ae89b3471fdfa37c2b81a2d849fd62e6cf49302 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 00:11:10 -0400 Subject: [PATCH 17/48] Create huey-filesystem/type --- config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/type | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/type diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/type b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/type new file mode 100644 index 00000000..5883cff0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/type @@ -0,0 +1 @@ +longrun From 6e5099e199fbe14122851f474238336f2ffb80b8 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 00:29:55 -0400 Subject: [PATCH 18/48] Create huey-net-limited/run --- config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run new file mode 100644 index 00000000..c40cd786 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run @@ -0,0 +1,5 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py djangohuey \ + --queue limited From c1159088e533c018e7ddc5d95da55027609b7755 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 00:31:31 -0400 Subject: [PATCH 19/48] Create huey-network/run --- config/root/etc/s6-overlay/s6-rc.d/huey-network/run | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-network/run diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-network/run b/config/root/etc/s6-overlay/s6-rc.d/huey-network/run new file mode 100644 index 00000000..0e5f0e8f --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-network/run @@ -0,0 +1,5 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py djangohuey \ + --queue network From 2b0d3db90a2630417d762a8f75cb9b113fd0258c Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 00:33:07 -0400 Subject: [PATCH 20/48] Create huey-database/run --- config/root/etc/s6-overlay/s6-rc.d/huey-database/run | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-database/run diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-database/run b/config/root/etc/s6-overlay/s6-rc.d/huey-database/run new file mode 100644 index 00000000..c803c4c3 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-database/run @@ -0,0 +1,5 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py djangohuey \ + --queue database From 5e224254b57a8e0fd5f45a7383b5722ce00a91ba Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 00:34:14 -0400 Subject: [PATCH 21/48] Create huey-filesystem/run --- config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run new file mode 100644 index 00000000..f52ee7c6 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run @@ -0,0 +1,5 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py djangohuey \ + --queue filesystem From c8f95456f2f4c7c06b4979b79b4c7890dba62bb5 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 19:56:57 -0400 Subject: [PATCH 22/48] Create gunicorn dependency for huey-database --- .../etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/gunicorn | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/gunicorn diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/gunicorn b/config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/gunicorn new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/gunicorn @@ -0,0 +1 @@ + From 7078c2b1ed1da940ac993eceb5eacba9e86d60e5 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 19:58:16 -0400 Subject: [PATCH 23/48] Create gunicorn dependency for huey-filesystem --- .../s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/gunicorn | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/gunicorn diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/gunicorn b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/gunicorn new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/gunicorn @@ -0,0 +1 @@ + From 16b8f9ad3c3c030a416d2650c3bd1c216715dbf1 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 19:59:40 -0400 Subject: [PATCH 24/48] Create gunicorn dependency for huey-network --- .../etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/gunicorn | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/gunicorn diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/gunicorn b/config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/gunicorn new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/gunicorn @@ -0,0 +1 @@ + From 7e9be86789023a7d7d42eb7afcf85be699cfa575 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:00:47 -0400 Subject: [PATCH 25/48] Create gunicorn dependency for huey-net-limited --- .../s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/gunicorn | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/gunicorn diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/gunicorn b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/gunicorn new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/gunicorn @@ -0,0 +1 @@ + From 9b33606168cc042da81f33d13e7f4ebf156bed15 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:08:00 -0400 Subject: [PATCH 26/48] Create background-task-workers bundle --- config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type diff --git a/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type new file mode 100644 index 00000000..757b4221 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type @@ -0,0 +1 @@ +bundle From 71c0ae6ed2898e67c7f29697314d5ab6e8b5809d Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:10:49 -0400 Subject: [PATCH 27/48] Add tubesync-network-worker to background-task-workers bundle --- .../background-task-workers/contents.d/tubesync-network-worker | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker diff --git a/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker @@ -0,0 +1 @@ + From b4c14ea1dc76b72b4952a6d9ba61b79c72582f8b Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:12:56 -0400 Subject: [PATCH 28/48] Add tubesync-db-worker to background-task-workers bundle --- .../background-task-workers/contents.d/tubesync-db-worker | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-db-worker diff --git a/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-db-worker b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-db-worker new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-db-worker @@ -0,0 +1 @@ + From 3890ad880696c5ea7bdf5d0f5397fca429661727 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:14:23 -0400 Subject: [PATCH 29/48] Add tubesync-fs-worker to background-task-workers bundle --- .../background-task-workers/contents.d/tubesync-fs-worker | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-fs-worker diff --git a/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-fs-worker b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-fs-worker new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-fs-worker @@ -0,0 +1 @@ + From b72fc2d0229728e3658896cf4ce88b6d78ffbea5 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:21:50 -0400 Subject: [PATCH 30/48] Create huey-consumers bundle --- config/root/etc/s6-overlay/s6-rc.d/huey-consumers/type | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-consumers/type diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/type b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/type new file mode 100644 index 00000000..757b4221 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/type @@ -0,0 +1 @@ +bundle From 95181c45f019a4a01cb85aabe68c992ab0d2f517 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:25:08 -0400 Subject: [PATCH 31/48] Add huey-net-limited to huey-consumers bundle --- .../s6-rc.d/huey-consumers/contents.d/huey-net-limited | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited @@ -0,0 +1 @@ + From 3c8d18929a25f612825af1c55004a27c8f31e820 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:27:53 -0400 Subject: [PATCH 32/48] Add huey-network to huey-consumers bundle --- .../s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network @@ -0,0 +1 @@ + From a2abe4f584fd0b7a3cb720a7d2f737946f42447b Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:29:37 -0400 Subject: [PATCH 33/48] Add huey-database to huey-consumers bundle --- .../s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database @@ -0,0 +1 @@ + From a8e112b024fd8d1c8a2587fa97b2a5a862a043c5 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:30:55 -0400 Subject: [PATCH 34/48] Add huey-filesystem to huey-consumers bundle --- .../s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem @@ -0,0 +1 @@ + From 4a090ab5538487c6b1010b3dbdd24cd37abb1ef6 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:32:52 -0400 Subject: [PATCH 35/48] Add huey-consumers to user bundle --- .../root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers diff --git a/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers b/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers @@ -0,0 +1 @@ + From 894f33254e7efb71e04e003d6ddab5e3a05b5ab8 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:43:58 -0400 Subject: [PATCH 36/48] Match the current version --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 938f7ecd..ea2f668b 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -9,7 +9,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.7' +VERSION = '0.15.6' SECRET_KEY = '' DEBUG = False ALLOWED_HOSTS = [] From 36059c0c62d04588216a4de1a4945747d0808ea0 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 29 May 2025 20:50:16 -0400 Subject: [PATCH 37/48] Update ci.yaml --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5ae58c0a..ef45ca10 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -103,6 +103,7 @@ jobs: pip install --system --strict --requirements requirements.txt - name: Set up Django environment run: | + mkdir -v -p /config/tasks cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/* cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* From d08bbf24e25459de85f0f8d3ebde3df423fac1f0 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 30 May 2025 00:13:46 -0400 Subject: [PATCH 38/48] Use a symbolic link for `/config` --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ef45ca10..55224a1b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -103,7 +103,8 @@ jobs: pip install --system --strict --requirements requirements.txt - name: Set up Django environment run: | - mkdir -v -p /config/tasks + mkdir -v -p ~/.config/TubeSync/config/tasks + sudo ln -v -s -f -T ~/.config/TubeSync/config /config cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/* cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* From 6f5d35d33c7978c3038723bb0c60f77fff3f868e Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 30 May 2025 00:35:39 -0400 Subject: [PATCH 39/48] Support local settings stored in `/config/tubesync` --- config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run index 7b085515..1b136c2d 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run @@ -7,6 +7,18 @@ usermod -o -u "${PUID:=911}" app # Ensure /config directories exist mkdir -v -p /config/tasks +# Copy local_settings.py for the user +if [ -f /config/tubesync/local_settings.py ] +then + # from the image for an example or comparison + cp -v -p /app/tubesync/local_settings.py \ + /config/tubesync/local_settings.py.image + + # to the image to apply the user's settings + cp -v -p /config/tubesync/local_settings.py \ + /app/tubesync/local_settings.py +fi + # Reset permissions chown -R app:app /run/app chmod -R 0700 /run/app From bd06c5b888733af1660cb29d675f0af04591bbcf Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 30 May 2025 01:17:11 -0400 Subject: [PATCH 40/48] Add the rest of the `/config` sub-directories --- config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run index 1b136c2d..df003540 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run @@ -5,7 +5,7 @@ groupmod -o -g "${PGID:=911}" app usermod -o -u "${PUID:=911}" app # Ensure /config directories exist -mkdir -v -p /config/tasks +mkdir -v -p /config/{cache,media,tasks,tubesync} # Copy local_settings.py for the user if [ -f /config/tubesync/local_settings.py ] From 0b63d70fd7cddc0fdf289384bf6f964044eae795 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 30 May 2025 01:29:52 -0400 Subject: [PATCH 41/48] Make directories before creating SQLite database --- tubesync/tubesync/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index ea2f668b..0eb4ad58 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -58,6 +58,11 @@ DJANGO_HUEY = { 'network': sqlite_tasks('network'), }, } +for django_huey_queue in DJANGO_HUEY['queues'].items(): + connection = django_huey_queue.get('connection') + if connection: + filepath = Path('/.' + connection.get('filename') or '').resolve(strict=False) + filepath.parent.mkdir(exist_ok=True, parents=True) TEMPLATES = [ From f561de8140b1453a496a1a0c4f34244acd745529 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 30 May 2025 01:34:10 -0400 Subject: [PATCH 42/48] Clean up any files created in Python --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index da1e84c5..118eda78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -484,9 +484,10 @@ RUN set -x && \ # Run any required app commands /usr/bin/python3 -B /app/manage.py compilescss && \ /usr/bin/python3 -B /app/manage.py collectstatic --no-input --link && \ + rm -rf /config /downloads /run/app && \ # Create config, downloads and run dirs mkdir -v -p /run/app && \ - mkdir -v -p /config/media && \ + mkdir -v -p /config/media /config/tasks && \ mkdir -v -p /config/cache/pycache && \ mkdir -v -p /downloads/audio && \ mkdir -v -p /downloads/video && \ From 33491edb9e15cc156bd4ca3d23ad95707d14254a Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 30 May 2025 01:53:16 -0400 Subject: [PATCH 43/48] Only create `/config` for tests --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 55224a1b..05aab5a0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -103,7 +103,7 @@ jobs: pip install --system --strict --requirements requirements.txt - name: Set up Django environment run: | - mkdir -v -p ~/.config/TubeSync/config/tasks + mkdir -v -p ~/.config/TubeSync/config sudo ln -v -s -f -T ~/.config/TubeSync/config /config cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/* From ea43d540218fffa27c48de91a481ff9d9c789a6f Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 30 May 2025 01:55:12 -0400 Subject: [PATCH 44/48] fixup: values instead of items --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 0eb4ad58..f5689dd1 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -58,7 +58,7 @@ DJANGO_HUEY = { 'network': sqlite_tasks('network'), }, } -for django_huey_queue in DJANGO_HUEY['queues'].items(): +for django_huey_queue in DJANGO_HUEY['queues'].values(): connection = django_huey_queue.get('connection') if connection: filepath = Path('/.' + connection.get('filename') or '').resolve(strict=False) From 98df36a1fc887bb6d868e57cecf321981e3269ef Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 30 May 2025 02:18:01 -0400 Subject: [PATCH 45/48] Fail the build rather than hiding the file that is in a lower layer --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 118eda78..314eb98d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -480,7 +480,7 @@ COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings. # Build app RUN set -x && \ # Make absolutely sure we didn't accidentally bundle a SQLite dev database - rm -rf /app/db.sqlite3 && \ + test '!' -e /app/db.sqlite3 && \ # Run any required app commands /usr/bin/python3 -B /app/manage.py compilescss && \ /usr/bin/python3 -B /app/manage.py collectstatic --no-input --link && \ From 479f36b7f18668017ad7caeb8d747f1ddaa243c7 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 30 May 2025 08:29:58 -0400 Subject: [PATCH 46/48] Add and document logging settings --- tubesync/common/huey.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tubesync/common/huey.py b/tubesync/common/huey.py index 97bb6ee2..e88e319b 100644 --- a/tubesync/common/huey.py +++ b/tubesync/common/huey.py @@ -25,6 +25,12 @@ def sqlite_tasks(key, /, prefix=None): worker_type='process', max_delay=20.0, scheduler_interval=10, + simple_log=False, + # verbose has three positions: + # DEBUG: True + # INFO: None + # WARNING: False + verbose=False, ), ) From 78a57211b2dc52c971442966dd71b7c54fc87f0a Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 30 May 2025 08:35:36 -0400 Subject: [PATCH 47/48] Unlock previously locked tasks when the consumer starts --- tubesync/common/huey.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/common/huey.py b/tubesync/common/huey.py index e88e319b..66d13fc2 100644 --- a/tubesync/common/huey.py +++ b/tubesync/common/huey.py @@ -24,6 +24,7 @@ def sqlite_tasks(key, /, prefix=None): workers=1, worker_type='process', max_delay=20.0, + flush_locks=True, scheduler_interval=10, simple_log=False, # verbose has three positions: From bfaa76c3ddefccc3320d113f93fd0bab770c6bec Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 4 Jun 2025 04:06:01 -0400 Subject: [PATCH 48/48] Add `TaskQueue.LIMIT` for the new `huey` queue --- tubesync/sync/choices.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/choices.py b/tubesync/sync/choices.py index 92cb6c8f..238aa160 100644 --- a/tubesync/sync/choices.py +++ b/tubesync/sync/choices.py @@ -165,6 +165,7 @@ class TaskQueue(models.TextChoices): DB = 'database', _('Database') FS = 'filesystem', _('Filesystem') NET = 'network', _('Networking') + LIMIT = 'limited', _('Limited Networking') class WeekDay(models.IntegerChoices):