diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 117ec78e..ee842358 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -132,6 +132,8 @@ jobs: pip install --system --strict --requirements requirements.txt - name: Set up Django environment run: | + mkdir -v -p ~/.config/TubeSync/config + sudo ln -v -s -f -T ~/.config/TubeSync/config /config cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/* cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* diff --git a/Dockerfile b/Dockerfile index c8ef6171..1246f166 100644 --- a/Dockerfile +++ b/Dockerfile @@ -512,13 +512,14 @@ COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings. # Build app RUN set -x && \ # Make absolutely sure we didn't accidentally bundle a SQLite dev database - rm -rf /app/db.sqlite3 && \ + test '!' -e /app/db.sqlite3 && \ # Run any required app commands /usr/bin/python3 -B /app/manage.py compilescss && \ /usr/bin/python3 -B /app/manage.py collectstatic --no-input --link && \ + rm -rf /config /downloads /run/app && \ # Create config, downloads and run dirs mkdir -v -p /run/app && \ - mkdir -v -p /config/media && \ + mkdir -v -p /config/media /config/tasks && \ mkdir -v -p /config/cache/pycache && \ mkdir -v -p /downloads/audio && \ mkdir -v -p /downloads/video && \ diff --git a/Pipfile b/Pipfile index 34a52aa8..1d27d783 100644 --- a/Pipfile +++ b/Pipfile @@ -8,6 +8,7 @@ autopep8 = "*" [packages] django = "~=5.2.1" +django-huey = "*" django-sass-processor = {extras = ["management-command"], version = "*"} pillow = "*" whitenoise = "*" diff --git a/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-db-worker b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-db-worker new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-db-worker @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-fs-worker b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-fs-worker new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-fs-worker @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type new file mode 100644 index 00000000..757b4221 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type @@ -0,0 +1 @@ +bundle diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/type b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/type new file mode 100644 index 00000000..757b4221 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/type @@ -0,0 +1 @@ +bundle diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/base new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/base @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/gunicorn b/config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/gunicorn new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/gunicorn @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-database/down-signal b/config/root/etc/s6-overlay/s6-rc.d/huey-database/down-signal new file mode 100644 index 00000000..d751378e --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-database/down-signal @@ -0,0 +1 @@ +SIGINT diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-database/run b/config/root/etc/s6-overlay/s6-rc.d/huey-database/run new file mode 100644 index 00000000..c803c4c3 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-database/run @@ -0,0 +1,5 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py djangohuey \ + --queue database diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-database/type b/config/root/etc/s6-overlay/s6-rc.d/huey-database/type new file mode 100644 index 00000000..5883cff0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-database/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/base new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/base @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/gunicorn b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/gunicorn new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/gunicorn @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/down-signal b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/down-signal new file mode 100644 index 00000000..d751378e --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/down-signal @@ -0,0 +1 @@ +SIGINT diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run new file mode 100644 index 00000000..f52ee7c6 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run @@ -0,0 +1,5 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py djangohuey \ + --queue filesystem diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/type b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/type new file mode 100644 index 00000000..5883cff0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/base new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/base @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/gunicorn b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/gunicorn new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/gunicorn @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/down-signal b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/down-signal new file mode 100644 index 00000000..d751378e --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/down-signal @@ -0,0 +1 @@ +SIGINT diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run new file mode 100644 index 00000000..c40cd786 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run @@ -0,0 +1,5 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py djangohuey \ + --queue limited diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/type b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/type new file mode 100644 index 00000000..5883cff0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/base b/config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/base new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/base @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/gunicorn b/config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/gunicorn new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/gunicorn @@ -0,0 +1 @@ + diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-network/down-signal b/config/root/etc/s6-overlay/s6-rc.d/huey-network/down-signal new file mode 100644 index 00000000..d751378e --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-network/down-signal @@ -0,0 +1 @@ +SIGINT diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-network/run b/config/root/etc/s6-overlay/s6-rc.d/huey-network/run new file mode 100644 index 00000000..0e5f0e8f --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-network/run @@ -0,0 +1,5 @@ +#!/command/with-contenv bash + +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ + /usr/bin/python3 /app/manage.py djangohuey \ + --queue network diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-network/type b/config/root/etc/s6-overlay/s6-rc.d/huey-network/type new file mode 100644 index 00000000..5883cff0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-network/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run index baaf6e0c..df003540 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run @@ -4,6 +4,21 @@ groupmod -o -g "${PGID:=911}" app usermod -o -u "${PUID:=911}" app +# Ensure /config directories exist +mkdir -v -p /config/{cache,media,tasks,tubesync} + +# Copy local_settings.py for the user +if [ -f /config/tubesync/local_settings.py ] +then + # from the image for an example or comparison + cp -v -p /app/tubesync/local_settings.py \ + /config/tubesync/local_settings.py.image + + # to the image to apply the user's settings + cp -v -p /config/tubesync/local_settings.py \ + /app/tubesync/local_settings.py +fi + # Reset permissions chown -R app:app /run/app chmod -R 0700 /run/app diff --git a/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers b/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers @@ -0,0 +1 @@ + diff --git a/tubesync/common/huey.py b/tubesync/common/huey.py new file mode 100644 index 00000000..66d13fc2 --- /dev/null +++ b/tubesync/common/huey.py @@ -0,0 +1,37 @@ + + +def sqlite_tasks(key, /, prefix=None): + name_fmt = 'huey_{}' + if prefix is None: + prefix = '' + if prefix: + name_fmt = f'huey_{prefix}_' + '{}' + name = name_fmt.format(key) + return dict( + huey_class='huey.SqliteHuey', + name=name, + immediate=False, + results=True, + store_none=False, + utc=True, + compression=True, + connection=dict( + filename=f'/config/tasks/{name}.db', + fsync=True, + strict_fifo=True, + ), + consumer=dict( + workers=1, + worker_type='process', + max_delay=20.0, + flush_locks=True, + scheduler_interval=10, + simple_log=False, + # verbose has three positions: + # DEBUG: True + # INFO: None + # WARNING: False + verbose=False, + ), + ) + diff --git a/tubesync/sync/choices.py b/tubesync/sync/choices.py index 92cb6c8f..238aa160 100644 --- a/tubesync/sync/choices.py +++ b/tubesync/sync/choices.py @@ -165,6 +165,7 @@ class TaskQueue(models.TextChoices): DB = 'database', _('Database') FS = 'filesystem', _('Filesystem') NET = 'network', _('Networking') + LIMIT = 'limited', _('Limited Networking') class WeekDay(models.IntegerChoices): diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 7a824e7c..f5689dd1 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -1,5 +1,6 @@ from django import VERSION as DJANGO_VERSION from pathlib import Path +from common.huey import sqlite_tasks from common.utils import getenv @@ -24,6 +25,7 @@ INSTALLED_APPS = [ 'django.contrib.humanize', 'sass_processor', 'background_task', + 'django_huey', 'common', 'sync', ] @@ -47,6 +49,22 @@ ROOT_URLCONF = 'tubesync.urls' FORCE_SCRIPT_NAME = None +DJANGO_HUEY = { + 'default': 'network', + 'queues': { + 'database': sqlite_tasks('database'), + 'filesystem': sqlite_tasks('filesystem'), + 'limited': sqlite_tasks('limited', prefix='net'), + 'network': sqlite_tasks('network'), + }, +} +for django_huey_queue in DJANGO_HUEY['queues'].values(): + connection = django_huey_queue.get('connection') + if connection: + filepath = Path('/.' + connection.get('filename') or '').resolve(strict=False) + filepath.parent.mkdir(exist_ok=True, parents=True) + + TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates',