wimby/tubesync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1075 from tcely/patch-6

Browse Source 
Queues managed with `django-huey`
This commit is contained in:
meeb
2025-06-11 20:42:56 +10:00
committed by GitHub
parent 07c9c458ad bfaa76c3dd
commit 9d5cb13bed
37 changed files with 123 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci.yaml vendored
View File

@@ -132,6 +132,8 @@ jobs:
pip install --system --strict --requirements requirements.txt
- name: Set up Django environment
run: |
mkdir -v -p ~/.config/TubeSync/config
sudo ln -v -s -f -T ~/.config/TubeSync/config /config
cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py
cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/*
cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/*

5
Dockerfile
View File

@@ -512,13 +512,14 @@ COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings.
# Build app
RUN set -x && \
# Make absolutely sure we didn't accidentally bundle a SQLite dev database
rm -rf /app/db.sqlite3 && \
test '!' -e /app/db.sqlite3 && \
# Run any required app commands
/usr/bin/python3 -B /app/manage.py compilescss && \
/usr/bin/python3 -B /app/manage.py collectstatic --no-input --link && \
rm -rf /config /downloads /run/app && \
# Create config, downloads and run dirs
mkdir -v -p /run/app && \
mkdir -v -p /config/media && \
mkdir -v -p /config/media /config/tasks && \
mkdir -v -p /config/cache/pycache && \
mkdir -v -p /downloads/audio && \
mkdir -v -p /downloads/video && \

1
Pipfile
View File

@@ -8,6 +8,7 @@ autopep8 = "*"
[packages]
django = "~=5.2.1"
django-huey = "*"
django-sass-processor = {extras = ["management-command"], version = "*"}
pillow = "*"
whitenoise = "*"

1
config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-db-worker Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-fs-worker Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type Normal file
View File

@@ -0,0 +1 @@
bundle

1
config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/huey-consumers/type Normal file
View File

@@ -0,0 +1 @@
bundle

1
config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/base Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/huey-database/dependencies.d/gunicorn Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/huey-database/down-signal Normal file
View File

@@ -0,0 +1 @@
SIGINT

5
config/root/etc/s6-overlay/s6-rc.d/huey-database/run Normal file
View File

@@ -0,0 +1,5 @@
#!/command/with-contenv bash
exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \
/usr/bin/python3 /app/manage.py djangohuey \
--queue database

1
config/root/etc/s6-overlay/s6-rc.d/huey-database/type Normal file
View File

@@ -0,0 +1 @@
longrun

1
config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/base Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/dependencies.d/gunicorn Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/down-signal Normal file
View File

@@ -0,0 +1 @@
SIGINT

5
config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run Normal file
View File

@@ -0,0 +1,5 @@
#!/command/with-contenv bash
exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \
/usr/bin/python3 /app/manage.py djangohuey \
--queue filesystem

1
config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/type Normal file
View File

@@ -0,0 +1 @@
longrun

1
config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/base Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/dependencies.d/gunicorn Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/down-signal Normal file
View File

@@ -0,0 +1 @@
SIGINT

5
config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run Normal file
View File

@@ -0,0 +1,5 @@
#!/command/with-contenv bash
exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \
/usr/bin/python3 /app/manage.py djangohuey \
--queue limited

1
config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/type Normal file
View File

@@ -0,0 +1 @@
longrun

1
config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/base Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/huey-network/dependencies.d/gunicorn Normal file
View File

@@ -0,0 +1 @@

1
config/root/etc/s6-overlay/s6-rc.d/huey-network/down-signal Normal file
View File

@@ -0,0 +1 @@
SIGINT

5
config/root/etc/s6-overlay/s6-rc.d/huey-network/run Normal file
View File

@@ -0,0 +1,5 @@
#!/command/with-contenv bash
exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \
/usr/bin/python3 /app/manage.py djangohuey \
--queue network

1
config/root/etc/s6-overlay/s6-rc.d/huey-network/type Normal file
View File

@@ -0,0 +1 @@
longrun

15
config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run
View File

@@ -4,6 +4,21 @@
groupmod -o -g "${PGID:=911}" app
usermod -o -u "${PUID:=911}" app
# Ensure /config directories exist
mkdir -v -p /config/{cache,media,tasks,tubesync}
# Copy local_settings.py for the user
if [ -f /config/tubesync/local_settings.py ]
then
# from the image for an example or comparison
cp -v -p /app/tubesync/local_settings.py \
/config/tubesync/local_settings.py.image
# to the image to apply the user's settings
cp -v -p /config/tubesync/local_settings.py \
/app/tubesync/local_settings.py
fi
# Reset permissions
chown -R app:app /run/app
chmod -R 0700 /run/app

1
config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers Normal file
View File

@@ -0,0 +1 @@

37
tubesync/common/huey.py Normal file
View File

@@ -0,0 +1,37 @@
def sqlite_tasks(key, /, prefix=None):
name_fmt = 'huey_{}'
if prefix is None:
prefix = ''
if prefix:
name_fmt = f'huey_{prefix}_' + '{}'
name = name_fmt.format(key)
return dict(
huey_class='huey.SqliteHuey',
name=name,
immediate=False,
results=True,
store_none=False,
utc=True,
compression=True,
connection=dict(
filename=f'/config/tasks/{name}.db',
fsync=True,
strict_fifo=True,
),
consumer=dict(
workers=1,
worker_type='process',
max_delay=20.0,
flush_locks=True,
scheduler_interval=10,
simple_log=False,
# verbose has three positions:
# DEBUG: True
# INFO: None
# WARNING: False
verbose=False,
),
)

1
tubesync/sync/choices.py
View File

@@ -165,6 +165,7 @@ class TaskQueue(models.TextChoices):
DB = 'database', _('Database')
FS = 'filesystem', _('Filesystem')
NET = 'network', _('Networking')
LIMIT = 'limited', _('Limited Networking')
class WeekDay(models.IntegerChoices):

18
tubesync/tubesync/settings.py
View File

@@ -1,5 +1,6 @@
from django import VERSION as DJANGO_VERSION
from pathlib import Path
from common.huey import sqlite_tasks
from common.utils import getenv
@@ -24,6 +25,7 @@ INSTALLED_APPS = [
'django.contrib.humanize',
'sass_processor',
'background_task',
'django_huey',
'common',
'sync',
]
@@ -47,6 +49,22 @@ ROOT_URLCONF = 'tubesync.urls'
FORCE_SCRIPT_NAME = None
DJANGO_HUEY = {
'default': 'network',
'queues': {
'database': sqlite_tasks('database'),
'filesystem': sqlite_tasks('filesystem'),
'limited': sqlite_tasks('limited', prefix='net'),
'network': sqlite_tasks('network'),
},
}
for django_huey_queue in DJANGO_HUEY['queues'].values():
connection = django_huey_queue.get('connection')
if connection:
filepath = Path('/.' + connection.get('filename') or '').resolve(strict=False)
filepath.parent.mkdir(exist_ok=True, parents=True)
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',