From fb7d120f8117e2c8d70096610ead274aaa9f4360 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 23 Nov 2025 14:19:39 -0500 Subject: [PATCH 01/25] Eliminate duplicate ensure_connection calls --- tubesync/sync/management/commands/fix-mariadb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 9b21d2df..16810272 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -6,6 +6,7 @@ from django.core.management.base import BaseCommand, CommandError from common.logger import log +db.connection.ensure_connection() db_tables = db.connection.introspection.table_names db_quote_name = db.connection.ops.quote_name new_tables = { @@ -24,7 +25,6 @@ def SQLTable(arg_table): needle = arg_table if needle.startswith('new__'): needle = arg_table[len('new__'):] - db.connection.ensure_connection() valid_table_name = ( needle in new_tables and arg_table in db_tables(include_views=False) @@ -123,7 +123,6 @@ class Command(BaseCommand): + f': {db.connection.vendor}' ) - db.connection.ensure_connection() db_is_mariadb = ( hasattr(db.connection, 'mysql_is_mariadb') and db.connection.is_usable() and From c0550dd419e29557286dc7cd91ae1acc5114af64 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 25 Nov 2025 21:14:03 -0500 Subject: [PATCH 02/25] Release for 2025-012 --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index f9567ad8..61df6d55 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -10,7 +10,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.11' +VERSION = '0.15.12' SECRET_KEY = '' DEBUG = 'true' == getenv('TUBESYNC_DEBUG').strip().lower() ALLOWED_HOSTS = [] From 9700460d5382f5c5f38ac554b3fc533f5bebe5f9 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 15 Dec 2025 00:00:29 -0500 Subject: [PATCH 03/25] Include the latest `yt-dlp` --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 61df6d55..a9f5f120 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -10,7 +10,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.12' +VERSION = '0.15.13' SECRET_KEY = '' DEBUG = 'true' == getenv('TUBESYNC_DEBUG').strip().lower() ALLOWED_HOSTS = [] From 146be62b67b0bf58c557cc8e63af4d4e11b4b682 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 23 Dec 2025 00:07:44 -0500 Subject: [PATCH 04/25] Use bind mount for `deno` and `uv` --- Dockerfile | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6e6212f6..097f0010 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,10 +65,14 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va rm -f /var/cache/debconf/*.dat-old FROM ghcr.io/astral-sh/uv:latest AS uv-binaries + +FROM scratch AS uv +COPY --from=uv-binaries /uv /uvx /usr/local/bin/ + FROM denoland/deno:bin AS deno-binaries FROM scratch AS deno -COPY --from=deno-binaries /deno /usr/local/bin/deno +COPY --from=deno-binaries /deno /usr/local/bin/ FROM alpine:${ALPINE_VERSION} AS openresty-debian ARG TARGETARCH @@ -354,9 +358,6 @@ RUN --mount=type=bind,source=fontawesome-free,target=/fontawesome-free \ FROM scratch AS tubesync-app COPY --from=tubesync-prepare-app /app /app -FROM tubesync-base AS tubesync-uv -COPY --from=uv-binaries /uv /uvx /usr/local/bin/ - FROM tubesync-base AS tubesync-openresty COPY --from=openresty-debian \ @@ -486,13 +487,19 @@ ARG YTDLP_DATE # Set up the app RUN --mount=type=tmpfs,target=/cache \ + --mount=type=cache,id=deno-cache,sharing=locked,target=/cache/deno \ --mount=type=cache,id=uv-cache,sharing=locked,target=/cache/uv \ --mount=type=cache,id=pipenv-cache,sharing=locked,target=/cache/pipenv \ --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/var/lib/apt \ --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \ - --mount=type=bind,source=/uv,target=/usr/local/bin/uv,from=uv-binaries \ + --mount=type=bind,source=/usr/local/bin/deno,target=/usr/local/bin/deno,from=deno \ + --mount=type=bind,source=/usr/local/bin/uv,target=/usr/local/bin/uv,from=uv \ + --mount=type=bind,source=/usr/local/bin/uvx,target=/usr/local/bin/uvx,from=uv \ --mount=type=bind,source=Pipfile,target=/app/Pipfile \ set -x && \ + DENO_DIR=/cache/deno && export DENO_DIR && \ + deno --version && \ + uv --version && \ apt-get update && \ # Install required build packages apt-get -y --no-install-recommends install \ From fa2438e5d50e831010ca24abc61564ca8dcba37f Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 23 Dec 2025 00:31:51 -0500 Subject: [PATCH 05/25] Add `--only-record-version` flag to install_deno.sh --- tubesync/install_deno.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tubesync/install_deno.sh b/tubesync/install_deno.sh index c9f5e6e8..93824007 100755 --- a/tubesync/install_deno.sh +++ b/tubesync/install_deno.sh @@ -48,6 +48,8 @@ work_dir="$(mktemp -d)" trap "rm -rf -- '${work_dir}'" EXIT cd "${work_dir}" -download_deno "${deno_archive}" -extract_deno "${deno_archive}" '/usr/local/bin' +if [ '--only-record-version' != "${1-unset}" ]; then + download_deno "${deno_archive}" + extract_deno "${deno_archive}" '/usr/local/bin' +fi record_deno_version '/usr/local/bin/deno' From 7f701fa63e6ade7a83d4b0f16d4f4e9a55711eb8 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 23 Dec 2025 01:32:01 -0500 Subject: [PATCH 06/25] Use `/app/install_deno.sh --only-record-version` --- Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Dockerfile b/Dockerfile index 097f0010..a1dcac9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -577,6 +577,9 @@ RUN --mount=type=tmpfs,target=/cache \ exit 1 ; \ fi +# Bundle deno with the image +##COPY --from=deno /usr/local/bin/ /usr/local/bin/ + # Copy root COPY config/root / @@ -589,6 +592,11 @@ COPY --from=tubesync-app /app /app # Build app RUN set -x && \ + # Record the bundled deno version when it was included + ( deno_binary="$(command -v deno)" ; \ + test '!' -n "${deno_binary}" || \ + /app/install_deno.sh --only-record-version ; \ + ) && \ # Make absolutely sure we didn't accidentally bundle a SQLite dev database test '!' -e /app/db.sqlite3 && \ # Run any required app commands From 3b56565e8cd653b1116b7fb2e0b1526538f01e3a Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 23 Dec 2025 02:11:41 -0500 Subject: [PATCH 07/25] Find `deno` in `PATH` or default to `./deno` --- tubesync/install_deno.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/install_deno.sh b/tubesync/install_deno.sh index 93824007..b6584acf 100755 --- a/tubesync/install_deno.sh +++ b/tubesync/install_deno.sh @@ -51,5 +51,7 @@ cd "${work_dir}" if [ '--only-record-version' != "${1-unset}" ]; then download_deno "${deno_archive}" extract_deno "${deno_archive}" '/usr/local/bin' + record_deno_version '/usr/local/bin/deno' +else + record_deno_version "$(command -v deno)" fi -record_deno_version '/usr/local/bin/deno' From 6618816eabf8760dd1938a867ec929b71a3526d0 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 23 Dec 2025 06:01:58 -0500 Subject: [PATCH 08/25] Add `asfald` download tool --- Dockerfile | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/Dockerfile b/Dockerfile index a1dcac9e..3fc63114 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,11 @@ ARG FFMPEG_VERSION="N" +ARG ASFALD_VERSION="0.9.0" + +ARG SHA256_ASFALD_AMD64="0b7e1270ecc5a4c37785511b032477ddc3a7cf718b0b2f8542229f39adf2ce6a" +ARG SHA256_ASFALD_ARM64="7c18868e3b8d0edf134ac101cdbf4b243c31735d3afe58f90f0b5a82394a6154" + ARG S6_VERSION="3.2.0.3" ARG SHA256_S6_AMD64="01eb9a6dce10b5428655974f1903f48e7ba7074506dfb262e85ffab64a5498f2" @@ -19,6 +24,7 @@ ARG DEBIAN_VERSION="bookworm-slim" ARG FFMPEG_PREFIX_FILE="ffmpeg-${FFMPEG_VERSION}" ARG FFMPEG_SUFFIX_FILE=".tar.xz" +ARG ASFALD_CHECKSUM_ALGORITHM="sha256" ARG FFMPEG_CHECKSUM_ALGORITHM="sha256" ARG S6_CHECKSUM_ALGORITHM="sha256" ARG QJS_CHECKSUM_ALGORITHM="sha256" @@ -64,6 +70,72 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va apt-get -y autoclean && \ rm -f /var/cache/debconf/*.dat-old +FROM alpine:${ALPINE_VERSION} AS asfald-download +ARG ASFALD_VERSION +ARG SHA256_ASFALD_AMD64 +ARG SHA256_ASFALD_ARM64 + +ARG DESTDIR="/downloaded" +ARG ASFALD_CHECKSUM_ALGORITHM +ARG CHECKSUM_ALGORITHM="${ASFALD_CHECKSUM_ALGORITHM}" + +ARG ASFALD_CHECKSUM_AMD64="${CHECKSUM_ALGORITHM}:${SHA256_ASFALD_AMD64}" +ARG ASFALD_CHECKSUM_ARM64="${CHECKSUM_ALGORITHM}:${SHA256_ASFALD_ARM64}" + +ARG ASFALD_DOWNLOAD_URI="asfaload/asfald/releases/download/v${ASFALD_VERSION}" +ARG ASFALD_URL="https://github.com/${ASFALD_DOWNLOAD_URI}" +ARG ASFALD_SUMS_URL="https://gh.checksums.asfaload.com/github.com/${ASFALD_DOWNLOAD_URI}/checksums.txt" +ARG ASFALD_PREFIX_FILE="asfald-" +ARG ASFALD_SUFFIX_FILE="-unknown-linux-musl" + +ARG ASFALD_FILE_AMD64="${ASFALD_PREFIX_FILE}x86_64${ASFALD_SUFFIX_FILE}" +ARG ASFALD_FILE_ARM64="${ASFALD_PREFIX_FILE}aarch64${ASFALD_SUFFIX_FILE}" + +ADD "${ASFALD_SUMS_URL}" "${DESTDIR}/" +ADD "${ASFALD_URL}/${ASFALD_FILE_AMD64}" "${DESTDIR}/" +ADD "${ASFALD_URL}/${ASFALD_FILE_ARM64}" "${DESTDIR}/" + +##ADD --checksum="${ASFALD_CHECKSUM_AMD64}" "${ASFALD_URL}/${ASFALD_FILE_AMD64}" "${DESTDIR}/" +##ADD --checksum="${ASFALD_CHECKSUM_ARM64}" "${ASFALD_URL}/${ASFALD_FILE_ARM64}" "${DESTDIR}/" + +# --checksum wasn't recognized, so use busybox to check the sums instead +ARG TARGETARCH +RUN set -eu ; \ + apk --no-cache --no-progress add "cmd:${CHECKSUM_ALGORITHM}sum" ; \ +\ + decide_expected() { \ + case "${TARGETARCH}" in \ + (amd64) printf -- '%s' "${ASFALD_CHECKSUM_AMD64}" ;; \ + (arm64) printf -- '%s' "${ASFALD_CHECKSUM_ARM64}" ;; \ + (*) exit 1 ;; \ + esac ; \ + } ; \ +\ + decide_fn() { \ + case "${TARGETARCH}" in \ + (amd64) printf -- '%s\n' "${ASFALD_FILE_AMD64}" ;; \ + (arm64) printf -- '%s\n' "${ASFALD_FILE_ARM64}" ;; \ + (*) exit 1 ;; \ + esac ; \ + } ; \ +\ + checksum="$(decide_expected)" ; \ + file="$(decide_fn)" ; \ + mkdir -v -p "/verified/${TARGETARCH}" ; \ + cd "${DESTDIR}/" && \ + "${CHECKSUM_ALGORITHM}sum" --check --warn --strict --ignore-missing checksums.txt && \ + printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw && \ + mv -v "${file}" "/verified/${TARGETARCH}/asfald" && \ + chmod -v 00755 "/verified/${TARGETARCH}/asfald" && \ + chown -v root:root "/verified/${TARGETARCH}/asfald" + +FROM scratch AS asfald +ARG TARGETARCH +COPY --from=asfald-download "/verified/${TARGETARCH}/asfald" /usr/local/sbin/ + +FROM tubesync-base AS tubesync-asfald +COPY --from=asfald /usr/local/sbin/ /usr/local/sbin/ + FROM ghcr.io/astral-sh/uv:latest AS uv-binaries FROM scratch AS uv From 0683c03ea3969860f4c5c49156aa5a29360609cf Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 23 Dec 2025 10:34:07 -0500 Subject: [PATCH 09/25] Use `asfald` to download s6_overlay files - Pin to `v0.6.0` to use the `--hash` flag --- Dockerfile | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3fc63114..11ea62d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,10 @@ ARG FFMPEG_VERSION="N" -ARG ASFALD_VERSION="0.9.0" +ARG ASFALD_VERSION="0.6.0" -ARG SHA256_ASFALD_AMD64="0b7e1270ecc5a4c37785511b032477ddc3a7cf718b0b2f8542229f39adf2ce6a" -ARG SHA256_ASFALD_ARM64="7c18868e3b8d0edf134ac101cdbf4b243c31735d3afe58f90f0b5a82394a6154" +ARG SHA256_ASFALD_AMD64="017cdc44d767bb4733a3bd3fa5f97719e3f58236d006321dfae1924fdda3de9d" +ARG SHA256_ASFALD_ARM64="4fc112617a71f97592b8760b98c16339d5243577186c970c784fbcab2ef8abd1" ARG S6_VERSION="3.2.0.3" @@ -273,7 +273,7 @@ RUN set -eux ; \ FROM scratch AS ffmpeg COPY --from=ffmpeg-extracted /extracted /usr/local/bin/ -FROM alpine:${ALPINE_VERSION} AS s6-overlay-download +FROM tubesync-asfald AS s6-overlay-download ARG S6_VERSION ARG SHA256_S6_AMD64 ARG SHA256_S6_ARM64 @@ -303,18 +303,11 @@ ADD "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}.${CHECKSUM_ALGORITHM}" "${DESTDIR}/" ##ADD --checksum="${S6_CHECKSUM_ARM64}" "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" "${DESTDIR}/" ##ADD --checksum="${S6_CHECKSUM_NOARCH}" "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" "${DESTDIR}/" -# --checksum wasn't recognized, so use busybox to check the sums instead -ADD "${S6_OVERLAY_URL}/${S6_FILE_AMD64}" "${DESTDIR}/" -RUN set -eu ; checksum="${S6_CHECKSUM_AMD64}" ; file="${S6_FILE_AMD64}" ; cd "${DESTDIR}/" && \ - printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw - -ADD "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" "${DESTDIR}/" -RUN set -eu ; checksum="${S6_CHECKSUM_ARM64}" ; file="${S6_FILE_ARM64}" ; cd "${DESTDIR}/" && \ - printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw - -ADD "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" "${DESTDIR}/" -RUN set -eu ; checksum="${S6_CHECKSUM_NOARCH}" ; file="${S6_FILE_NOARCH}" ; cd "${DESTDIR}/" && \ - printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw +# --checksum wasn't recognized, so use asfald to check the sums instead +RUN set -eux ; cd "${DESTDIR}/" && \ + asfald --hash="${SHA256_S6_AMD64}" -- "${S6_OVERLAY_URL}/${S6_FILE_AMD64}" && \ + asfald --hash="${SHA256_S6_ARM64}" -- "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" && \ + asfald --hash="${SHA256_S6_NOARCH}" -- "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" FROM alpine:${ALPINE_VERSION} AS s6-overlay-extracted COPY --from=s6-overlay-download /downloaded /downloaded From 141e21fa996a91133693d28e4720111b9f07fe2b Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 23 Dec 2025 10:34:10 -0500 Subject: [PATCH 10/25] Also install the latest `asfald` version --- Dockerfile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Dockerfile b/Dockerfile index 11ea62d0..302c751b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -136,6 +136,24 @@ COPY --from=asfald-download "/verified/${TARGETARCH}/asfald" /usr/local/sbin/ FROM tubesync-base AS tubesync-asfald COPY --from=asfald /usr/local/sbin/ /usr/local/sbin/ +ARG TARGETARCH +RUN set -eu ; \ +\ + decide_arch() { \ + case "${TARGETARCH}" in \ + (amd64) printf -- 'x86_64' ;; \ + (arm64) printf -- 'aarch64' ;; \ + (*) exit 1 ;; \ + esac ; \ + } ; \ +\ + set -x ; arch="$(decide_arch)" ; \ + dest='/usr/local/sbin/asfald-latest' ; \ + asfald --overwrite --output "${dest}" \ + --pattern '${path}/checksums.txt' -- \ + "https://github.com/asfaload/asfald/releases/latest/download/asfald-${arch}-unknown-linux-musl" && \ + chmod -c 00755 "${dest}" && chown -c root:root "${dest}" + FROM ghcr.io/astral-sh/uv:latest AS uv-binaries FROM scratch AS uv From e032d28aa1039d9af5cdee542b36203d756d58a3 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 23 Dec 2025 10:34:13 -0500 Subject: [PATCH 11/25] Work around rename limitations by overriding `TMPDIR` --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 302c751b..c692d5f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -149,6 +149,7 @@ RUN set -eu ; \ \ set -x ; arch="$(decide_arch)" ; \ dest='/usr/local/sbin/asfald-latest' ; \ + TMPDIR="$(dirname "${dest}")" \ asfald --overwrite --output "${dest}" \ --pattern '${path}/checksums.txt' -- \ "https://github.com/asfaload/asfald/releases/latest/download/asfald-${arch}-unknown-linux-musl" && \ @@ -322,7 +323,7 @@ ADD "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}.${CHECKSUM_ALGORITHM}" "${DESTDIR}/" ##ADD --checksum="${S6_CHECKSUM_NOARCH}" "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" "${DESTDIR}/" # --checksum wasn't recognized, so use asfald to check the sums instead -RUN set -eux ; cd "${DESTDIR}/" && \ +RUN set -eux ; TMPDIR="${DESTDIR}" ; export TMPDIR ; cd "${DESTDIR}/" && \ asfald --hash="${SHA256_S6_AMD64}" -- "${S6_OVERLAY_URL}/${S6_FILE_AMD64}" && \ asfald --hash="${SHA256_S6_ARM64}" -- "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" && \ asfald --hash="${SHA256_S6_NOARCH}" -- "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" From b6d83dd09964aa5860b0a78ef974b9fafcd55396 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 23 Dec 2025 10:34:16 -0500 Subject: [PATCH 12/25] Use `asfald` to download quickjs files --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index c692d5f7..0e1b67d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -380,7 +380,7 @@ RUN set -eu ; \ ## COPY --from=s6-overlay-extracted /s6-overlay-rootfs / FROM ghcr.io/meeb/s6-overlay:v${S6_VERSION} AS s6-overlay -FROM alpine:${ALPINE_VERSION} AS quickjs-download +FROM tubesync-asfald AS quickjs-download ARG QJS_VERSION ARG SHA256_QJS @@ -390,7 +390,7 @@ ARG CHECKSUM_ALGORITHM="${QJS_CHECKSUM_ALGORITHM}" ARG QJS_CHECKSUM="${CHECKSUM_ALGORITHM}:${SHA256_QJS}" -ARG QJS_URL="https://bellard.org/quickjs/binary_releases/" +ARG QJS_URL="https://bellard.org/quickjs/binary_releases" ARG QJS_PREFIX_FILE="quickjs-cosmo-" ARG QJS_SUFFIX_FILE=".zip" @@ -398,10 +398,10 @@ ARG QJS_FILE="${QJS_PREFIX_FILE}${QJS_VERSION}${QJS_SUFFIX_FILE}" ##ADD --checksum="${QJS_CHECKSUM}" "${QJS_URL}/${QJS_FILE}" "${DESTDIR}/" -# --checksum wasn't recognized, so use busybox to check the sums instead -ADD "${QJS_URL}/${QJS_FILE}" "${DESTDIR}/" -RUN set -eu ; checksum="${QJS_CHECKSUM}" ; file="${QJS_FILE}" ; cd "${DESTDIR}/" && \ - printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw +# --checksum wasn't recognized, so use asfald to check the sums instead +RUN set -eux ; TMPDIR="${DESTDIR}" ; export TMPDIR ; \ + mkdir -v -p "${DESTDIR}" && cd "${DESTDIR}/" && \ + asfald --hash="${SHA256_QJS}" -- "${QJS_URL}/${QJS_FILE}" FROM alpine:${ALPINE_VERSION} AS quickjs-extracted COPY --from=quickjs-download /downloaded /downloaded From 78aca2b018a3102ec389194589d5b8c64a512a2e Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 23 Dec 2025 11:01:53 -0500 Subject: [PATCH 13/25] Tweak ffmpeg downloading --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0e1b67d3..bb65b7c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -213,12 +213,12 @@ RUN set -eu ; \ bytes="${CHECKSUM_ALGORITHM#${algorithm}}" ; \ hash="$( awk -v fn="${1##*/}" '$0 ~ fn"$" { print $1; exit; }' "${DESTDIR}/${FFMPEG_FILE_SUMS}" )" ; \ \ + printf -- '%s\n' "${1}" ; \ printf -- '\t%s\n' \ 'allow-overwrite=true' \ 'always-resume=false' \ 'check-integrity=true' \ "checksum=${algorithm}-${bytes}=${hash}" \ - 'max-connection-per-server=2' \ ; \ printf -- '\n' ; \ } ; \ @@ -237,15 +237,15 @@ RUN set -eu ; \ ' "${DESTDIR}/${FFMPEG_FILE_SUMS}") ; \ do \ url="${FFMPEG_URL}/${url# }" ; \ - printf -- '%s\n' "${url}" ; \ aria2c_options "${url}" ; \ - printf -- '\n' ; \ done > /tmp/downloads ; \ + cat -v -n /tmp/downloads ; \ unset -v url ; \ \ aria2c --no-conf=true \ - --dir /downloaded \ + --dir "${DESTDIR}" \ --lowest-speed-limit='16K' \ + --max-connection-per-server='2' \ --show-console-readout=false \ --summary-interval=0 \ --input-file /tmp/downloads ; \ From 7cd5eea296a6d4d2a4a2e482ab66c70cf4ffc9bf Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 23 Dec 2025 11:14:47 -0500 Subject: [PATCH 14/25] Use `asfald-latest` to download ffmpeg files --- Dockerfile | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb65b7c3..5333114b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -187,7 +187,7 @@ RUN set -eu ; \ 'deb http://openresty.org/package/%sdebian %s openresty' \ "$(decide_arch)" "${DEBIAN_VERSION%-slim}" -FROM alpine:${ALPINE_VERSION} AS ffmpeg-download +FROM tubesync-asfald AS ffmpeg-download ARG FFMPEG_DATE ARG FFMPEG_VERSION ARG FFMPEG_PREFIX_FILE @@ -206,22 +206,6 @@ ARG DESTDIR="/downloaded" ARG TARGETARCH ADD "${FFMPEG_URL}/${FFMPEG_FILE_SUMS}" "${DESTDIR}/" RUN set -eu ; \ - apk --no-cache --no-progress add cmd:aria2c cmd:awk "cmd:${CHECKSUM_ALGORITHM}sum" ; \ -\ - aria2c_options() { \ - algorithm="${CHECKSUM_ALGORITHM%[0-9]??}" ; \ - bytes="${CHECKSUM_ALGORITHM#${algorithm}}" ; \ - hash="$( awk -v fn="${1##*/}" '$0 ~ fn"$" { print $1; exit; }' "${DESTDIR}/${FFMPEG_FILE_SUMS}" )" ; \ -\ - printf -- '%s\n' "${1}" ; \ - printf -- '\t%s\n' \ - 'allow-overwrite=true' \ - 'always-resume=false' \ - 'check-integrity=true' \ - "checksum=${algorithm}-${bytes}=${hash}" \ -; \ - printf -- '\n' ; \ - } ; \ \ decide_arch() { \ case "${TARGETARCH}" in \ @@ -232,23 +216,15 @@ RUN set -eu ; \ \ FFMPEG_ARCH="$(decide_arch)" ; \ FFMPEG_PREFIX_FILE="$( printf -- '%s' "${FFMPEG_PREFIX_FILE}" | cut -d '-' -f 1,2 )" ; \ + cd "${DESTDIR}" && \ for url in $(awk ' \ $2 ~ /^[*]?'"${FFMPEG_PREFIX_FILE}"'/ && /-'"${FFMPEG_ARCH}"'-/ { $1=""; print; } \ ' "${DESTDIR}/${FFMPEG_FILE_SUMS}") ; \ do \ url="${FFMPEG_URL}/${url# }" ; \ - aria2c_options "${url}" ; \ - done > /tmp/downloads ; \ - cat -v -n /tmp/downloads ; \ + TMPDIR="${DESTDIR}" asfald-latest -qv -- "${url}" ; \ + done ; \ unset -v url ; \ -\ - aria2c --no-conf=true \ - --dir "${DESTDIR}" \ - --lowest-speed-limit='16K' \ - --max-connection-per-server='2' \ - --show-console-readout=false \ - --summary-interval=0 \ - --input-file /tmp/downloads ; \ \ decide_expected() { \ case "${TARGETARCH}" in \ @@ -259,7 +235,6 @@ RUN set -eu ; \ \ FFMPEG_HASH="$(decide_expected)" ; \ \ - cd "${DESTDIR}" ; \ if [ -n "${FFMPEG_HASH}" ] ; \ then \ printf -- '%s *%s\n' "${FFMPEG_HASH}" "${FFMPEG_PREFIX_FILE}"*-"${FFMPEG_ARCH}"-*"${FFMPEG_SUFFIX_FILE}" >> /tmp/SUMS ; \ From faa5464075f182431472f75ff8de27ef51383bed Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 27 Dec 2025 05:59:51 -0500 Subject: [PATCH 15/25] Run the token web server with `deno` --- Dockerfile | 21 ++++++++++++++ config/root/etc/nginx/token_server.conf | 2 +- .../dependencies | 2 ++ .../bgutil-ytdlp-pot-provider-build/run | 28 +++++++++++++++++++ .../bgutil-ytdlp-pot-provider-build/type | 1 + .../bgutil-ytdlp-pot-provider-build/up | 3 ++ .../bgutil-ytdlp-pot-provider/dependencies | 2 ++ .../s6-rc.d/bgutil-ytdlp-pot-provider/run | 7 +++++ .../s6-rc.d/bgutil-ytdlp-pot-provider/type | 1 + .../s6-overlay/s6-rc.d/gunicorn/dependencies | 3 +- .../etc/s6-overlay/s6-rc.d/huey-database/run | 0 .../s6-overlay/s6-rc.d/huey-filesystem/run | 0 .../s6-overlay/s6-rc.d/huey-net-limited/run | 0 .../etc/s6-overlay/s6-rc.d/huey-network/run | 0 .../etc/s6-overlay/s6-rc.d/nginx/dependencies | 3 +- .../s6-rc.d/tubesync-init/dependancies | 0 .../s6-rc.d/tubesync-init/dependencies | 1 + tubesync/bgutil-ytdlp-pot-provider/node | 14 ++++++++++ 18 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/dependencies create mode 100755 config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run create mode 100644 config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/type create mode 100755 config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/up create mode 100644 config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/dependencies create mode 100755 config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/run create mode 100644 config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/type mode change 100644 => 100755 config/root/etc/s6-overlay/s6-rc.d/huey-database/run mode change 100644 => 100755 config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run mode change 100644 => 100755 config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run mode change 100644 => 100755 config/root/etc/s6-overlay/s6-rc.d/huey-network/run delete mode 100644 config/root/etc/s6-overlay/s6-rc.d/tubesync-init/dependancies create mode 100644 config/root/etc/s6-overlay/s6-rc.d/tubesync-init/dependencies create mode 100755 tubesync/bgutil-ytdlp-pot-provider/node diff --git a/Dockerfile b/Dockerfile index 5333114b..68f59d74 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ # syntax=docker/dockerfile:1 # check=error=true +ARG BGUTIL_YTDLP_POT_PROVIDER_VERSION="1.2.2" ARG FFMPEG_VERSION="N" ARG ASFALD_VERSION="0.6.0" @@ -414,6 +415,14 @@ RUN --mount=type=bind,source=fontawesome-free,target=/fontawesome-free \ rm -v /app/tubesync/local_settings.py.example && \ mv -v /app/tubesync/local_settings.py.container /app/tubesync/local_settings.py +ARG BGUTIL_YTDLP_POT_PROVIDER_VERSION +ADD "https://github.com/Brainicism/bgutil-ytdlp-pot-provider/archive/refs/tags/${BGUTIL_YTDLP_POT_PROVIDER_VERSION}.tar.gz" /tmp/ +RUN mkdir -v /tmp/extracted && \ + tar -C /tmp/extracted/ -xvvpf "/tmp/${BGUTIL_YTDLP_POT_PROVIDER_VERSION}.tar.gz" && \ + mv -v /tmp/extracted/*/server /app/bgutil-ytdlp-pot-provider/ && \ + ls -alR /app/bgutil-ytdlp-pot-provider && \ + rm -rf /tmp/extracted + FROM scratch AS tubesync-app COPY --from=tubesync-prepare-app /app /app @@ -649,6 +658,18 @@ COPY patches/yt_dlp/ \ # Copy app COPY --from=tubesync-app /app /app +# Build the bgutil-ytdlp-pot-provider server when deno is bundled +RUN --mount=type=tmpfs,target=/cache \ + --mount=type=cache,id=deno-cache,sharing=locked,target=/cache/deno \ + command -v deno || exit 0 ; \ + set -x && \ + DENO_DIR=/cache/deno && export DENO_DIR && \ + cd /app/bgutil-ytdlp-pot-provider/server && \ + install -v -t /usr/local/bin ../node && \ + DENO_COMPAT=1 deno run -A npm:npm ci --no-fund && \ + DENO_COMPAT=1 deno run -A npm:npm audit fix && \ + node typescript:/tsc + # Build app RUN set -x && \ # Record the bundled deno version when it was included diff --git a/config/root/etc/nginx/token_server.conf b/config/root/etc/nginx/token_server.conf index 6c0c4e3a..1c985ab8 100644 --- a/config/root/etc/nginx/token_server.conf +++ b/config/root/etc/nginx/token_server.conf @@ -1,5 +1,5 @@ upstream token_server { - server 127.0.0.2:4416 down; + server 127.0.0.1:4406 max_fails=0; } server { diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/dependencies b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/dependencies new file mode 100644 index 00000000..b1248844 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/dependencies @@ -0,0 +1,2 @@ +base +tubesync-init diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run new file mode 100755 index 00000000..7609ba0f --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run @@ -0,0 +1,28 @@ +#!/command/with-contenv bash + +set -eu +cd /app/bgutil-ytdlp-pot-provider/server + +command -v deno >/dev/null || \ + /app/install_deno.sh + +test '!' -d build || \ + test '!' -f build/main.js || \ + test '!' -d node_modules || \ + exit 0 # nothing needs to be built + +command -v node > /dev/null || \ + install -v -t /usr/local/bin ../node + +chown -R app:app . + +s6-setuidgid app \ + env DENO_COMPAT=1 deno run -A npm:npm ci --no-fund + +s6-setuidgid app \ + env DENO_COMPAT=1 deno run -A npm:npm audit fix + +s6-setuidgid app \ + ../node typescript:/tsc + +chown -R root:app . diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/type b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/type new file mode 100644 index 00000000..bdd22a18 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/type @@ -0,0 +1 @@ +oneshot diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/up b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/up new file mode 100755 index 00000000..67732022 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/up @@ -0,0 +1,3 @@ +#!/command/execlineb -P + +/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/dependencies b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/dependencies new file mode 100644 index 00000000..c15ef0c0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/dependencies @@ -0,0 +1,2 @@ +base +bgutil-ytdlp-pot-provider-build diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/run b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/run new file mode 100755 index 00000000..48c52dcf --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/run @@ -0,0 +1,7 @@ +#!/command/with-contenv bash + +set -e +cd /app/bgutil-ytdlp-pot-provider + +exec s6-setuidgid app \ + ./node server/build/main.js -p 4406 diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/type b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/type new file mode 100644 index 00000000..5883cff0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/s6-rc.d/gunicorn/dependencies b/config/root/etc/s6-overlay/s6-rc.d/gunicorn/dependencies index fe757c20..b1248844 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/gunicorn/dependencies +++ b/config/root/etc/s6-overlay/s6-rc.d/gunicorn/dependencies @@ -1 +1,2 @@ -tubesync-init \ No newline at end of file +base +tubesync-init diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-database/run b/config/root/etc/s6-overlay/s6-rc.d/huey-database/run old mode 100644 new mode 100755 diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run old mode 100644 new mode 100755 diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run old mode 100644 new mode 100755 diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-network/run b/config/root/etc/s6-overlay/s6-rc.d/huey-network/run old mode 100644 new mode 100755 diff --git a/config/root/etc/s6-overlay/s6-rc.d/nginx/dependencies b/config/root/etc/s6-overlay/s6-rc.d/nginx/dependencies index 283e1305..c4f03c23 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/nginx/dependencies +++ b/config/root/etc/s6-overlay/s6-rc.d/nginx/dependencies @@ -1 +1,2 @@ -gunicorn \ No newline at end of file +base +gunicorn diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/dependancies b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/dependancies deleted file mode 100644 index e69de29b..00000000 diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/dependencies b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/dependencies new file mode 100644 index 00000000..df967b96 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/dependencies @@ -0,0 +1 @@ +base diff --git a/tubesync/bgutil-ytdlp-pot-provider/node b/tubesync/bgutil-ytdlp-pot-provider/node new file mode 100755 index 00000000..a3b440aa --- /dev/null +++ b/tubesync/bgutil-ytdlp-pot-provider/node @@ -0,0 +1,14 @@ +#!/usr/bin/env sh + +command -v deno >/dev/null || \ + sleep 24h + +case "${1}%${#}" in + (--version%1) printf -- '%s\n' 'v24.12.0' ; exit ;; +esac + +DENO_COMPAT=1 exec deno run --allow-env --allow-net --allow-sys \ + --allow-ffi="${HOME}" \ + --allow-read=. --allow-read="${HOME}" --allow-read="${XDG_CACHE_HOME}" \ + --allow-write=. --allow-write="${HOME}" --allow-write="${XDG_CACHE_HOME}" \ + "$@" From 7c6f4e000d183209534696d2ad114897f46c21f5 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 27 Dec 2025 06:26:37 -0500 Subject: [PATCH 16/25] Store actually empty files --- .../s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database | 1 - .../s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem | 1 - .../s6-rc.d/huey-consumers/contents.d/huey-net-limited | 1 - .../s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network | 1 - .../root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers | 1 - 5 files changed, 5 deletions(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database index 8b137891..e69de29b 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database @@ -1 +0,0 @@ - diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem index 8b137891..e69de29b 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem @@ -1 +0,0 @@ - diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited index 8b137891..e69de29b 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited @@ -1 +0,0 @@ - diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network index 8b137891..e69de29b 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network @@ -1 +0,0 @@ - diff --git a/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers b/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers index 8b137891..e69de29b 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers +++ b/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers @@ -1 +0,0 @@ - From 0b965b52ed3d9c20d5d43d8c65d3557d84e361e8 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 27 Dec 2025 07:41:07 -0500 Subject: [PATCH 17/25] Fix errors when starting `bgutil-ytdlp-pot-provider-build` - `tubesync-init` changed the permissions of `node` before us - `npm` really likes to write to "${HOME}" - use the correct way to run `typescript/tsc` --- .../s6-rc.d/bgutil-ytdlp-pot-provider-build/run | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run index 7609ba0f..1ee47284 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run @@ -2,6 +2,9 @@ set -eu cd /app/bgutil-ytdlp-pot-provider/server +chmod -c 00755 ../node + +tmp_app_home="$(realpath -e .)" command -v deno >/dev/null || \ /app/install_deno.sh @@ -17,12 +20,12 @@ command -v node > /dev/null || \ chown -R app:app . s6-setuidgid app \ - env DENO_COMPAT=1 deno run -A npm:npm ci --no-fund + env HOME="${tmp_app_home}" DENO_COMPAT=1 deno run -A npm:npm ci --no-fund s6-setuidgid app \ - env DENO_COMPAT=1 deno run -A npm:npm audit fix + env HOME="${tmp_app_home}" DENO_COMPAT=1 deno run -A npm:npm audit fix s6-setuidgid app \ - ../node typescript:/tsc + env HOME="${tmp_app_home}" ../node npm:typescript/tsc chown -R root:app . From 9f4d19de51772fb2065cf48038055d704bc037a2 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 27 Dec 2025 08:16:20 -0500 Subject: [PATCH 18/25] Make and remove a temporary directory --- .../etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run index 1ee47284..c9efae37 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run @@ -4,7 +4,8 @@ set -eu cd /app/bgutil-ytdlp-pot-provider/server chmod -c 00755 ../node -tmp_app_home="$(realpath -e .)" +tmp_app_home="$(realpath -e "$(mktemp --directory --tmpdir=.)")" +trap "rm -rf ${tmp_app_home}" EXIT command -v deno >/dev/null || \ /app/install_deno.sh From ef312be76a9959a6ccd2ecc17b9e32856f382eeb Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 27 Dec 2025 08:37:04 -0500 Subject: [PATCH 19/25] Use a HOME on /cache when bundling `deno` --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 68f59d74..04554338 100644 --- a/Dockerfile +++ b/Dockerfile @@ -666,9 +666,12 @@ RUN --mount=type=tmpfs,target=/cache \ DENO_DIR=/cache/deno && export DENO_DIR && \ cd /app/bgutil-ytdlp-pot-provider/server && \ install -v -t /usr/local/bin ../node && \ + mkdir -v -p /cache/.home-directories && \ + cp -at /cache/.home-directories/ "${HOME}" && \ + HOME="/cache/.home-directories/${HOME#/}" && \ DENO_COMPAT=1 deno run -A npm:npm ci --no-fund && \ DENO_COMPAT=1 deno run -A npm:npm audit fix && \ - node typescript:/tsc + node npm:typescript/tsc # Build app RUN set -x && \ From 958441514fe8d1c7b7d3ca47a61c06c8b460ea89 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 27 Dec 2025 12:28:51 -0500 Subject: [PATCH 20/25] Add a variable used by the `node` wrapper --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 04554338..ccc78411 100644 --- a/Dockerfile +++ b/Dockerfile @@ -662,14 +662,17 @@ COPY --from=tubesync-app /app /app RUN --mount=type=tmpfs,target=/cache \ --mount=type=cache,id=deno-cache,sharing=locked,target=/cache/deno \ command -v deno || exit 0 ; \ + # python might be used, so keep the .pyc files in /cache + PYTHONPYCACHEPREFIX='/cache/pycache' && export PYTHONPYCACHEPREFIX && \ set -x && \ - DENO_DIR=/cache/deno && export DENO_DIR && \ + XDG_CACHE_HOME='/cache' && export XDG_CACHE_HOME && \ + DENO_DIR='/cache/deno' && export DENO_DIR && \ cd /app/bgutil-ytdlp-pot-provider/server && \ install -v -t /usr/local/bin ../node && \ mkdir -v -p /cache/.home-directories && \ cp -at /cache/.home-directories/ "${HOME}" && \ HOME="/cache/.home-directories/${HOME#/}" && \ - DENO_COMPAT=1 deno run -A npm:npm ci --no-fund && \ + DENO_COMPAT=1 deno run -A npm:npm ci --no-audit --no-fund && \ DENO_COMPAT=1 deno run -A npm:npm audit fix && \ node npm:typescript/tsc From e6e24dff7f189c6062a2d7c0263e87bf1c6e8e3c Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 30 Dec 2025 05:47:38 -0500 Subject: [PATCH 21/25] Improve the `node` wrapper When `node` has no arguments, it uses a read-execute-print loop so copy that behavior. Only require that HOME must be set for shell, then let the cryptic error from `deno` handle a foolishly empty/null HOME variable. --- tubesync/bgutil-ytdlp-pot-provider/node | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tubesync/bgutil-ytdlp-pot-provider/node b/tubesync/bgutil-ytdlp-pot-provider/node index a3b440aa..2f28f7a5 100755 --- a/tubesync/bgutil-ytdlp-pot-provider/node +++ b/tubesync/bgutil-ytdlp-pot-provider/node @@ -1,14 +1,16 @@ #!/usr/bin/env sh +set -eu command -v deno >/dev/null || \ sleep 24h -case "${1}%${#}" in +case "${1-}%${#}" in + (%0) DENO_COMPAT=1 exec deno repl ;; (--version%1) printf -- '%s\n' 'v24.12.0' ; exit ;; esac DENO_COMPAT=1 exec deno run --allow-env --allow-net --allow-sys \ --allow-ffi="${HOME}" \ - --allow-read=. --allow-read="${HOME}" --allow-read="${XDG_CACHE_HOME}" \ - --allow-write=. --allow-write="${HOME}" --allow-write="${XDG_CACHE_HOME}" \ + --allow-read=. --allow-read="${HOME}" --allow-read="${XDG_CACHE_HOME:-.}" \ + --allow-write=. --allow-write="${HOME}" --allow-write="${XDG_CACHE_HOME:-.}" \ "$@" From e69a0656591c9e0a8b6a3f403b269ef6daba4fa3 Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 30 Dec 2025 08:28:46 -0500 Subject: [PATCH 22/25] Remove duplication of `qjs` in the final stage --- Dockerfile | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index ccc78411..4beadd17 100644 --- a/Dockerfile +++ b/Dockerfile @@ -356,7 +356,7 @@ RUN set -eu ; \ ## COPY --from=s6-overlay-extracted /s6-overlay-rootfs / FROM ghcr.io/meeb/s6-overlay:v${S6_VERSION} AS s6-overlay -FROM tubesync-asfald AS quickjs-download +FROM tubesync-asfald AS quickjs-extracted ARG QJS_VERSION ARG SHA256_QJS @@ -379,25 +379,24 @@ RUN set -eux ; TMPDIR="${DESTDIR}" ; export TMPDIR ; \ mkdir -v -p "${DESTDIR}" && cd "${DESTDIR}/" && \ asfald --hash="${SHA256_QJS}" -- "${QJS_URL}/${QJS_FILE}" -FROM alpine:${ALPINE_VERSION} AS quickjs-extracted -COPY --from=quickjs-download /downloaded /downloaded - -ARG QJS_CHECKSUM_ALGORITHM -ARG CHECKSUM_ALGORITHM="${QJS_CHECKSUM_ALGORITHM}" - -RUN set -eu ; \ -\ - apk --no-cache --no-progress add "cmd:${CHECKSUM_ALGORITHM}sum" cmd:unzip ; \ +RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/var/lib/apt \ + --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \ + set -eux ; \ + apt-get update ; \ + apt-get -y --no-install-recommends install unzip ; \ mkdir -v /extracted ; \ cd /extracted ; \ for f in /downloaded/*.zip ; \ do \ unzip "${f}" || exit ; \ done ; \ - unset -v f ; + unset -v f ; \ + mkdir -v /assimilated ; \ + install -v -p -t /assimilated /extracted/qjs ; \ + /assimilated/qjs --assimilate FROM scratch AS quickjs -COPY --from=quickjs-extracted /extracted/qjs /usr/local/sbin/ +COPY --from=quickjs-extracted /assimilated/qjs /usr/local/sbin/ FROM tubesync-base AS tubesync-prepare-app @@ -538,7 +537,6 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va /usr/local/bin/ffmpeg -version && \ file /usr/local/bin/ff* && \ # Installed quickjs (using COPY earlier) - /usr/local/sbin/qjs --assimilate && \ /usr/local/sbin/qjs --help | grep -Fe 'QuickJS version' && \ file /usr/local/sbin/qjs && \ # Clean up file From 7b8e02a8deeb9f4d87a75b64b1a5aa6e2cbbc5e0 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 31 Dec 2025 06:29:25 -0500 Subject: [PATCH 23/25] Use `balancer_by_lua_block` in token_server.conf * Remove `YT_POT_BGUTIL_BASE_URL` * Prioritize the linked container details --- config/root/etc/nginx/nginx.conf | 8 +- config/root/etc/nginx/token_server.conf | 77 ++++++++++++++++---- config/root/etc/s6-overlay/s6-rc.d/nginx/run | 11 --- 3 files changed, 71 insertions(+), 25 deletions(-) diff --git a/config/root/etc/nginx/nginx.conf b/config/root/etc/nginx/nginx.conf index 4ca5bac4..43fed0bc 100644 --- a/config/root/etc/nginx/nginx.conf +++ b/config/root/etc/nginx/nginx.conf @@ -9,7 +9,13 @@ worker_processes auto; worker_cpu_affinity auto; pid /run/nginx.pid; -env YT_POT_BGUTIL_BASE_URL; +# link configuration +env POTSERVER_PORT_4416_TCP_ADDR; +env POTSERVER_PORT_4416_TCP_PORT; +# user configuration +env TUBESYNC_POT_HTTPS; +env TUBESYNC_POT_IPADDR; +env TUBESYNC_POT_PORT; events { worker_connections 1024; diff --git a/config/root/etc/nginx/token_server.conf b/config/root/etc/nginx/token_server.conf index 1c985ab8..25b1cd26 100644 --- a/config/root/etc/nginx/token_server.conf +++ b/config/root/etc/nginx/token_server.conf @@ -1,5 +1,68 @@ +lua_shared_dict token_server 1m; upstream token_server { server 127.0.0.1:4406 max_fails=0; + + balancer_by_lua_block { + local balancer = require "ngx.balancer" + local upstream = ngx.shared.token_server + local exit_error = ngx.HTTP_INTERNAL_SERVER_ERROR + + local ok, err, empty_port, host, port + local default_host, default_port, tries = '127.0.0.1', '4406', 1 + + local user_host = os.getenv('TUBESYNC_POT_IPADDR') + local link_host = os.getenv('POTSERVER_PORT_4416_TCP_ADDR') + local use_https = os.getenv('TUBESYNC_POT_HTTPS') + if user_host and use_https then + use_https = true + empty_port = '443' + else + use_https = false + empty_port = '80' + end + + ok, err = balancer.get_last_failure() + if ok then + host, port, tries = default_host, default_port, 0 + elseif link_host then + host = link_host + local link_port = os.getenv('POTSERVER_PORT_4416_TCP_PORT') + if link_port then + port = link_port + else + port = empty_port + end + elseif user_host then + host = user_host + local user_port = os.getenv('TUBESYNC_POT_PORT') + if user_port then + port = user_port + else + port = empty_port + end + ok, err = balancer.set_upstream_tls(use_https) + if not ok then + ngx.log(ngx.ERR, "failed to set upstream TLS: ", err) + end + end + if not host then + host, port, tries = default_host, default_port, 0 + end + + ok, err = balancer.set_more_tries(tries) + if not ok then + ngx.log(ngx.err, ("failed to set tries to %d: "):format(tries), err) + end + + ok, err = balancer.set_current_peer(host, port) + if ok then + upstream:set('addr', host) + upstream:set('port', port) + else + ngx.log(ngx.err, "failed to set the current peer: ", err) + return ngx.exit(exit_error) + end + } } server { @@ -11,19 +74,7 @@ server { # Server domain name server_name _; - set_by_lua_block $pot_url { - local default = 'http://token_server' - local url = os.getenv('YT_POT_BGUTIL_BASE_URL') - if not url then - return default - end - if #url and url:find('://') then - return url - end - return default - } - location / { - proxy_pass $pot_url; + proxy_pass http://token_server; } } diff --git a/config/root/etc/s6-overlay/s6-rc.d/nginx/run b/config/root/etc/s6-overlay/s6-rc.d/nginx/run index fff967d1..63653343 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/nginx/run +++ b/config/root/etc/s6-overlay/s6-rc.d/nginx/run @@ -2,15 +2,4 @@ cd / -https="${TUBESYNC_POT_HTTPS:+https}" -ip_address="${TUBESYNC_POT_IPADDR:-${POTSERVER_PORT_4416_TCP_ADDR}}" -: "${TUBESYNC_POT_PORT:=${POTSERVER_PORT_4416_TCP_PORT}}" -port="${TUBESYNC_POT_PORT:+:}${TUBESYNC_POT_PORT}" - -if [ -n "${ip_address}" ] -then - YT_POT_BGUTIL_BASE_URL="${https:-http}://${ip_address}${port}" - export YT_POT_BGUTIL_BASE_URL -fi - exec /usr/bin/openresty -c /etc/nginx/nginx.conf -e stderr From fc8b6ca7676b51dd297d3e0ac79163c8cddfdedd Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 31 Dec 2025 06:47:13 -0500 Subject: [PATCH 24/25] Release for 2026-001 --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index a9f5f120..b4f07579 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -10,7 +10,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.13' +VERSION = '0.16.1' SECRET_KEY = '' DEBUG = 'true' == getenv('TUBESYNC_DEBUG').strip().lower() ALLOWED_HOSTS = [] From 14b9e3589af48a3a4307c8f19520b58e7e3f6679 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 31 Dec 2025 07:15:22 -0500 Subject: [PATCH 25/25] Mention that we bundle `openresty` I am no longer trying to ensure `nginx` works in the container. It should still work without the `token_server.conf` included. --- Dockerfile | 19 ------------------- README.md | 3 ++- docs/youtube-pot.md | 3 +-- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4beadd17..1ea41c04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -445,25 +445,6 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va apt-get -y autoclean && \ rm -v -f /var/cache/debconf/*.dat-old -FROM tubesync-base AS tubesync-nginx - -RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/var/lib/apt \ - --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \ - set -x && \ - apt-get update && \ - apt-get -y --no-install-recommends install \ - nginx-light \ - libnginx-mod-http-lua \ - && \ - # openresty binary should still work - ln -v -s -T ../sbin/nginx /usr/bin/openresty && \ - # Clean up - apt-get -y autopurge && \ - apt-get -y autoclean && \ - rm -v -f /var/cache/debconf/*.dat-old - -# The preference for openresty over nginx, -# is for the newer version. FROM tubesync-openresty AS tubesync ARG S6_VERSION diff --git a/README.md b/README.md index eb2f1402..dfc084a9 100644 --- a/README.md +++ b/README.md @@ -333,10 +333,11 @@ Notable libraries and software used: * [Django](https://www.djangoproject.com/) * [yt-dlp](https://github.com/yt-dlp/yt-dlp) * [ffmpeg](https://ffmpeg.org/) + * [QuickJS](https://bellard.org/quickjs/) * [Django Huey](https://github.com/gaiacoop/django-huey) * [Huey](https://github.com/coleifer/huey) * [django-sass](https://github.com/coderedcorp/django-sass/) - * The container bundles with `s6-init` and `nginx` + * The container bundles with `s6-init` and `openresty` See the [Pipfile](https://github.com/meeb/tubesync/blob/main/Pipfile) for a full list. diff --git a/docs/youtube-pot.md b/docs/youtube-pot.md index cfcbd267..ef47f6b3 100644 --- a/docs/youtube-pot.md +++ b/docs/youtube-pot.md @@ -15,7 +15,7 @@ Support for using the plugin was added in: https://github.com/meeb/tubesync/pull This plugin communicates with a web service to retrieve tokens, for which it uses a default URL of: `http://127.0.0.1:4416` -Because for TubeSync `openresty` or `nginx` are what most users will connect to with their web browser, the [configuration](../config/root/etc/nginx/token_server.conf) to listen on port `4416` has been added. +Because for TubeSync `openresty` is what most users will connect to with their web browser, the [configuration](../config/root/etc/nginx/token_server.conf) to listen on port `4416` has been added. If you are using another web server instead, you should configure similar proxying of the requests, or configure a different URL for the plugin to use. @@ -108,5 +108,4 @@ Added environment variables: - TUBESYNC_POT_IPADDR - TUBESYNC_POT_PORT - TUBESYNC_POT_HTTPS (use https:// when set) -- YT_POT_BGUTIL_BASE_URL (`youtubepot-bgutilhttp:base_url`) (not set by the user)