diff --git a/Dockerfile b/Dockerfile index 6e6212f6..1ea41c04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,14 @@ # syntax=docker/dockerfile:1 # check=error=true +ARG BGUTIL_YTDLP_POT_PROVIDER_VERSION="1.2.2" ARG FFMPEG_VERSION="N" +ARG ASFALD_VERSION="0.6.0" + +ARG SHA256_ASFALD_AMD64="017cdc44d767bb4733a3bd3fa5f97719e3f58236d006321dfae1924fdda3de9d" +ARG SHA256_ASFALD_ARM64="4fc112617a71f97592b8760b98c16339d5243577186c970c784fbcab2ef8abd1" + ARG S6_VERSION="3.2.0.3" ARG SHA256_S6_AMD64="01eb9a6dce10b5428655974f1903f48e7ba7074506dfb262e85ffab64a5498f2" @@ -19,6 +25,7 @@ ARG DEBIAN_VERSION="bookworm-slim" ARG FFMPEG_PREFIX_FILE="ffmpeg-${FFMPEG_VERSION}" ARG FFMPEG_SUFFIX_FILE=".tar.xz" +ARG ASFALD_CHECKSUM_ALGORITHM="sha256" ARG FFMPEG_CHECKSUM_ALGORITHM="sha256" ARG S6_CHECKSUM_ALGORITHM="sha256" ARG QJS_CHECKSUM_ALGORITHM="sha256" @@ -64,11 +71,100 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va apt-get -y autoclean && \ rm -f /var/cache/debconf/*.dat-old +FROM alpine:${ALPINE_VERSION} AS asfald-download +ARG ASFALD_VERSION +ARG SHA256_ASFALD_AMD64 +ARG SHA256_ASFALD_ARM64 + +ARG DESTDIR="/downloaded" +ARG ASFALD_CHECKSUM_ALGORITHM +ARG CHECKSUM_ALGORITHM="${ASFALD_CHECKSUM_ALGORITHM}" + +ARG ASFALD_CHECKSUM_AMD64="${CHECKSUM_ALGORITHM}:${SHA256_ASFALD_AMD64}" +ARG ASFALD_CHECKSUM_ARM64="${CHECKSUM_ALGORITHM}:${SHA256_ASFALD_ARM64}" + +ARG ASFALD_DOWNLOAD_URI="asfaload/asfald/releases/download/v${ASFALD_VERSION}" +ARG ASFALD_URL="https://github.com/${ASFALD_DOWNLOAD_URI}" +ARG ASFALD_SUMS_URL="https://gh.checksums.asfaload.com/github.com/${ASFALD_DOWNLOAD_URI}/checksums.txt" +ARG ASFALD_PREFIX_FILE="asfald-" +ARG ASFALD_SUFFIX_FILE="-unknown-linux-musl" + +ARG ASFALD_FILE_AMD64="${ASFALD_PREFIX_FILE}x86_64${ASFALD_SUFFIX_FILE}" +ARG ASFALD_FILE_ARM64="${ASFALD_PREFIX_FILE}aarch64${ASFALD_SUFFIX_FILE}" + +ADD "${ASFALD_SUMS_URL}" "${DESTDIR}/" +ADD "${ASFALD_URL}/${ASFALD_FILE_AMD64}" "${DESTDIR}/" +ADD "${ASFALD_URL}/${ASFALD_FILE_ARM64}" "${DESTDIR}/" + +##ADD --checksum="${ASFALD_CHECKSUM_AMD64}" "${ASFALD_URL}/${ASFALD_FILE_AMD64}" "${DESTDIR}/" +##ADD --checksum="${ASFALD_CHECKSUM_ARM64}" "${ASFALD_URL}/${ASFALD_FILE_ARM64}" "${DESTDIR}/" + +# --checksum wasn't recognized, so use busybox to check the sums instead +ARG TARGETARCH +RUN set -eu ; \ + apk --no-cache --no-progress add "cmd:${CHECKSUM_ALGORITHM}sum" ; \ +\ + decide_expected() { \ + case "${TARGETARCH}" in \ + (amd64) printf -- '%s' "${ASFALD_CHECKSUM_AMD64}" ;; \ + (arm64) printf -- '%s' "${ASFALD_CHECKSUM_ARM64}" ;; \ + (*) exit 1 ;; \ + esac ; \ + } ; \ +\ + decide_fn() { \ + case "${TARGETARCH}" in \ + (amd64) printf -- '%s\n' "${ASFALD_FILE_AMD64}" ;; \ + (arm64) printf -- '%s\n' "${ASFALD_FILE_ARM64}" ;; \ + (*) exit 1 ;; \ + esac ; \ + } ; \ +\ + checksum="$(decide_expected)" ; \ + file="$(decide_fn)" ; \ + mkdir -v -p "/verified/${TARGETARCH}" ; \ + cd "${DESTDIR}/" && \ + "${CHECKSUM_ALGORITHM}sum" --check --warn --strict --ignore-missing checksums.txt && \ + printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw && \ + mv -v "${file}" "/verified/${TARGETARCH}/asfald" && \ + chmod -v 00755 "/verified/${TARGETARCH}/asfald" && \ + chown -v root:root "/verified/${TARGETARCH}/asfald" + +FROM scratch AS asfald +ARG TARGETARCH +COPY --from=asfald-download "/verified/${TARGETARCH}/asfald" /usr/local/sbin/ + +FROM tubesync-base AS tubesync-asfald +COPY --from=asfald /usr/local/sbin/ /usr/local/sbin/ + +ARG TARGETARCH +RUN set -eu ; \ +\ + decide_arch() { \ + case "${TARGETARCH}" in \ + (amd64) printf -- 'x86_64' ;; \ + (arm64) printf -- 'aarch64' ;; \ + (*) exit 1 ;; \ + esac ; \ + } ; \ +\ + set -x ; arch="$(decide_arch)" ; \ + dest='/usr/local/sbin/asfald-latest' ; \ + TMPDIR="$(dirname "${dest}")" \ + asfald --overwrite --output "${dest}" \ + --pattern '${path}/checksums.txt' -- \ + "https://github.com/asfaload/asfald/releases/latest/download/asfald-${arch}-unknown-linux-musl" && \ + chmod -c 00755 "${dest}" && chown -c root:root "${dest}" + FROM ghcr.io/astral-sh/uv:latest AS uv-binaries + +FROM scratch AS uv +COPY --from=uv-binaries /uv /uvx /usr/local/bin/ + FROM denoland/deno:bin AS deno-binaries FROM scratch AS deno -COPY --from=deno-binaries /deno /usr/local/bin/deno +COPY --from=deno-binaries /deno /usr/local/bin/ FROM alpine:${ALPINE_VERSION} AS openresty-debian ARG TARGETARCH @@ -92,7 +188,7 @@ RUN set -eu ; \ 'deb http://openresty.org/package/%sdebian %s openresty' \ "$(decide_arch)" "${DEBIAN_VERSION%-slim}" -FROM alpine:${ALPINE_VERSION} AS ffmpeg-download +FROM tubesync-asfald AS ffmpeg-download ARG FFMPEG_DATE ARG FFMPEG_VERSION ARG FFMPEG_PREFIX_FILE @@ -111,22 +207,6 @@ ARG DESTDIR="/downloaded" ARG TARGETARCH ADD "${FFMPEG_URL}/${FFMPEG_FILE_SUMS}" "${DESTDIR}/" RUN set -eu ; \ - apk --no-cache --no-progress add cmd:aria2c cmd:awk "cmd:${CHECKSUM_ALGORITHM}sum" ; \ -\ - aria2c_options() { \ - algorithm="${CHECKSUM_ALGORITHM%[0-9]??}" ; \ - bytes="${CHECKSUM_ALGORITHM#${algorithm}}" ; \ - hash="$( awk -v fn="${1##*/}" '$0 ~ fn"$" { print $1; exit; }' "${DESTDIR}/${FFMPEG_FILE_SUMS}" )" ; \ -\ - printf -- '\t%s\n' \ - 'allow-overwrite=true' \ - 'always-resume=false' \ - 'check-integrity=true' \ - "checksum=${algorithm}-${bytes}=${hash}" \ - 'max-connection-per-server=2' \ -; \ - printf -- '\n' ; \ - } ; \ \ decide_arch() { \ case "${TARGETARCH}" in \ @@ -137,23 +217,15 @@ RUN set -eu ; \ \ FFMPEG_ARCH="$(decide_arch)" ; \ FFMPEG_PREFIX_FILE="$( printf -- '%s' "${FFMPEG_PREFIX_FILE}" | cut -d '-' -f 1,2 )" ; \ + cd "${DESTDIR}" && \ for url in $(awk ' \ $2 ~ /^[*]?'"${FFMPEG_PREFIX_FILE}"'/ && /-'"${FFMPEG_ARCH}"'-/ { $1=""; print; } \ ' "${DESTDIR}/${FFMPEG_FILE_SUMS}") ; \ do \ url="${FFMPEG_URL}/${url# }" ; \ - printf -- '%s\n' "${url}" ; \ - aria2c_options "${url}" ; \ - printf -- '\n' ; \ - done > /tmp/downloads ; \ + TMPDIR="${DESTDIR}" asfald-latest -qv -- "${url}" ; \ + done ; \ unset -v url ; \ -\ - aria2c --no-conf=true \ - --dir /downloaded \ - --lowest-speed-limit='16K' \ - --show-console-readout=false \ - --summary-interval=0 \ - --input-file /tmp/downloads ; \ \ decide_expected() { \ case "${TARGETARCH}" in \ @@ -164,7 +236,6 @@ RUN set -eu ; \ \ FFMPEG_HASH="$(decide_expected)" ; \ \ - cd "${DESTDIR}" ; \ if [ -n "${FFMPEG_HASH}" ] ; \ then \ printf -- '%s *%s\n' "${FFMPEG_HASH}" "${FFMPEG_PREFIX_FILE}"*-"${FFMPEG_ARCH}"-*"${FFMPEG_SUFFIX_FILE}" >> /tmp/SUMS ; \ @@ -197,7 +268,7 @@ RUN set -eux ; \ FROM scratch AS ffmpeg COPY --from=ffmpeg-extracted /extracted /usr/local/bin/ -FROM alpine:${ALPINE_VERSION} AS s6-overlay-download +FROM tubesync-asfald AS s6-overlay-download ARG S6_VERSION ARG SHA256_S6_AMD64 ARG SHA256_S6_ARM64 @@ -227,18 +298,11 @@ ADD "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}.${CHECKSUM_ALGORITHM}" "${DESTDIR}/" ##ADD --checksum="${S6_CHECKSUM_ARM64}" "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" "${DESTDIR}/" ##ADD --checksum="${S6_CHECKSUM_NOARCH}" "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" "${DESTDIR}/" -# --checksum wasn't recognized, so use busybox to check the sums instead -ADD "${S6_OVERLAY_URL}/${S6_FILE_AMD64}" "${DESTDIR}/" -RUN set -eu ; checksum="${S6_CHECKSUM_AMD64}" ; file="${S6_FILE_AMD64}" ; cd "${DESTDIR}/" && \ - printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw - -ADD "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" "${DESTDIR}/" -RUN set -eu ; checksum="${S6_CHECKSUM_ARM64}" ; file="${S6_FILE_ARM64}" ; cd "${DESTDIR}/" && \ - printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw - -ADD "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" "${DESTDIR}/" -RUN set -eu ; checksum="${S6_CHECKSUM_NOARCH}" ; file="${S6_FILE_NOARCH}" ; cd "${DESTDIR}/" && \ - printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw +# --checksum wasn't recognized, so use asfald to check the sums instead +RUN set -eux ; TMPDIR="${DESTDIR}" ; export TMPDIR ; cd "${DESTDIR}/" && \ + asfald --hash="${SHA256_S6_AMD64}" -- "${S6_OVERLAY_URL}/${S6_FILE_AMD64}" && \ + asfald --hash="${SHA256_S6_ARM64}" -- "${S6_OVERLAY_URL}/${S6_FILE_ARM64}" && \ + asfald --hash="${SHA256_S6_NOARCH}" -- "${S6_OVERLAY_URL}/${S6_FILE_NOARCH}" FROM alpine:${ALPINE_VERSION} AS s6-overlay-extracted COPY --from=s6-overlay-download /downloaded /downloaded @@ -292,7 +356,7 @@ RUN set -eu ; \ ## COPY --from=s6-overlay-extracted /s6-overlay-rootfs / FROM ghcr.io/meeb/s6-overlay:v${S6_VERSION} AS s6-overlay -FROM alpine:${ALPINE_VERSION} AS quickjs-download +FROM tubesync-asfald AS quickjs-extracted ARG QJS_VERSION ARG SHA256_QJS @@ -302,7 +366,7 @@ ARG CHECKSUM_ALGORITHM="${QJS_CHECKSUM_ALGORITHM}" ARG QJS_CHECKSUM="${CHECKSUM_ALGORITHM}:${SHA256_QJS}" -ARG QJS_URL="https://bellard.org/quickjs/binary_releases/" +ARG QJS_URL="https://bellard.org/quickjs/binary_releases" ARG QJS_PREFIX_FILE="quickjs-cosmo-" ARG QJS_SUFFIX_FILE=".zip" @@ -310,30 +374,29 @@ ARG QJS_FILE="${QJS_PREFIX_FILE}${QJS_VERSION}${QJS_SUFFIX_FILE}" ##ADD --checksum="${QJS_CHECKSUM}" "${QJS_URL}/${QJS_FILE}" "${DESTDIR}/" -# --checksum wasn't recognized, so use busybox to check the sums instead -ADD "${QJS_URL}/${QJS_FILE}" "${DESTDIR}/" -RUN set -eu ; checksum="${QJS_CHECKSUM}" ; file="${QJS_FILE}" ; cd "${DESTDIR}/" && \ - printf -- '%s *%s\n' "$(printf -- '%s' "${checksum}" | cut -d : -f 2-)" "${file}" | "${CHECKSUM_ALGORITHM}sum" -cw +# --checksum wasn't recognized, so use asfald to check the sums instead +RUN set -eux ; TMPDIR="${DESTDIR}" ; export TMPDIR ; \ + mkdir -v -p "${DESTDIR}" && cd "${DESTDIR}/" && \ + asfald --hash="${SHA256_QJS}" -- "${QJS_URL}/${QJS_FILE}" -FROM alpine:${ALPINE_VERSION} AS quickjs-extracted -COPY --from=quickjs-download /downloaded /downloaded - -ARG QJS_CHECKSUM_ALGORITHM -ARG CHECKSUM_ALGORITHM="${QJS_CHECKSUM_ALGORITHM}" - -RUN set -eu ; \ -\ - apk --no-cache --no-progress add "cmd:${CHECKSUM_ALGORITHM}sum" cmd:unzip ; \ +RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/var/lib/apt \ + --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \ + set -eux ; \ + apt-get update ; \ + apt-get -y --no-install-recommends install unzip ; \ mkdir -v /extracted ; \ cd /extracted ; \ for f in /downloaded/*.zip ; \ do \ unzip "${f}" || exit ; \ done ; \ - unset -v f ; + unset -v f ; \ + mkdir -v /assimilated ; \ + install -v -p -t /assimilated /extracted/qjs ; \ + /assimilated/qjs --assimilate FROM scratch AS quickjs -COPY --from=quickjs-extracted /extracted/qjs /usr/local/sbin/ +COPY --from=quickjs-extracted /assimilated/qjs /usr/local/sbin/ FROM tubesync-base AS tubesync-prepare-app @@ -351,12 +414,17 @@ RUN --mount=type=bind,source=fontawesome-free,target=/fontawesome-free \ rm -v /app/tubesync/local_settings.py.example && \ mv -v /app/tubesync/local_settings.py.container /app/tubesync/local_settings.py +ARG BGUTIL_YTDLP_POT_PROVIDER_VERSION +ADD "https://github.com/Brainicism/bgutil-ytdlp-pot-provider/archive/refs/tags/${BGUTIL_YTDLP_POT_PROVIDER_VERSION}.tar.gz" /tmp/ +RUN mkdir -v /tmp/extracted && \ + tar -C /tmp/extracted/ -xvvpf "/tmp/${BGUTIL_YTDLP_POT_PROVIDER_VERSION}.tar.gz" && \ + mv -v /tmp/extracted/*/server /app/bgutil-ytdlp-pot-provider/ && \ + ls -alR /app/bgutil-ytdlp-pot-provider && \ + rm -rf /tmp/extracted + FROM scratch AS tubesync-app COPY --from=tubesync-prepare-app /app /app -FROM tubesync-base AS tubesync-uv -COPY --from=uv-binaries /uv /uvx /usr/local/bin/ - FROM tubesync-base AS tubesync-openresty COPY --from=openresty-debian \ @@ -377,25 +445,6 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va apt-get -y autoclean && \ rm -v -f /var/cache/debconf/*.dat-old -FROM tubesync-base AS tubesync-nginx - -RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/var/lib/apt \ - --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \ - set -x && \ - apt-get update && \ - apt-get -y --no-install-recommends install \ - nginx-light \ - libnginx-mod-http-lua \ - && \ - # openresty binary should still work - ln -v -s -T ../sbin/nginx /usr/bin/openresty && \ - # Clean up - apt-get -y autopurge && \ - apt-get -y autoclean && \ - rm -v -f /var/cache/debconf/*.dat-old - -# The preference for openresty over nginx, -# is for the newer version. FROM tubesync-openresty AS tubesync ARG S6_VERSION @@ -469,7 +518,6 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va /usr/local/bin/ffmpeg -version && \ file /usr/local/bin/ff* && \ # Installed quickjs (using COPY earlier) - /usr/local/sbin/qjs --assimilate && \ /usr/local/sbin/qjs --help | grep -Fe 'QuickJS version' && \ file /usr/local/sbin/qjs && \ # Clean up file @@ -486,13 +534,19 @@ ARG YTDLP_DATE # Set up the app RUN --mount=type=tmpfs,target=/cache \ + --mount=type=cache,id=deno-cache,sharing=locked,target=/cache/deno \ --mount=type=cache,id=uv-cache,sharing=locked,target=/cache/uv \ --mount=type=cache,id=pipenv-cache,sharing=locked,target=/cache/pipenv \ --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/var/lib/apt \ --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \ - --mount=type=bind,source=/uv,target=/usr/local/bin/uv,from=uv-binaries \ + --mount=type=bind,source=/usr/local/bin/deno,target=/usr/local/bin/deno,from=deno \ + --mount=type=bind,source=/usr/local/bin/uv,target=/usr/local/bin/uv,from=uv \ + --mount=type=bind,source=/usr/local/bin/uvx,target=/usr/local/bin/uvx,from=uv \ --mount=type=bind,source=Pipfile,target=/app/Pipfile \ set -x && \ + DENO_DIR=/cache/deno && export DENO_DIR && \ + deno --version && \ + uv --version && \ apt-get update && \ # Install required build packages apt-get -y --no-install-recommends install \ @@ -570,6 +624,9 @@ RUN --mount=type=tmpfs,target=/cache \ exit 1 ; \ fi +# Bundle deno with the image +##COPY --from=deno /usr/local/bin/ /usr/local/bin/ + # Copy root COPY config/root / @@ -580,8 +637,31 @@ COPY patches/yt_dlp/ \ # Copy app COPY --from=tubesync-app /app /app +# Build the bgutil-ytdlp-pot-provider server when deno is bundled +RUN --mount=type=tmpfs,target=/cache \ + --mount=type=cache,id=deno-cache,sharing=locked,target=/cache/deno \ + command -v deno || exit 0 ; \ + # python might be used, so keep the .pyc files in /cache + PYTHONPYCACHEPREFIX='/cache/pycache' && export PYTHONPYCACHEPREFIX && \ + set -x && \ + XDG_CACHE_HOME='/cache' && export XDG_CACHE_HOME && \ + DENO_DIR='/cache/deno' && export DENO_DIR && \ + cd /app/bgutil-ytdlp-pot-provider/server && \ + install -v -t /usr/local/bin ../node && \ + mkdir -v -p /cache/.home-directories && \ + cp -at /cache/.home-directories/ "${HOME}" && \ + HOME="/cache/.home-directories/${HOME#/}" && \ + DENO_COMPAT=1 deno run -A npm:npm ci --no-audit --no-fund && \ + DENO_COMPAT=1 deno run -A npm:npm audit fix && \ + node npm:typescript/tsc + # Build app RUN set -x && \ + # Record the bundled deno version when it was included + ( deno_binary="$(command -v deno)" ; \ + test '!' -n "${deno_binary}" || \ + /app/install_deno.sh --only-record-version ; \ + ) && \ # Make absolutely sure we didn't accidentally bundle a SQLite dev database test '!' -e /app/db.sqlite3 && \ # Run any required app commands diff --git a/README.md b/README.md index eb2f1402..dfc084a9 100644 --- a/README.md +++ b/README.md @@ -333,10 +333,11 @@ Notable libraries and software used: * [Django](https://www.djangoproject.com/) * [yt-dlp](https://github.com/yt-dlp/yt-dlp) * [ffmpeg](https://ffmpeg.org/) + * [QuickJS](https://bellard.org/quickjs/) * [Django Huey](https://github.com/gaiacoop/django-huey) * [Huey](https://github.com/coleifer/huey) * [django-sass](https://github.com/coderedcorp/django-sass/) - * The container bundles with `s6-init` and `nginx` + * The container bundles with `s6-init` and `openresty` See the [Pipfile](https://github.com/meeb/tubesync/blob/main/Pipfile) for a full list. diff --git a/config/root/etc/nginx/nginx.conf b/config/root/etc/nginx/nginx.conf index 1e8e0532..85ecbfef 100644 --- a/config/root/etc/nginx/nginx.conf +++ b/config/root/etc/nginx/nginx.conf @@ -9,14 +9,21 @@ worker_processes auto; worker_cpu_affinity auto; pid /run/nginx.pid; +# link configuration env CIPHERSERVER_PORT_8001_TCP_ADDR; env CIPHERSERVER_PORT_8001_TCP_PORT; - +# user configuration env TUBESYNC_CIPHER_HTTPS; env TUBESYNC_CIPHER_IPADDR; env TUBESYNC_CIPHER_PORT; -env YT_POT_BGUTIL_BASE_URL; +# link configuration +env POTSERVER_PORT_4416_TCP_ADDR; +env POTSERVER_PORT_4416_TCP_PORT; +# user configuration +env TUBESYNC_POT_HTTPS; +env TUBESYNC_POT_IPADDR; +env TUBESYNC_POT_PORT; events { worker_connections 1024; diff --git a/config/root/etc/nginx/token_server.conf b/config/root/etc/nginx/token_server.conf index 6c0c4e3a..25b1cd26 100644 --- a/config/root/etc/nginx/token_server.conf +++ b/config/root/etc/nginx/token_server.conf @@ -1,5 +1,68 @@ +lua_shared_dict token_server 1m; upstream token_server { - server 127.0.0.2:4416 down; + server 127.0.0.1:4406 max_fails=0; + + balancer_by_lua_block { + local balancer = require "ngx.balancer" + local upstream = ngx.shared.token_server + local exit_error = ngx.HTTP_INTERNAL_SERVER_ERROR + + local ok, err, empty_port, host, port + local default_host, default_port, tries = '127.0.0.1', '4406', 1 + + local user_host = os.getenv('TUBESYNC_POT_IPADDR') + local link_host = os.getenv('POTSERVER_PORT_4416_TCP_ADDR') + local use_https = os.getenv('TUBESYNC_POT_HTTPS') + if user_host and use_https then + use_https = true + empty_port = '443' + else + use_https = false + empty_port = '80' + end + + ok, err = balancer.get_last_failure() + if ok then + host, port, tries = default_host, default_port, 0 + elseif link_host then + host = link_host + local link_port = os.getenv('POTSERVER_PORT_4416_TCP_PORT') + if link_port then + port = link_port + else + port = empty_port + end + elseif user_host then + host = user_host + local user_port = os.getenv('TUBESYNC_POT_PORT') + if user_port then + port = user_port + else + port = empty_port + end + ok, err = balancer.set_upstream_tls(use_https) + if not ok then + ngx.log(ngx.ERR, "failed to set upstream TLS: ", err) + end + end + if not host then + host, port, tries = default_host, default_port, 0 + end + + ok, err = balancer.set_more_tries(tries) + if not ok then + ngx.log(ngx.err, ("failed to set tries to %d: "):format(tries), err) + end + + ok, err = balancer.set_current_peer(host, port) + if ok then + upstream:set('addr', host) + upstream:set('port', port) + else + ngx.log(ngx.err, "failed to set the current peer: ", err) + return ngx.exit(exit_error) + end + } } server { @@ -11,19 +74,7 @@ server { # Server domain name server_name _; - set_by_lua_block $pot_url { - local default = 'http://token_server' - local url = os.getenv('YT_POT_BGUTIL_BASE_URL') - if not url then - return default - end - if #url and url:find('://') then - return url - end - return default - } - location / { - proxy_pass $pot_url; + proxy_pass http://token_server; } } diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/dependencies b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/dependencies new file mode 100644 index 00000000..b1248844 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/dependencies @@ -0,0 +1,2 @@ +base +tubesync-init diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run new file mode 100755 index 00000000..c9efae37 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run @@ -0,0 +1,32 @@ +#!/command/with-contenv bash + +set -eu +cd /app/bgutil-ytdlp-pot-provider/server +chmod -c 00755 ../node + +tmp_app_home="$(realpath -e "$(mktemp --directory --tmpdir=.)")" +trap "rm -rf ${tmp_app_home}" EXIT + +command -v deno >/dev/null || \ + /app/install_deno.sh + +test '!' -d build || \ + test '!' -f build/main.js || \ + test '!' -d node_modules || \ + exit 0 # nothing needs to be built + +command -v node > /dev/null || \ + install -v -t /usr/local/bin ../node + +chown -R app:app . + +s6-setuidgid app \ + env HOME="${tmp_app_home}" DENO_COMPAT=1 deno run -A npm:npm ci --no-fund + +s6-setuidgid app \ + env HOME="${tmp_app_home}" DENO_COMPAT=1 deno run -A npm:npm audit fix + +s6-setuidgid app \ + env HOME="${tmp_app_home}" ../node npm:typescript/tsc + +chown -R root:app . diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/type b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/type new file mode 100644 index 00000000..bdd22a18 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/type @@ -0,0 +1 @@ +oneshot diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/up b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/up new file mode 100755 index 00000000..67732022 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/up @@ -0,0 +1,3 @@ +#!/command/execlineb -P + +/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider-build/run diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/dependencies b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/dependencies new file mode 100644 index 00000000..c15ef0c0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/dependencies @@ -0,0 +1,2 @@ +base +bgutil-ytdlp-pot-provider-build diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/run b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/run new file mode 100755 index 00000000..48c52dcf --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/run @@ -0,0 +1,7 @@ +#!/command/with-contenv bash + +set -e +cd /app/bgutil-ytdlp-pot-provider + +exec s6-setuidgid app \ + ./node server/build/main.js -p 4406 diff --git a/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/type b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/type new file mode 100644 index 00000000..5883cff0 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/bgutil-ytdlp-pot-provider/type @@ -0,0 +1 @@ +longrun diff --git a/config/root/etc/s6-overlay/s6-rc.d/gunicorn/dependencies b/config/root/etc/s6-overlay/s6-rc.d/gunicorn/dependencies index fe757c20..b1248844 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/gunicorn/dependencies +++ b/config/root/etc/s6-overlay/s6-rc.d/gunicorn/dependencies @@ -1 +1,2 @@ -tubesync-init \ No newline at end of file +base +tubesync-init diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database index 8b137891..e69de29b 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-database @@ -1 +0,0 @@ - diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem index 8b137891..e69de29b 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-filesystem @@ -1 +0,0 @@ - diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited index 8b137891..e69de29b 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-net-limited @@ -1 +0,0 @@ - diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network index 8b137891..e69de29b 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network +++ b/config/root/etc/s6-overlay/s6-rc.d/huey-consumers/contents.d/huey-network @@ -1 +0,0 @@ - diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-database/run b/config/root/etc/s6-overlay/s6-rc.d/huey-database/run old mode 100644 new mode 100755 diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run b/config/root/etc/s6-overlay/s6-rc.d/huey-filesystem/run old mode 100644 new mode 100755 diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run b/config/root/etc/s6-overlay/s6-rc.d/huey-net-limited/run old mode 100644 new mode 100755 diff --git a/config/root/etc/s6-overlay/s6-rc.d/huey-network/run b/config/root/etc/s6-overlay/s6-rc.d/huey-network/run old mode 100644 new mode 100755 diff --git a/config/root/etc/s6-overlay/s6-rc.d/nginx/dependencies b/config/root/etc/s6-overlay/s6-rc.d/nginx/dependencies index 283e1305..c4f03c23 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/nginx/dependencies +++ b/config/root/etc/s6-overlay/s6-rc.d/nginx/dependencies @@ -1 +1,2 @@ -gunicorn \ No newline at end of file +base +gunicorn diff --git a/config/root/etc/s6-overlay/s6-rc.d/nginx/run b/config/root/etc/s6-overlay/s6-rc.d/nginx/run index fff967d1..63653343 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/nginx/run +++ b/config/root/etc/s6-overlay/s6-rc.d/nginx/run @@ -2,15 +2,4 @@ cd / -https="${TUBESYNC_POT_HTTPS:+https}" -ip_address="${TUBESYNC_POT_IPADDR:-${POTSERVER_PORT_4416_TCP_ADDR}}" -: "${TUBESYNC_POT_PORT:=${POTSERVER_PORT_4416_TCP_PORT}}" -port="${TUBESYNC_POT_PORT:+:}${TUBESYNC_POT_PORT}" - -if [ -n "${ip_address}" ] -then - YT_POT_BGUTIL_BASE_URL="${https:-http}://${ip_address}${port}" - export YT_POT_BGUTIL_BASE_URL -fi - exec /usr/bin/openresty -c /etc/nginx/nginx.conf -e stderr diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/dependancies b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/dependancies deleted file mode 100644 index e69de29b..00000000 diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/dependencies b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/dependencies new file mode 100644 index 00000000..df967b96 --- /dev/null +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/dependencies @@ -0,0 +1 @@ +base diff --git a/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers b/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers index 8b137891..e69de29b 100644 --- a/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers +++ b/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/huey-consumers @@ -1 +0,0 @@ - diff --git a/docs/youtube-pot.md b/docs/youtube-pot.md index cfcbd267..ef47f6b3 100644 --- a/docs/youtube-pot.md +++ b/docs/youtube-pot.md @@ -15,7 +15,7 @@ Support for using the plugin was added in: https://github.com/meeb/tubesync/pull This plugin communicates with a web service to retrieve tokens, for which it uses a default URL of: `http://127.0.0.1:4416` -Because for TubeSync `openresty` or `nginx` are what most users will connect to with their web browser, the [configuration](../config/root/etc/nginx/token_server.conf) to listen on port `4416` has been added. +Because for TubeSync `openresty` is what most users will connect to with their web browser, the [configuration](../config/root/etc/nginx/token_server.conf) to listen on port `4416` has been added. If you are using another web server instead, you should configure similar proxying of the requests, or configure a different URL for the plugin to use. @@ -108,5 +108,4 @@ Added environment variables: - TUBESYNC_POT_IPADDR - TUBESYNC_POT_PORT - TUBESYNC_POT_HTTPS (use https:// when set) -- YT_POT_BGUTIL_BASE_URL (`youtubepot-bgutilhttp:base_url`) (not set by the user) diff --git a/tubesync/bgutil-ytdlp-pot-provider/node b/tubesync/bgutil-ytdlp-pot-provider/node new file mode 100755 index 00000000..2f28f7a5 --- /dev/null +++ b/tubesync/bgutil-ytdlp-pot-provider/node @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +set -eu +command -v deno >/dev/null || \ + sleep 24h + +case "${1-}%${#}" in + (%0) DENO_COMPAT=1 exec deno repl ;; + (--version%1) printf -- '%s\n' 'v24.12.0' ; exit ;; +esac + +DENO_COMPAT=1 exec deno run --allow-env --allow-net --allow-sys \ + --allow-ffi="${HOME}" \ + --allow-read=. --allow-read="${HOME}" --allow-read="${XDG_CACHE_HOME:-.}" \ + --allow-write=. --allow-write="${HOME}" --allow-write="${XDG_CACHE_HOME:-.}" \ + "$@" diff --git a/tubesync/install_deno.sh b/tubesync/install_deno.sh index c9f5e6e8..b6584acf 100755 --- a/tubesync/install_deno.sh +++ b/tubesync/install_deno.sh @@ -48,6 +48,10 @@ work_dir="$(mktemp -d)" trap "rm -rf -- '${work_dir}'" EXIT cd "${work_dir}" -download_deno "${deno_archive}" -extract_deno "${deno_archive}" '/usr/local/bin' -record_deno_version '/usr/local/bin/deno' +if [ '--only-record-version' != "${1-unset}" ]; then + download_deno "${deno_archive}" + extract_deno "${deno_archive}" '/usr/local/bin' + record_deno_version '/usr/local/bin/deno' +else + record_deno_version "$(command -v deno)" +fi diff --git a/tubesync/sync/management/commands/fix-mariadb.py b/tubesync/sync/management/commands/fix-mariadb.py index 9b21d2df..16810272 100644 --- a/tubesync/sync/management/commands/fix-mariadb.py +++ b/tubesync/sync/management/commands/fix-mariadb.py @@ -6,6 +6,7 @@ from django.core.management.base import BaseCommand, CommandError from common.logger import log +db.connection.ensure_connection() db_tables = db.connection.introspection.table_names db_quote_name = db.connection.ops.quote_name new_tables = { @@ -24,7 +25,6 @@ def SQLTable(arg_table): needle = arg_table if needle.startswith('new__'): needle = arg_table[len('new__'):] - db.connection.ensure_connection() valid_table_name = ( needle in new_tables and arg_table in db_tables(include_views=False) @@ -123,7 +123,6 @@ class Command(BaseCommand): + f': {db.connection.vendor}' ) - db.connection.ensure_connection() db_is_mariadb = ( hasattr(db.connection, 'mysql_is_mariadb') and db.connection.is_usable() and diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index f9567ad8..b4f07579 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -10,7 +10,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.11' +VERSION = '0.16.1' SECRET_KEY = '' DEBUG = 'true' == getenv('TUBESYNC_DEBUG').strip().lower() ALLOWED_HOSTS = []