diff --git a/.dockerignore b/.dockerignore index be9b48c0..af344e0b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,5 @@ .github .gitattributes README.md +tubesync/media +tubesync/downloads diff --git a/Dockerfile b/Dockerfile index 8ffac3fb..42c312bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,45 @@ FROM debian:buster-slim -ARG DEBIAN_FRONTEND="noninteractive" - -# Third party software versions +ARG ARCH="amd64" +ARG S6_VERSION="2.1.0.2" ARG FFMPEG_VERSION="4.3.1" -ENV FFMPEG_EXPECTED_MD5="ee235393ec7778279144ee6cbdd9eb64" -ENV FFMPEG_TARBALL="https://johnvansickle.com/ffmpeg/releases/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz" + +ENV DEBIAN_FRONTEND="noninteractive" \ + HOME="/root" \ + LANGUAGE="en_US.UTF-8" \ + LANG="en_US.UTF-8" \ + LC_ALL="en_US.UTF-8" \ + TERM="xterm" \ + S6_EXPECTED_SHA256="52460473413601ff7a84ae690b161a074217ddc734990c2cdee9847166cf669e" \ + S6_DOWNLOAD="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${ARCH}.tar.gz" \ + FFMPEG_EXPECTED_SHA256="47d95c0129fba27d051748a442a44a73ce1bd38d1e3f9fe1e9dd7258c7581fa5" \ + FFMPEG_DOWNLOAD="https://johnvansickle.com/ffmpeg/releases/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static.tar.xz" # Install third party software RUN set -x && \ - # Install required distro packages apt-get update && \ + apt-get -y --no-install-recommends install locales && \ + echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen en_US.UTF-8 && \ + # Install required distro packages apt-get -y --no-install-recommends install curl xz-utils ca-certificates binutils && \ + # Install s6 + curl -L ${S6_DOWNLOAD} --output /tmp/s6-overlay-${ARCH}.tar.gz && \ + sha256sum /tmp/s6-overlay-${ARCH}.tar.gz && \ + echo "${S6_EXPECTED_SHA256} /tmp/s6-overlay-${ARCH}.tar.gz" | sha256sum -c - && \ + tar xzf /tmp/s6-overlay-${ARCH}.tar.gz -C / && \ # Install ffmpeg - curl -L ${FFMPEG_TARBALL} --output /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz && \ - echo "${FFMPEG_EXPECTED_MD5} tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz" | md5sum -c - && \ - xz --decompress /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar.xz && \ - tar -xvf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar -C /tmp && \ - ls -lat /tmp/ffmpeg-4.3.1-amd64-static && \ - install -v -s -g root -o root -m 0755 -s /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static/ffmpeg -t /usr/local/bin && \ + curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg-${ARCH}-static.tar.xz && \ + echo "${FFMPEG_EXPECTED_SHA256} /tmp/ffmpeg-${ARCH}-static.tar.xz" | sha256sum -c - && \ + xz --decompress /tmp/ffmpeg-${ARCH}-static.tar.xz && \ + tar -xvf /tmp/ffmpeg-${ARCH}-static.tar -C /tmp && \ + install -v -s -g root -o root -m 0755 -s /tmp/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static/ffmpeg -t /usr/local/bin && \ # Clean up - rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static.tar && \ - rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-amd64-static && \ + rm -rf /tmp/s6-overlay-${ARCH}.tar.gz && \ + rm -rf /tmp/ffmpeg-${ARCH}-static.tar && \ + rm -rf /tmp/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static && \ apt-get -y autoremove --purge curl xz-utils binutils -# Defaults -ARG default_uid="10000" -ARG default_gid="10000" - # Copy app COPY tubesync /app COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings.py @@ -35,7 +47,7 @@ COPY tubesync/tubesync/local_settings.py.container /app/tubesync/local_settings. # Append container bundled software versions RUN echo "ffmpeg_version = '${FFMPEG_VERSION}-static'" >> /app/common/third_party_versions.py -# Add Pipfiles +# Add Pipfile COPY Pipfile /app/Pipfile COPY Pipfile.lock /app/Pipfile.lock @@ -43,40 +55,27 @@ COPY Pipfile.lock /app/Pipfile.lock WORKDIR /app # Set up the app -ENV UID="${default_uid}" -ENV GID="${default_gid}" RUN set -x && \ # Install required distro packages - apt-get -y --no-install-recommends install python3 python3-setuptools python3-pip python3-dev gcc make psmisc procps && \ - # Install wheel which is required for pipenv - pip3 --disable-pip-version-check install wheel && \ - # Then install pipenv + apt-get -y install nginx-light && \ + apt-get -y --no-install-recommends install python3 python3-setuptools python3-pip python3-dev gcc make && \ + # Install pipenv pip3 --disable-pip-version-check install pipenv && \ - # Create a 'www' user which the workers drop to - groupadd -g ${GID} www && \ - useradd -M -d /app -s /bin/false -u ${UID} -g www www && \ + # Create a 'app' user which the application will run as + groupadd app && \ + useradd -M -d /app -s /bin/false -g app app && \ # Install non-distro packages - pipenv install --system --verbose && \ + pipenv install --system && \ # Make absolutely sure we didn't accidentally bundle a SQLite dev database rm -rf /app/db.sqlite3 && \ - # Create config, downloads and run dirs we can write to - mkdir -p /run/www && \ - chown -R www:www /run/www && \ - chmod -R 0700 /run/www && \ + # Run any required app commands + /usr/bin/python3 /app/manage.py compilescss && \ + /usr/bin/python3 /app/manage.py collectstatic --no-input --link && \ + # Create config, downloads and run dirs + mkdir -p /run/app && \ mkdir -p /config/media && \ - chown -R www:www /config && \ - chmod -R 0755 /config && \ - mkdir -p /downloads/{audio,video} && \ - chown -R www:www /downloads && \ - chmod -R 0755 /downloads && \ - # Reset permissions - mkdir -p /app/static && \ - chown -R root:www /app && \ - chown -R www:www /app/common/static && \ - chown -R www:www /app/static && \ - chmod -R 0750 /app && \ - find /app -type f -exec chmod 640 {} \; && \ - chmod 0750 /app/entrypoint.sh && \ + mkdir -p /downloads/audio && \ + mkdir -p /downloads/video && \ # Clean up rm /app/Pipfile && \ rm /app/Pipfile.lock && \ @@ -94,18 +93,18 @@ RUN set -x && \ chown root:root /root && \ chmod 0700 /root +# Copy root +COPY config/root / + # Create a healthcheck HEALTHCHECK --interval=1m --timeout=10s CMD /app/healthcheck.py http://127.0.0.1:8080/healthcheck -# Drop to the www user -#USER www - # ENVS and ports ENV PYTHONPATH "/app:${PYTHONPATH}" EXPOSE 8080 -# Entrypoint -ENTRYPOINT ["/app/entrypoint.sh"] +# Volumes +VOLUME ["/config", "/downloads"] -# Run gunicorn -CMD ["/usr/local/bin/gunicorn", "-c", "/app/tubesync/gunicorn.py", "--capture-output", "tubesync.wsgi:application"] +# Entrypoint, start s6 init +ENTRYPOINT ["/init"] diff --git a/Makefile b/Makefile index 82195047..2f4b3469 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ container: clean runcontainer: - $(docker) run --rm --name $(name) --env-file dev.env --log-opt max-size=50m -ti -p 8080:8080 $(image) + $(docker) run --rm --name $(name) --env-file dev.env --log-opt max-size=50m -ti -p 4848:4848 $(image) test: diff --git a/config/root/docs/index.html b/config/root/docs/index.html new file mode 100644 index 00000000..196cdf20 --- /dev/null +++ b/config/root/docs/index.html @@ -0,0 +1,27 @@ + + + + + + Something has gone very wrong + + + +
+

Something has gone very wrong

+
+
+

+ If you can see this message then the front end web server has not forwarded the + connection on to the TubeSync back end server. This probably means something has + gone wrong with the container build or a process has crashed. Try restarting it. +

+
+ + diff --git a/config/root/etc/cont-init.d/10-tubesync b/config/root/etc/cont-init.d/10-tubesync new file mode 100644 index 00000000..d72068f1 --- /dev/null +++ b/config/root/etc/cont-init.d/10-tubesync @@ -0,0 +1,22 @@ +#!/usr/bin/with-contenv bash + +# Change runtime user UID and GID +PUID=${PUID:-911} +PGID=${PGID:-911} +groupmod -o -g "$PGID" app +usermod -o -u "$PUID" app + +# Reset permissions +chown -R app:app /run/app && \ +chmod -R 0700 /run/app && \ +chown -R app:app /config && \ +chmod -R 0755 /config && \ +chown -R app:app /downloads && \ +chmod -R 0755 /downloads && \ +chown -R root:app /app && \ +chmod -R 0750 /app && \ +find /app -type f -exec chmod 640 {} \; && \ + +# Run migrations +exec s6-setuidgid app \ + /usr/bin/python3 /app/manage.py migrate diff --git a/config/root/etc/nginx/nginx.conf b/config/root/etc/nginx/nginx.conf new file mode 100644 index 00000000..0bb871c0 --- /dev/null +++ b/config/root/etc/nginx/nginx.conf @@ -0,0 +1,85 @@ +daemon off; + +user app; +worker_processes auto; +pid /run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + + # Basic settings + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 300; + types_hash_max_size 2048; + server_tokens off; + server_names_hash_bucket_size 64; + server_name_in_redirect off; + client_body_in_file_only clean; + client_body_buffer_size 32K; + client_max_body_size 100M; + send_timeout 300s; + large_client_header_buffers 4 8k; + + # Mime type handling + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Default security headers + add_header X-Frame-Options SAMEORIGIN; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; + + # Logging + log_format host '$remote_addr - $remote_user [$time_local] "[$host] $request" $status $bytes_sent "$http_referer" "$http_user_agent" "$gzip_ratio"'; + access_log /dev/stdout; + error_log stderr; + + # GZIP + gzip on; + gzip_disable "msie6"; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + + # Site + server { + + # Ports + listen 4848; + listen [::]:4848; + + # Web root + root /docs; + index index.html; + + # Proxy + proxy_buffers 32 4k; + proxy_set_header Connection ""; + + # Server domain name + server_name _; + + # Authentication and proxying + location / { + proxy_pass http://127.0.0.1:8080; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_redirect off; + proxy_read_timeout 59; + proxy_connect_timeout 10; + } + + } + + +} diff --git a/config/root/etc/services.d/gunicorn/run b/config/root/etc/services.d/gunicorn/run new file mode 100755 index 00000000..08f356e8 --- /dev/null +++ b/config/root/etc/services.d/gunicorn/run @@ -0,0 +1,9 @@ +#!/usr/bin/with-contenv bash + +UMASK_SET=${UMASK_SET:-022} +umask "$UMASK_SET" + +cd /app || exit + +exec s6-setuidgid app \ + /usr/local/bin/gunicorn -c /app/tubesync/gunicorn.py --capture-output tubesync.wsgi:application diff --git a/config/root/etc/services.d/nginx/run b/config/root/etc/services.d/nginx/run new file mode 100755 index 00000000..9ff6f462 --- /dev/null +++ b/config/root/etc/services.d/nginx/run @@ -0,0 +1,5 @@ +#!/usr/bin/with-contenv bash + +cd / + +/usr/sbin/nginx diff --git a/config/root/etc/services.d/tubesync-worker/run b/config/root/etc/services.d/tubesync-worker/run new file mode 100755 index 00000000..cb674290 --- /dev/null +++ b/config/root/etc/services.d/tubesync-worker/run @@ -0,0 +1,4 @@ +#!/usr/bin/with-contenv bash + +exec s6-setuidgid app \ + /usr/bin/python3 /app/manage.py process_tasks diff --git a/dev.env b/dev.env index 8a78969a..7a622e6f 100644 --- a/dev.env +++ b/dev.env @@ -1,3 +1,5 @@ GUNICORN_WORKERS=1 DJANGO_ALLOWED_HOSTS=localhost DJANGO_SECRET_KEY=not-a-secret +PUID=1234 +PGID=1234 diff --git a/tubesync/entrypoint.sh b/tubesync/entrypoint.sh deleted file mode 100755 index 7d8fdf6e..00000000 --- a/tubesync/entrypoint.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -x - -# Compile SCSS files -/usr/bin/python3 /app/manage.py compilescss - -# Collect the static files -/usr/bin/python3 /app/manage.py collectstatic --no-input --link - -# Run migrations -/usr/bin/python3 /app/manage.py migrate - -# Run what's in CMD -exec "$@" - -# eof diff --git a/tubesync/tubesync/gunicorn.py b/tubesync/tubesync/gunicorn.py index 2c0883f7..d59c1389 100644 --- a/tubesync/tubesync/gunicorn.py +++ b/tubesync/tubesync/gunicorn.py @@ -3,9 +3,11 @@ import multiprocessing def get_num_workers(): + # Sane max workers to allow to be spawned cpu_workers = multiprocessing.cpu_count() * 2 + 1 + # But default to 3 try: - num_workers = int(os.getenv('GUNICORN_WORKERS', 2)) + num_workers = int(os.getenv('GUNICORN_WORKERS', 3)) except ValueError: num_workers = cpu_workers if 0 > num_workers > cpu_workers: @@ -14,7 +16,7 @@ def get_num_workers(): def get_bind(): - host = os.getenv('LISTEN_HOST', '0.0.0.0') + host = os.getenv('LISTEN_HOST', '127.0.0.1') port = os.getenv('LISTEN_PORT', '8080') return '{}:{}'.format(host, port) @@ -23,11 +25,11 @@ workers = get_num_workers() timeout = 30 chdir = '/app' daemon = False -pidfile = '/run/www/gunicorn.pid' -user = 'www' -group = 'www' +pidfile = '/run/app/gunicorn.pid' +user = 'app' +group = 'app' loglevel = 'info' errorlog = '-' -accesslog = '-' +accesslog = '/dev/null' # Access logs are printed to stdout from nginx django_settings = 'django.settings' bind = get_bind() diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index c51183bb..e8a82aa8 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -1,6 +1,6 @@ import os from pathlib import Path -from +from binascii import hexlify BASE_DIR = Path(__file__).resolve().parent.parent @@ -9,20 +9,20 @@ ROOT_DIR = Path('/') RANDOM_SECRET = hexlify(os.urandom(32)).decode() SECRET_KEY = str(os.getenv('DJANGO_SECRET_KEY', RANDOM_SECRET)) -ALLOWED_HOSTS_STR = str(os.getenv('TUBESYNC_HOSTS', 'localhost')) +ALLOWED_HOSTS_STR = str(os.getenv('TUBESYNC_HOSTS', '127.0.0.1,localhost')) ALLOWED_HOSTS = ALLOWED_HOSTS_STR.split(',') +TIME_ZONE = os.getenv('TZ', 'UTC') + + DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': '/config/db.sqlite3', + 'NAME': ROOT_DIR / 'config' / 'db.sqlite3', } } -BACKGROUND_TASK_ASYNC_THREADS = int(os.get('TUBESYNC_WORKERS', 2)) - - MEDIA_ROOT = ROOT_DIR / 'config' / 'media' DOWNLOAD_ROOT = ROOT_DIR / 'downloads' diff --git a/tubesync/tubesync/local_settings.py.example b/tubesync/tubesync/local_settings.py.example index e69de29b..dc511a66 100644 --- a/tubesync/tubesync/local_settings.py.example +++ b/tubesync/tubesync/local_settings.py.example @@ -0,0 +1,19 @@ +from pathlib import Path + + +BASE_DIR = Path(__file__).resolve().parent.parent + + +SECRET_KEY = 'example-secret-key' +DEBUG = False + + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } +} + + +DOWNLOAD_ROOT = BASE_DIR / 'downloads'