Merge branch 'main' into patch-2

.github/actions/get-tag/action.yml

@@ -0,0 +1,41 @@
+name: Get tag
+description: Get tag name from GITHUB_REF environment variable
+inputs:
+  strip_v:
+    required: false
+    default: false
+    description: Whether to strip "v" from the tag or not
+outputs:
+  tag:
+    value: ${{ steps.set.outputs.tag }}
+    description: Git tag name
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set outputs
+      id: 'set'
+      env:
+        INPUT_STRIP_V: '${{ inputs.strip_v }}'
+      shell: 'bash'
+      run: |
+        tag="${GITHUB_REF}"
+        printf -- 'Manipulating string: %s\n' "${tag}"
+        test -n "${tag}" || exit 1
+
+        case "${tag}" in
+          (refs/tags/*) tag="${tag#refs/tags/}" ;;
+          (*) printf -- 'Not a tag ref\n' ; exit 2 ;;
+        esac
+
+        if [ 'true' = "${INPUT_STRIP_V,,}" ]
+        then
+            tag="${tag#[Vv]}"
+        fi
+        
+        set_sl_var() { local f='%s=%s\n' ; printf -- "${f}" "$@" ; } ;
+
+        set_sl_var tag "${tag}" >> "${GITHUB_OUTPUT}"
+
+        set_sl_var 'tag ' " ${tag}"
+

.github/workflows/ci.yaml

@@ -93,15 +93,61 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install uv
-          pip install pipenv
+          uv --no-config --no-managed-python --no-progress \
-          PIPENV_VERBOSITY=64 pipenv install --system --skip-lock
+            pip install --system --strict pipenv
+          pipenv lock
+          pipenv requirements | tee requirements.txt
+          #PIPENV_VERBOSITY=64 pipenv install --system --skip-lock
+          uv --no-config --no-managed-python --no-progress \
+            pip install --system --strict --requirements requirements.txt
       - name: Set up Django environment
         run: |
           cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py
           cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/*
           cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/*
           cd tubesync && python3 -B manage.py collectstatic --no-input --link
+      - name: Check with ruff
+        continue-on-error: false
+        run: |
+          target_version='py310'
+          ignore_csv_list='E701,E722,E731'
+          cd tubesync
+          # output formats:
+          # "full" | "concise" | "grouped" |
+          # "json" | "junit" | "github" | "gitlab" |
+          # "pylint" | "azure"
+          {
+              echo '## Output from `ruff check` for `tubesync`'
+              echo ''
+              echo '### Formats'
+              echo ''
+              for fmt in full concise grouped pylint
+              do
+              echo '<details>'
+              echo '<summary>'"${fmt}"'</summary>'
+              echo ''
+              echo '#### '"${fmt}"' output format'
+              echo ''
+              echo '```'
+              uvx --no-config --no-managed-python --no-progress --isolated \
+                  ruff check --exit-zero \
+                  --target-version "${target_version}" \
+                  --output-format "${fmt}" \
+                  --extend-select RUF100 \
+                  --ignore "${ignore_csv_list}"
+              echo ''
+              echo '```'
+              echo ''
+              echo '</details>'
+              echo ''
+              done
+          } >> "${GITHUB_STEP_SUMMARY}"
+          uvx --no-config --no-managed-python --no-progress --isolated \
+              ruff check --exit-zero \
+              --target-version "${target_version}" \
+              --output-format github \
+              --ignore "${ignore_csv_list}"
       - name: Run Django tests
         run: cd tubesync && python3 -B -W default manage.py test --verbosity=2
 

.github/workflows/release.yaml

@@ -8,31 +8,89 @@ on:
     types: [published]
 
 jobs:
+  info:
+    runs-on: ubuntu-latest
+    outputs:
+      ffmpeg-date: ${{ steps.jq.outputs.FFMPEG_DATE }}
+      ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }}
+      ffmpeg-version: ${{ steps.jq.outputs.FFMPEG_VERSION }}
+      lowercase-github-actor: ${{ steps.github-actor.outputs.lowercase }}
+      lowercase-github-repository_owner: ${{ steps.github-repository_owner.outputs.lowercase }}
+      tag: ${{ steps.tag.outputs.tag }}
+      ytdlp-latest-release: ${{ steps.yt-dlp.outputs.latest-release }}
+      ytdlp-releases: ${{ steps.yt-dlp.outputs.releases }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get tag
+        id: tag
+        uses: ./.github/actions/get-tag
+      - name: Lowercase github username
+        id: github-actor
+        uses: ./.github/actions/string-case
+        with:
+          string: ${{ github.actor }}
+      - name: Lowercase github repository owner
+        id: github-repository_owner
+        uses: ./.github/actions/string-case
+        with:
+          string: ${{ github.repository_owner }}
+      - name: Retrieve yt-dlp/FFmpeg-Builds releases with GitHub CLI
+        id: ffmpeg
+        uses: ./.github/actions/FFmpeg
+      - name: Retrieve yt-dlp/yt-dlp releases with GitHub CLI
+        id: yt-dlp
+        uses: ./.github/actions/yt-dlp
+      - name: Set outputs with jq
+        id: jq
+        run: |
+          cat >| .ffmpeg.releases.json <<'EOF'
+          ${{ steps.ffmpeg.outputs.releases }}
+          EOF
+          mk_delim() { local f='%s_EOF_%d_' ; printf -- "${f}" "$1" "${RANDOM}" ; } ;
+          open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ;
+          close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ;
+          {
+            var='FFMPEG_DATE' ;
+            delim="$(mk_delim "${var}")" ;
+            open_ml_var "${delim}" "${var}" ;
+            jq_arg='[foreach .[] as $release ([{}, []]; [ .[0] + {($release.commit): ([ $release.date ] + (.[0][($release.commit)] // []) ) }, [ .[1][0] // $release.commit ] ] ; .[0][(.[1][0])] ) ][-1][0]' ;
+            jq -r "${jq_arg}" -- .ffmpeg.releases.json ;
+            close_ml_var "${delim}" "${var}" ;
+
+            ffmpeg_date="$( jq -r "${jq_arg}" -- .ffmpeg.releases.json )"
+            
+            var='FFMPEG_VERSION' ;
+            delim="$(mk_delim "${var}")" ;
+            open_ml_var "${delim}" "${var}" ;
+            jq_arg='.[]|select(.date == $date)|.versions[]|select(startswith("N-"))' ;
+            jq -r --arg date "${ffmpeg_date}" "${jq_arg}" -- .ffmpeg.releases.json ;
+            close_ml_var "${delim}" "${var}" ;
+            unset -v delim jq_arg var ;
+          } >> "${GITHUB_OUTPUT}"
+          cat -v "${GITHUB_OUTPUT}"
+          rm -v -f .ffmpeg.releases.json
+
   containerise:
+    needs: ['info']
     runs-on: ubuntu-latest
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
-      - name: Get tag
-        id: tag
-        uses: dawidd6/action-get-tag@v1
-      - uses: docker/build-push-action@v2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Log into GitHub Container Registry
         run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
-      - name: Lowercase github username for ghcr
-        id: string
-        uses: ASzc/change-string-case-action@v1
-        with:
-          string: ${{ github.actor }}
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
+          tags: ghcr.io/${{ needs.info.outputs.lowercase-github-actor }}/${{ env.IMAGE_NAME }}:${{ needs.info.outputs.tag }}
-          cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
+          cache-from: |
-          cache-to: type=inline
+            type=registry,ref=ghcr.io/${{ needs.info.outputs.lowercase-github-repository_owner }}/${{ env.IMAGE_NAME }}:latest
+            type=gha
           build-args: |
             IMAGE_NAME=${{ env.IMAGE_NAME }}
+            FFMPEG_DATE=${{ needs.info.outputs.ffmpeg-date }}
+            FFMPEG_VERSION=${{ needs.info.outputs.ffmpeg-version }}
+            YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }}

Dockerfile

@@ -410,6 +410,12 @@ RUN --mount=type=tmpfs,target=/cache \
   PIPENV_VERBOSITY=64 \
   PYTHONPYCACHEPREFIX=/cache/pycache \
     pipenv install --system --skip-lock && \
+  # remove the getpot_bgutil_script plugin
+  find /usr/local/lib \
+  -name 'getpot_bgutil_script.py' \
+  -path '*/yt_dlp_plugins/extractor/getpot_bgutil_script.py' \
+  -type f -print -delete \
+  && \
   # Clean up
   apt-get -y autoremove --purge \
   default-libmysqlclient-dev \

Pipfile

@@ -7,7 +7,7 @@ verify_ssl = true
 autopep8 = "*"
 
 [packages]
-django = "*"
+django = "~=5.2.1"
 django-sass-processor = {extras = ["management-command"], version = "*"}
 pillow = "*"
 whitenoise = "*"
@@ -24,3 +24,5 @@ yt-dlp = {extras = ["default", "curl-cffi"], version = "*"}
 emoji = "*"
 brotli = "*"
 html5lib = "*"
+yt-dlp-get-pot = "*"
+bgutil-ytdlp-pot-provider = "*"

config/root/etc/nginx/nginx.conf

@@ -5,6 +5,8 @@ worker_processes auto;
 worker_cpu_affinity auto;
 pid /run/nginx.pid;
 
+env YT_POT_BGUTIL_BASE_URL;
+
 events {
     worker_connections 1024;
 }
@@ -135,4 +137,7 @@ http {
         }
     }
 
+    # Proof-of-Origin Token Server
+    include /etc/nginx/token_server.conf;
+
 }

config/root/etc/nginx/token_server.conf

@@ -0,0 +1,29 @@
+upstream token_server {
+    server 127.0.0.2:4416 down;
+}
+
+server {
+
+    # Ports
+    listen 4416;
+    listen [::]:4416;
+
+    # Server domain name
+    server_name _;
+
+    set_by_lua_block $pot_url {
+        local default = 'http://token_server'
+        local url = os.getenv('YT_POT_BGUTIL_BASE_URL')
+        if not url then
+            return default
+        end
+        if #url and url:find('://') then
+            return url
+        end
+        return default
+    }
+
+    location / {
+        proxy_pass $pot_url;
+    }
+}

config/root/etc/s6-overlay/s6-rc.d/nginx/run

@@ -2,4 +2,15 @@
 
 cd /
 
+https="${TUBESYNC_POT_HTTPS:+https}"
+ip_address="${TUBESYNC_POT_IPADDR:-${POTSERVER_PORT_4416_TCP_ADDR}}"
+: "${TUBESYNC_POT_PORT:=${POTSERVER_PORT_4416_TCP_PORT}}"
+port="${TUBESYNC_POT_PORT:+:}${TUBESYNC_POT_PORT}"
+
+if [ -n "${ip_address}" ]
+then
+    YT_POT_BGUTIL_BASE_URL="${https:-http}://${ip_address}${port}"
+    export YT_POT_BGUTIL_BASE_URL
+fi
+
 exec /usr/bin/openresty -c /etc/nginx/nginx.conf -e stderr

tubesync/common/json.py

@@ -1,4 +1,6 @@
+from datetime import datetime
 from django.core.serializers.json import DjangoJSONEncoder
+from yt_dlp.utils import LazyList
 
 
 class JSONEncoder(DjangoJSONEncoder):
@@ -14,3 +16,11 @@ class JSONEncoder(DjangoJSONEncoder):
             return list(iterable)
         return super().default(obj)
 
+
+def json_serial(obj):
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    if isinstance(obj, LazyList):
+        return list(obj)
+    raise TypeError(f'Type {type(obj)} is not json_serial()-able')
+

tubesync/common/urls.py

@@ -1,7 +1,6 @@
 from django.conf import settings
 from django.urls import path
 from django.views.generic.base import RedirectView
-from django.views.generic import TemplateView
 from django.http import HttpResponse
 from .views import error403, error404, error500, HealthCheckView
 

tubesync/common/utils.py

@@ -6,13 +6,11 @@ import os
 import pstats
 import string
 import time
-from datetime import datetime
 from django.core.paginator import Paginator
 from functools import partial
 from operator import attrgetter, itemgetter
 from pathlib import Path
 from urllib.parse import urlunsplit, urlencode, urlparse
-from yt_dlp.utils import LazyList
 from .errors import DatabaseConnectionError
 
 
@@ -140,14 +138,11 @@ def parse_database_connection_string(database_connection_string):
                                       f'invalid driver, must be one of {valid_drivers}')
     django_driver = django_backends.get(driver)
     host_parts = user_pass_host_port.split('@')
-    if len(host_parts) != 2:
+    user_pass_parts = host_parts[0].split(':')
-        raise DatabaseConnectionError(f'Database connection string netloc must be in '
+    if len(host_parts) != 2 or len(user_pass_parts) != 2:
-                                      f'the format of user:pass@host')
+        raise DatabaseConnectionError('Database connection string netloc must be in '
+                                      'the format of user:pass@host')
     user_pass, host_port = host_parts
-    user_pass_parts = user_pass.split(':')
-    if len(user_pass_parts) != 2:
-        raise DatabaseConnectionError(f'Database connection string netloc must be in '
-                                      f'the format of user:pass@host')
     username, password = user_pass_parts
     host_port_parts = host_port.split(':')
     if len(host_port_parts) == 1:
@@ -169,13 +164,13 @@ def parse_database_connection_string(database_connection_string):
                                           f'65535, got {port}')
     else:
         # Malformed
-        raise DatabaseConnectionError(f'Database connection host must be a hostname or '
+        raise DatabaseConnectionError('Database connection host must be a hostname or '
-                                      f'a hostname:port combination')
+                                      'a hostname:port combination')
     if database.startswith('/'):
         database = database[1:]
     if not database:
-        raise DatabaseConnectionError(f'Database connection string path must be a '
+        raise DatabaseConnectionError('Database connection string path must be a '
-                                      f'string in the format of /databasename')    
+                                      'string in the format of /databasename')    
     if '/' in database:
         raise DatabaseConnectionError(f'Database connection string path can only '
                                       f'contain a single string name, got: {database}')

tubesync/sync/fields.py

@@ -63,7 +63,7 @@ class CommaSepChoiceField(models.CharField):
     def __init__(self, *args, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, **kwargs):
         kwargs.setdefault('max_length', 128)
         self.separator = str(separator)
-        self.possible_choices = possible_choices or choices
+        self.possible_choices = possible_choices or kwargs.get('choices')
         self.selected_choices = list()
         self.allow_all = allow_all
         self.all_label = all_label

tubesync/sync/filtering.py

@@ -5,7 +5,6 @@
 from common.logger import log
 from .models import Media
 from datetime import datetime
-from django.utils import timezone
 from .overrides.custom_filter import filter_custom
 
 

tubesync/sync/hooks.py

@@ -1,9 +1,7 @@
 import os
-import yt_dlp
 
 from common.logger import log
 from common.utils import remove_enclosed
-from django.conf import settings
 
 
 progress_hook = {

tubesync/sync/management/commands/delete-source.py

@@ -1,16 +1,15 @@
-import os
 import uuid
-from django.utils.translation import gettext_lazy as _
 from django.core.management.base import BaseCommand, CommandError
 from django.db.transaction import atomic
+from django.utils.translation import gettext_lazy as _
 from common.logger import log
-from sync.models import Source, Media, MediaServer
+from sync.models import Source
 from sync.tasks import schedule_media_servers_update
 
 
 class Command(BaseCommand):
 
-    help = _('Deletes a source by UUID')
+    help = 'Deletes a source by UUID'
 
     def add_arguments(self, parser):
         parser.add_argument('--source', action='store', required=True, help=_('Source UUID'))

tubesync/sync/management/commands/import-existing-media.py

@@ -1,6 +1,6 @@
 import os
 from pathlib import Path
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError # noqa
 from common.logger import log
 from common.timestamp import timestamp_to_datetime
 from sync.choices import FileExtension
@@ -18,7 +18,7 @@ class Command(BaseCommand):
         dirmap = {}
         for s in Source.objects.all():
             dirmap[str(s.directory_path)] = s
-        log.info(f'Scanning sources...')
+        log.info('Scanning sources...')
         file_extensions = list(FileExtension.values) + self.extra_extensions
         for sourceroot, source in dirmap.items():
             media = list(Media.objects.filter(source=source, downloaded=False,

tubesync/sync/management/commands/list-sources.py

@@ -1,7 +1,6 @@
-import os
+from django.core.management.base import BaseCommand, CommandError # noqa
-from django.core.management.base import BaseCommand, CommandError
 from common.logger import log
-from sync.models import Source, Media, MediaServer
+from sync.models import Source
 
 
 class Command(BaseCommand):

tubesync/sync/management/commands/reset-tasks.py

@@ -1,14 +1,12 @@
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError # noqa
 from django.db.transaction import atomic
 from django.utils.translation import gettext_lazy as _
 from background_task.models import Task
+from common.logger import log
 from sync.models import Source
 from sync.tasks import index_source_task, check_source_directory_exists
 
 
-from common.logger import log
-
-
 class Command(BaseCommand):
 
     help = 'Resets all tasks'
@@ -31,10 +29,10 @@ class Command(BaseCommand):
                 index_source_task(
                     str(source.pk),
                     repeat=source.index_schedule,
+                    schedule=source.index_schedule,
                     verbose_name=verbose_name.format(source.name),
                 )
-        with atomic(durable=True):
-            for source in Source.objects.all():
                 # This also chains down to call each Media objects .save() as well
                 source.save()
+
         log.info('Done')

tubesync/sync/management/commands/sync-missing-metadata.py

@@ -1,6 +1,5 @@
-import os
 from shutil import copyfile
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError # noqa
 from django.db.models import Q
 from common.logger import log
 from sync.models import Source, Media

tubesync/sync/management/commands/youtube-dl-info.py

@@ -1,7 +1,7 @@
 import json
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError # noqa
 from sync.youtube import get_media_info
-from common.utils import json_serial
+from common.json import JSONEncoder
 
 
 class Command(BaseCommand):
@@ -15,6 +15,6 @@ class Command(BaseCommand):
         url = options['url']
         self.stdout.write(f'Showing information for URL: {url}')
         info = get_media_info(url)
-        d = json.dumps(info, indent=4, sort_keys=True, default=json_serial)
+        d = json.dumps(info, indent=4, sort_keys=True, cls=JSONEncoder)
         self.stdout.write(d)
         self.stdout.write('Done')

tubesync/sync/mediaservers.py

@@ -117,9 +117,6 @@ class PlexMediaServer(MediaServer):
             raise ValidationError('Plex Media Server "port" must be between 1 '
                                   'and 65535')
         options = self.object.options
-        if 'token' not in options:
-            raise ValidationError('Plex Media Server requires a "token"')
-        token = options['token'].strip()
         if 'token' not in options:
             raise ValidationError('Plex Media Server requires a "token"')
         if 'libraries' not in options:

tubesync/sync/migrations/0011_auto_20220201_1654.py

@@ -1,8 +1,6 @@
 # Generated by Django 3.2.11 on 2022-02-01 16:54
 
-import django.core.files.storage
 from django.db import migrations, models
-import sync.models
 
 
 class Migration(migrations.Migration):

tubesync/sync/migrations/0013_fix_elative_media_file.py

@@ -1,7 +1,7 @@
 # Generated by Django 3.2.12 on 2022-04-06 06:19
 
 from django.conf import settings
-from django.db import migrations, models
+from django.db import migrations
 
 
 def fix_media_file(apps, schema_editor):

tubesync/sync/models/__init__.py

@@ -17,3 +17,9 @@ from .media import Media
 from .metadata import Metadata
 from .metadata_format import MetadataFormat
 
+__all__ = [
+    'get_media_file_path', 'get_media_thumb_path',
+    'media_file_storage', 'MediaServer', 'Source',
+    'Media', 'Metadata', 'MetadataFormat',
+]
+

tubesync/sync/models/_private.py

@@ -1,4 +1,4 @@
-from ..choices import Val, YouTube_SourceType
+from ..choices import Val, YouTube_SourceType # noqa
 
 
 _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) ))

tubesync/sync/models/media.py

@@ -15,9 +15,9 @@ from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from common.logger import log
 from common.errors import NoFormatException
+from common.json import JSONEncoder
 from common.utils import (
     clean_filename, clean_emoji,
-    django_queryset_generator as qs_gen,
 )
 from ..youtube import (
     get_media_info as get_youtube_media_info,
@@ -40,6 +40,9 @@ from ._migrations import (
     media_file_storage, get_media_thumb_path, get_media_file_path,
 )
 from ._private import _srctype_dict, _nfo_element
+from .media__tasks import (
+    download_checklist, download_finished, wait_for_premiere,
+)
 from .source import Source
 
 
@@ -575,14 +578,13 @@ class Media(models.Model):
 
 
     def metadata_dumps(self, arg_dict=dict()):
-        from common.utils import json_serial
         fallback = dict()
         try:
             fallback.update(self.new_metadata.with_formats)
         except ObjectDoesNotExist:
             pass
         data = arg_dict or fallback
-        return json.dumps(data, separators=(',', ':'), default=json_serial)
+        return json.dumps(data, separators=(',', ':'), cls=JSONEncoder)
 
 
     def metadata_loads(self, arg_str='{}'):
@@ -685,7 +687,7 @@ class Media(models.Model):
                 pass
             setattr(self, '_cached_metadata_dict', data)
             return data
-        except Exception as e:
+        except Exception:
             return {}
 
 
@@ -1216,6 +1218,12 @@ class Media(models.Model):
                             parent_dir.rmdir()
                             log.info(f'Removed empty directory: {parent_dir!s}')
                             parent_dir = parent_dir.parent
-                    except OSError as e:
+                    except OSError:
                         pass
 
+
+# add imported functions
+Media.download_checklist = download_checklist
+Media.download_finished = download_finished
+Media.wait_for_premiere = wait_for_premiere
+

tubesync/sync/models/media__tasks.py

@@ -0,0 +1,118 @@
+import os
+from pathlib import Path
+from common.logger import log
+from common.errors import (
+    NoMetadataException,
+)
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+from ..choices import Val, SourceResolution
+
+
+def download_checklist(self, skip_checks=False):
+    media = self
+    if skip_checks:
+        return True
+
+    if not media.source.download_media:
+        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
+                 f'the source {media.source} has since been marked to not download, '
+                 f'not downloading')
+        return False
+    if media.skip or media.manual_skip:
+        # Media was toggled to be skipped after the task was scheduled
+        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
+                 f'it is now marked to be skipped, not downloading')
+        return False
+    # metadata is required to generate the proper filepath
+    if not media.has_metadata:
+        raise NoMetadataException('Metadata is not yet available.')
+    downloaded_file_exists = (
+        media.downloaded and
+        media.has_metadata and
+        (
+            media.media_file_exists or
+            media.filepath.exists()
+        )
+    )
+    if downloaded_file_exists:
+        # Media has been marked as downloaded before the download_media task was fired,
+        # skip it
+        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
+                 f'it has already been marked as downloaded, not downloading again')
+        return False
+    max_cap_age = media.source.download_cap_date
+    published = media.published
+    if max_cap_age and published:
+        if published <= max_cap_age:
+            log.warn(f'Download task triggered media: {media} (UUID: {media.pk}) but '
+                     f'the source has a download cap and the media is now too old, '
+                     f'not downloading')
+            return False
+    return True
+
+
+def download_finished(self, format_str, container, downloaded_filepath=None):
+    media = self
+    if downloaded_filepath is None:
+        downloaded_filepath = self.filepath
+    filepath = Path(downloaded_filepath)
+
+    # Media has been downloaded successfully
+    log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: '
+             f'"{filepath}"')
+    # Link the media file to the object and update info about the download
+    self.media_file.name = str(filepath.relative_to(self.media_file.storage.location))
+    media.downloaded = True
+    media.download_date = timezone.now()
+    media.downloaded_filesize = os.path.getsize(filepath)
+    media.downloaded_container = container
+    if '+' in format_str:
+        # Seperate audio and video streams
+        vformat_code, aformat_code = format_str.split('+')
+        aformat = media.get_format_by_code(aformat_code)
+        vformat = media.get_format_by_code(vformat_code)
+        media.downloaded_format = vformat['format']
+        media.downloaded_height = vformat['height']
+        media.downloaded_width = vformat['width']
+        media.downloaded_audio_codec = aformat['acodec']
+        media.downloaded_video_codec = vformat['vcodec']
+        media.downloaded_container = container
+        media.downloaded_fps = vformat['fps']
+        media.downloaded_hdr = vformat['is_hdr']
+    else:
+        # Combined stream or audio-only stream
+        cformat_code = format_str
+        cformat = media.get_format_by_code(cformat_code)
+        media.downloaded_audio_codec = cformat['acodec']
+        if cformat['vcodec']:
+            # Combined
+            media.downloaded_format = cformat['format']
+            media.downloaded_height = cformat['height']
+            media.downloaded_width = cformat['width']
+            media.downloaded_video_codec = cformat['vcodec']
+            media.downloaded_fps = cformat['fps']
+            media.downloaded_hdr = cformat['is_hdr']
+        else:
+            self.downloaded_format = Val(SourceResolution.AUDIO)
+
+
+def wait_for_premiere(self):
+    hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60)))
+
+    in_hours = None
+    if self.has_metadata or not self.published:
+        return (False, in_hours,)
+
+    now = timezone.now()
+    if self.published < now:
+        in_hours = 0
+        self.manual_skip = False
+        self.skip = False
+    else:
+        in_hours = hours(self.published - now)
+        self.manual_skip = True
+        self.title = _(f'Premieres in {in_hours} hours')
+
+    return (True, in_hours,)
+

tubesync/sync/models/source.py

@@ -1,6 +1,7 @@
 import os
 import re
 import uuid
+from collections import deque as queue
 from pathlib import Path
 from django import db
 from django.conf import settings
@@ -510,7 +511,7 @@ class Source(db.models.Model):
     def get_example_media_format(self):
         try:
             return self.media_format.format(**self.example_media_format_dict)
-        except Exception as e:
+        except Exception:
             return ''
 
     def is_regex_match(self, media_item_title):
@@ -527,23 +528,31 @@ class Source(db.models.Model):
             days = timezone.timedelta(seconds=self.download_cap).days
         response = indexer(self.get_index_url(type=type), days=days)
         if not isinstance(response, dict):
-            return []
+            return list()
-        entries = response.get('entries', []) 
+        entries = response.get('entries', list()) 
         return entries
 
     def index_media(self):
         '''
-            Index the media source returning a list of media metadata as dicts.
+            Index the media source returning a queue of media metadata as dicts.
         '''
-        entries = list()
+        entries = queue(list(), getattr(settings, 'MAX_ENTRIES_PROCESSING', 0) or None)
         if self.index_videos:
-            entries += self.get_index('videos')
+            entries.extend(reversed(self.get_index('videos')))
+
         # Playlists do something different that I have yet to figure out
         if not self.is_playlist:
             if self.index_streams:
-                entries += self.get_index('streams')
+                streams = self.get_index('streams')
+                if entries.maxlen is None or 0 == len(entries):
+                    entries.extend(reversed(streams))
+                else:
+                    # share the queue between streams and videos
+                    allowed_streams = max(
+                        entries.maxlen // 2,
+                        entries.maxlen - len(entries),
+                    )
+                    entries.extend(reversed(streams[: allowed_streams]))
 
-        if settings.MAX_ENTRIES_PROCESSING:
-            entries = entries[:settings.MAX_ENTRIES_PROCESSING]
         return entries
 

tubesync/sync/signals.py

@@ -10,11 +10,11 @@ from django.utils.translation import gettext_lazy as _
 from background_task.signals import task_failed
 from background_task.models import Task
 from common.logger import log
-from .models import Source, Media, MediaServer, Metadata
+from .models import Source, Media, Metadata
 from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task,
                     download_media_thumbnail, download_media_metadata,
                     map_task_to_instance, check_source_directory_exists,
-                    download_media, rescan_media_server, download_source_images,
+                    download_media, download_source_images,
                     delete_all_media_for_source, save_all_media_for_source,
                     rename_media, get_media_metadata_task, get_media_download_task)
 from .utils import delete_file, glob_quote, mkdir_p
@@ -270,7 +270,7 @@ def media_post_save(sender, instance, created, **kwargs):
     if not (media_file_exists or existing_media_download_task):
         # The file was deleted after it was downloaded, skip this media.
         if instance.can_download and instance.downloaded:
-            skip_changed = True != instance.skip
+            skip_changed = True if not instance.skip else False
             instance.skip = True
         downloaded = False
     if (instance.source.download_media and instance.can_download) and not (
@@ -374,13 +374,13 @@ def media_post_delete(sender, instance, **kwargs):
                         try:
                             p.rmdir()
                             log.info(f'Deleted directory for: {instance} path: {p!s}')
-                        except OSError as e:
+                        except OSError:
                             pass
                 # Delete the directory itself
                 try:
                     other_path.rmdir()
                     log.info(f'Deleted directory for: {instance} path: {other_path!s}')
-                except OSError as e:
+                except OSError:
                     pass
         # Get all files that start with the bare file path
         all_related_files = video_path.parent.glob(f'{glob_quote(video_path.with_suffix("").name)}*')

tubesync/sync/tasks.py

@@ -6,7 +6,6 @@
 
 import os
 import json
-import math
 import random
 import requests
 import time
@@ -14,9 +13,8 @@ import uuid
 from io import BytesIO
 from hashlib import sha1
 from pathlib import Path
-from datetime import datetime, timedelta
+from datetime import timedelta
 from shutil import copyfile, rmtree
-from PIL import Image
 from django import db
 from django.conf import settings
 from django.core.files.base import ContentFile
@@ -30,13 +28,13 @@ from background_task.exceptions import InvalidTaskError
 from background_task.models import Task, CompletedTask
 from common.logger import log
 from common.errors import ( NoFormatException, NoMediaException,
-                            NoMetadataException, NoThumbnailException,
+                            NoThumbnailException,
                             DownloadFailedException, )
 from common.utils import (  django_queryset_generator as qs_gen,
                             remove_enclosed, )
 from .choices import Val, TaskQueue
 from .models import Source, Media, MediaServer
-from .utils import ( get_remote_image, resize_image_to_height, delete_file,
+from .utils import ( get_remote_image, resize_image_to_height,
                     write_text_file, filter_response, )
 from .youtube import YouTubeError
 
@@ -122,7 +120,7 @@ def get_error_message(task):
 def update_task_status(task, status):
     if not task:
         return False
-    if not task._verbose_name:
+    if not hasattr(task, '_verbose_name'):
         task._verbose_name = remove_enclosed(
             task.verbose_name, '[', ']', ' ',
         )
@@ -226,7 +224,7 @@ def save_model(instance):
 @atomic(durable=False)
 def schedule_media_servers_update():
     # Schedule a task to update media servers
-    log.info(f'Scheduling media server updates')
+    log.info('Scheduling media server updates')
     verbose_name = _('Request media server rescan for "{}"')
     for mediaserver in MediaServer.objects.all():
         rescan_media_server(
@@ -235,6 +233,35 @@ def schedule_media_servers_update():
         )
 
 
+def wait_for_errors(model, /, *, task_name=None):
+    if task_name is None:
+        task_name=tuple((
+            'sync.tasks.download_media',
+            'sync.tasks.download_media_metadata',
+        ))
+    elif isinstance(task_name, str):
+        task_name = tuple((task_name,))
+    tasks = list()
+    for tn in task_name:
+        ft = get_first_task(tn, instance=model)
+        if ft:
+            tasks.append(ft)
+    window = timezone.timedelta(hours=3) + timezone.now()
+    tqs = Task.objects.filter(
+        task_name__in=task_name,
+        attempts__gt=0,
+        locked_at__isnull=True,
+        run_at__lte=window,
+        last_error__contains='HTTPError 429: Too Many Requests',
+    )
+    for task in tasks:
+        update_task_status(task, 'paused (429)')
+    log.info(f'waiting for errors: 429 ({tqs.count()}): {model}')
+    time.sleep(10 * tqs.count())
+    for task in tasks:
+        update_task_status(task, None)
+
+
 def cleanup_old_media():
     with atomic():
         for source in qs_gen(Source.objects.filter(delete_old_media=True, days_to_keep__gt=0)):
@@ -255,7 +282,7 @@ def cleanup_old_media():
     schedule_media_servers_update()
 
 
-def cleanup_removed_media(source, videos):
+def cleanup_removed_media(source, video_keys):
     if not source.delete_removed_media:
         return
     log.info(f'Cleaning up media no longer in source: {source}')
@@ -265,8 +292,7 @@ def cleanup_removed_media(source, videos):
         source=source,
     )
     for media in qs_gen(mqs):
-        matching_source_item = [video['id'] for video in videos if video['id'] == media.key]
+        if media.key not in video_keys:
-        if not matching_source_item:
             log.info(f'{media.name} is no longer in source, removing')
             with atomic():
                 media.delete()
@@ -316,12 +342,17 @@ def index_source_task(source_id):
             end=task.verbose_name.find('Index'),
         )
     tvn_format = '{:,}' + f'/{num_videos:,}'
-    for vn, video in enumerate(videos, start=1):
+    vn = 0
+    video_keys = set()
+    while len(videos) > 0:
+        vn += 1
+        video = videos.popleft()
         # Create or update each video as a Media object
         key = video.get(source.key_field, None)
         if not key:
             # Video has no unique key (ID), it can't be indexed
             continue
+        video_keys.add(key)
         update_task_status(task, tvn_format.format(vn))
         # media, new_media = Media.objects.get_or_create(key=key, source=source)
         try:
@@ -376,7 +407,7 @@ def index_source_task(source_id):
     # Reset task.verbose_name to the saved value
     update_task_status(task, None)
     # Cleanup of media no longer available from the source
-    cleanup_removed_media(source, videos)
+    cleanup_removed_media(source, video_keys)
     videos = video = None
 
 
@@ -416,7 +447,7 @@ def download_source_images(source_id):
     log.info(f'Thumbnail URL for source with ID: {source_id} / {source} '
         f'Avatar: {avatar} '
         f'Banner: {banner}')
-    if banner != None:
+    if banner is not None:
         url = banner
         i = get_remote_image(url)
         image_file = BytesIO()
@@ -432,7 +463,7 @@ def download_source_images(source_id):
                 f.write(django_file.read())
         i = image_file = None
 
-    if avatar != None:
+    if avatar is not None:
         url = avatar
         i = get_remote_image(url)
         image_file = BytesIO()
@@ -467,6 +498,7 @@ def download_media_metadata(media_id):
         log.info(f'Task for ID: {media_id} / {media} skipped, due to task being manually skipped.')
         return
     source = media.source
+    wait_for_errors(media, task_name='sync.tasks.download_media_metadata')
     try:
         metadata = media.index_metadata()
     except YouTubeError as e:
@@ -604,7 +636,7 @@ def download_media_thumbnail(media_id, url):
 
 
 @background(schedule=dict(priority=30, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True)
-def download_media(media_id):
+def download_media(media_id, override=False):
     '''
         Downloads the media to disk and attaches it to the Media instance.
     '''
@@ -613,41 +645,13 @@ def download_media(media_id):
     except Media.DoesNotExist as e:
         # Task triggered but the media no longer exists, do nothing
         raise InvalidTaskError(_('no such media')) from e
-    if not media.source.download_media:
+    else:
-        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
+        if not media.download_checklist(override):
-                 f'the source {media.source} has since been marked to not download, '
+            # any condition that needs to reschedule the task
-                 f'not downloading')
+            # should raise an exception to avoid this
-        return
-    if media.skip or media.manual_skip:
-        # Media was toggled to be skipped after the task was scheduled
-        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
-                 f'it is now marked to be skipped, not downloading')
-        return
-    # metadata is required to generate the proper filepath
-    if not media.has_metadata:
-        raise NoMetadataException('Metadata is not yet available.')
-    downloaded_file_exists = (
-        media.downloaded and
-        media.has_metadata and
-        (
-            media.media_file_exists or
-            media.filepath.exists()
-        )
-    )
-    if downloaded_file_exists:
-        # Media has been marked as downloaded before the download_media task was fired,
-        # skip it
-        log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
-                 f'it has already been marked as downloaded, not downloading again')
-        return
-    max_cap_age = media.source.download_cap_date
-    published = media.published
-    if max_cap_age and published:
-        if published <= max_cap_age:
-            log.warn(f'Download task triggered media: {media} (UUID: {media.pk}) but '
-                     f'the source has a download cap and the media is now too old, '
-                     f'not downloading')
             return
+
+    wait_for_errors(media, task_name='sync.tasks.download_media')
     filepath = media.filepath
     container = format_str = None
     log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"')
@@ -680,42 +684,7 @@ def download_media(media_id):
             raise DownloadFailedException(err)
 
         # Media has been downloaded successfully
-        log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: '
+        media.download_finished(format_str, container, filepath)
-                 f'"{filepath}"')
-        # Link the media file to the object and update info about the download
-        media.media_file.name = str(media.source.type_directory_path / media.filename)
-        media.downloaded = True
-        media.download_date = timezone.now()
-        media.downloaded_filesize = os.path.getsize(filepath)
-        media.downloaded_container = container
-        if '+' in format_str:
-            # Seperate audio and video streams
-            vformat_code, aformat_code = format_str.split('+')
-            aformat = media.get_format_by_code(aformat_code)
-            vformat = media.get_format_by_code(vformat_code)
-            media.downloaded_format = vformat['format']
-            media.downloaded_height = vformat['height']
-            media.downloaded_width = vformat['width']
-            media.downloaded_audio_codec = aformat['acodec']
-            media.downloaded_video_codec = vformat['vcodec']
-            media.downloaded_container = container
-            media.downloaded_fps = vformat['fps']
-            media.downloaded_hdr = vformat['is_hdr']
-        else:
-            # Combined stream or audio-only stream
-            cformat_code = format_str
-            cformat = media.get_format_by_code(cformat_code)
-            media.downloaded_audio_codec = cformat['acodec']
-            if cformat['vcodec']:
-                # Combined
-                media.downloaded_format = cformat['format']
-                media.downloaded_height = cformat['height']
-                media.downloaded_width = cformat['width']
-                media.downloaded_video_codec = cformat['vcodec']
-                media.downloaded_fps = cformat['fps']
-                media.downloaded_hdr = cformat['is_hdr']
-            else:
-                media.downloaded_format = 'audio'
         save_model(media)
         # If selected, copy the thumbnail over as well
         if media.source.copy_thumbnails:
@@ -887,26 +856,18 @@ def rename_all_media_for_source(source_id):
 
 @background(schedule=dict(priority=0, run_at=60), queue=Val(TaskQueue.DB), remove_existing_tasks=True)
 def wait_for_media_premiere(media_id):
-    hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60)))
-
     try:
         media = Media.objects.get(pk=media_id)
     except Media.DoesNotExist as e:
         raise InvalidTaskError(_('no such media')) from e
     else:
-        if media.has_metadata:
+        valid, hours = media.wait_for_premiere()
+        if not valid:
             return
-        now = timezone.now()
+        
-        if media.published < now:
+        if hours:
-            media.manual_skip = False
-            media.skip = False
-            # the download tasks start after the media is saved
-        else:
-            media.manual_skip = True
-            media.title = _(f'Premieres in {hours(media.published - now)} hours')
             task = get_media_premiere_task(media_id)
-            if task:
+            update_task_status(task, f'available in {hours} hours')
-                update_task_status(task, f'available in {hours(media.published - now)} hours')
         save_model(media)
 
 
@@ -918,7 +879,7 @@ def delete_all_media_for_source(source_id, source_name, source_directory):
     assert source_directory
     try:
         source = Source.objects.get(pk=source_id)
-    except Source.DoesNotExist as e:
+    except Source.DoesNotExist:
         # Task triggered but the source no longer exists, do nothing
         log.warn(f'Task delete_all_media_for_source(pk={source_id}) called but no '
                   f'source exists with ID: {source_id}')

tubesync/sync/tests.py

@@ -6,7 +6,6 @@
 
 
 import logging
-import os
 from datetime import datetime, timedelta
 from pathlib import Path
 from urllib.parse import urlsplit
@@ -1822,13 +1821,13 @@ class TasksTestCase(TestCase):
 
         now = timezone.now()
 
-        m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5))
+        m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5)) # noqa
-        m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25))
+        m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25)) # noqa
-        m13 = Media.objects.create(source=src1, downloaded=False, key='a13')
+        m13 = Media.objects.create(source=src1, downloaded=False, key='a13') # noqa
 
-        m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5))
+        m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5)) # noqa
         m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25))
-        m23 = Media.objects.create(source=src2, downloaded=False, key='a23')
+        m23 = Media.objects.create(source=src2, downloaded=False, key='a23') # noqa
 
         self.assertEqual(src1.media_source.all().count(), 3)
         self.assertEqual(src2.media_source.all().count(), 3)

tubesync/sync/views.py

@@ -35,7 +35,7 @@ from .tasks import (map_task_to_instance, get_error_message,
 from .choices import (Val, MediaServerType, SourceResolution, IndexSchedule,
                         YouTube_SourceType, youtube_long_source_types,
                         youtube_help, youtube_validation_urls)
-from . import signals
+from . import signals # noqa
 from . import youtube
 
 
@@ -258,7 +258,7 @@ class ValidateSourceView(FormView):
                 self.key = youtube.get_channel_id(
                     index_url.replace('/channel/', '/')
                 )
-            except youtube.YouTubeError as e:
+            except youtube.YouTubeError:
                 # It did not work, revert to previous behavior
                 self.key = old_key
                 self.source_type = old_source_type
@@ -296,10 +296,13 @@ class EditSourceMixin:
     def form_valid(self, form: Form):
         # Perform extra validation to make sure the media_format is valid
         obj = form.save(commit=False)
-        source_type = form.cleaned_data['media_format']
+        # temporarily use media_format from the form
+        saved_media_format = obj.media_format
+        obj.media_format = form.cleaned_data['media_format']
         example_media_file = obj.get_example_media_format()
+        obj.media_format = saved_media_format
 
-        if example_media_file == '':
+        if '' == example_media_file:
             form.add_error(
                 'media_format',
                 ValidationError(self.errors['invalid_media_format'])
@@ -307,12 +310,16 @@ class EditSourceMixin:
 
         # Check for suspicious file path(s)
         try:
-            targetCheck = form.cleaned_data['directory']+"/.virt"
+            targetCheck = form.cleaned_data['directory'] + '/.virt'
-            newdir = safe_join(settings.DOWNLOAD_ROOT,targetCheck)
+            safe_join(settings.DOWNLOAD_ROOT, targetCheck)
         except SuspiciousFileOperation:
             form.add_error(
                 'directory',
-                ValidationError(self.errors['dir_outside_dlroot'].replace("%BASEDIR%",str(settings.DOWNLOAD_ROOT)))
+                ValidationError(
+                    self.errors['dir_outside_dlroot'].replace(
+                        "%BASEDIR%", str(settings.DOWNLOAD_ROOT)
+                    )
+                ),
             )
 
         if form.errors:

tubesync/sync/youtube.py

@@ -6,7 +6,6 @@
 
 import os
 
-from collections import namedtuple
 from common.logger import log
 from copy import deepcopy
 from pathlib import Path
@@ -102,7 +101,7 @@ def get_channel_image_info(url):
                     avatar_url = thumbnail['url']
                 if thumbnail['id'] == 'banner_uncropped':
                     banner_url = thumbnail['url']
-                if banner_url != None and avatar_url != None:
+                if banner_url is not None and avatar_url is not None:
                     break
 
             return avatar_url, banner_url
@@ -143,7 +142,7 @@ def get_media_info(url, /, *, days=None, info_json=None):
     if days is not None:
         try:
             days = int(str(days), 10)
-        except Exception as e:
+        except (TypeError, ValueError):
             days = None
         start = (
             f'yesterday-{days!s}days' if days else None
@@ -326,13 +325,10 @@ def download_media(
             final_path = Path(output_file).resolve(strict=False)
         expected_file = shell_quote(str(final_path))
         cmds = pp_opts.exec_cmd.get('after_move', list())
-        # It is important that we use a tuple for strings.
+        cmds.append(
-        # Otherwise, list adds each character instead.
-        # That last comma is really necessary!
-        cmds += (
             f'test -f {expected_file} || '
             'mv -T -u -- %(filepath,_filename|)q '
-            f'{expected_file}',
+            f'{expected_file}'
         )
         # assignment is the quickest way to cover both 'get' cases
         pp_opts.exec_cmd['after_move'] = cmds
@@ -387,7 +383,7 @@ def download_media(
     youtube_ea_dict = ytopts['extractor_args'].get('youtube', dict())
     formats_list = youtube_ea_dict.get('formats', list())
     if 'missing_pot' not in formats_list:
-        formats_list += ('missing_pot',)
+        formats_list.append('missing_pot')
         youtube_ea_dict.update({
             'formats': formats_list,
         })

tubesync/tubesync/settings.py

@@ -8,7 +8,7 @@ CONFIG_BASE_DIR = BASE_DIR
 DOWNLOADS_BASE_DIR = BASE_DIR
 
 
-VERSION = '0.15.2'
+VERSION = '0.15.3'
 SECRET_KEY = ''
 DEBUG = False
 ALLOWED_HOSTS = []
@@ -198,7 +198,7 @@ RENAME_SOURCES = None
 # You have been warned!
 
 try:
-    from .local_settings import *
+    from .local_settings import * # noqa
 except ImportError as e:
     import sys
     sys.stderr.write(f'Unable to import local_settings: {e}\n')
@@ -222,5 +222,5 @@ if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS:
     BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS
 
 
-from .dbutils import patch_ensure_connection
+from .dbutils import patch_ensure_connection # noqa
 patch_ensure_connection()