diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 22bd31b2..5ae58c0a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -144,7 +144,7 @@ jobs:
done
} >> "${GITHUB_STEP_SUMMARY}"
uvx --no-config --no-managed-python --no-progress --isolated \
- ruff check --exit-zero \
+ ruff check \
--target-version "${target_version}" \
--output-format github \
--ignore "${ignore_csv_list}"
diff --git a/Dockerfile b/Dockerfile
index 6ef178c5..c878890e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -410,6 +410,12 @@ RUN --mount=type=tmpfs,target=/cache \
PIPENV_VERBOSITY=64 \
PYTHONPYCACHEPREFIX=/cache/pycache \
pipenv install --system --skip-lock && \
+ # remove the getpot_bgutil_script plugin
+ find /usr/local/lib \
+ -name 'getpot_bgutil_script.py' \
+ -path '*/yt_dlp_plugins/extractor/getpot_bgutil_script.py' \
+ -type f -print -delete \
+ && \
# Clean up
apt-get -y autoremove --purge \
default-libmysqlclient-dev \
diff --git a/Pipfile b/Pipfile
index 2e3f3acf..49b5127b 100644
--- a/Pipfile
+++ b/Pipfile
@@ -24,5 +24,4 @@ yt-dlp = {extras = ["default", "curl-cffi"], version = "*"}
emoji = "*"
brotli = "*"
html5lib = "*"
-yt-dlp-get-pot = "*"
bgutil-ytdlp-pot-provider = "*"
diff --git a/tubesync/common/json.py b/tubesync/common/json.py
index e8a22e1c..5a56a019 100644
--- a/tubesync/common/json.py
+++ b/tubesync/common/json.py
@@ -1,4 +1,6 @@
+from datetime import datetime
from django.core.serializers.json import DjangoJSONEncoder
+from yt_dlp.utils import LazyList
class JSONEncoder(DjangoJSONEncoder):
@@ -14,3 +16,11 @@ class JSONEncoder(DjangoJSONEncoder):
return list(iterable)
return super().default(obj)
+
+def json_serial(obj):
+ if isinstance(obj, datetime):
+ return obj.isoformat()
+ if isinstance(obj, LazyList):
+ return list(obj)
+ raise TypeError(f'Type {type(obj)} is not json_serial()-able')
+
diff --git a/tubesync/common/urls.py b/tubesync/common/urls.py
index 1f29056d..a3e00a84 100644
--- a/tubesync/common/urls.py
+++ b/tubesync/common/urls.py
@@ -1,7 +1,6 @@
from django.conf import settings
from django.urls import path
from django.views.generic.base import RedirectView
-from django.views.generic import TemplateView
from django.http import HttpResponse
from .views import error403, error404, error500, HealthCheckView
diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py
index c4798943..8f7afc2c 100644
--- a/tubesync/common/utils.py
+++ b/tubesync/common/utils.py
@@ -6,10 +6,8 @@ import os
import pstats
import string
import time
-from datetime import datetime
from django.core.paginator import Paginator
from urllib.parse import urlunsplit, urlencode, urlparse
-from yt_dlp.utils import LazyList
from .errors import DatabaseConnectionError
@@ -84,14 +82,11 @@ def parse_database_connection_string(database_connection_string):
f'invalid driver, must be one of {valid_drivers}')
django_driver = django_backends.get(driver)
host_parts = user_pass_host_port.split('@')
- if len(host_parts) != 2:
- raise DatabaseConnectionError(f'Database connection string netloc must be in '
- f'the format of user:pass@host')
+ user_pass_parts = host_parts[0].split(':')
+ if len(host_parts) != 2 or len(user_pass_parts) != 2:
+ raise DatabaseConnectionError('Database connection string netloc must be in '
+ 'the format of user:pass@host')
user_pass, host_port = host_parts
- user_pass_parts = user_pass.split(':')
- if len(user_pass_parts) != 2:
- raise DatabaseConnectionError(f'Database connection string netloc must be in '
- f'the format of user:pass@host')
username, password = user_pass_parts
host_port_parts = host_port.split(':')
if len(host_port_parts) == 1:
@@ -113,13 +108,13 @@ def parse_database_connection_string(database_connection_string):
f'65535, got {port}')
else:
# Malformed
- raise DatabaseConnectionError(f'Database connection host must be a hostname or '
- f'a hostname:port combination')
+ raise DatabaseConnectionError('Database connection host must be a hostname or '
+ 'a hostname:port combination')
if database.startswith('/'):
database = database[1:]
if not database:
- raise DatabaseConnectionError(f'Database connection string path must be a '
- f'string in the format of /databasename')
+ raise DatabaseConnectionError('Database connection string path must be a '
+ 'string in the format of /databasename')
if '/' in database:
raise DatabaseConnectionError(f'Database connection string path can only '
f'contain a single string name, got: {database}')
@@ -172,14 +167,6 @@ def clean_emoji(s):
return emoji.replace_emoji(s)
-def json_serial(obj):
- if isinstance(obj, datetime):
- return obj.isoformat()
- if isinstance(obj, LazyList):
- return list(obj)
- raise TypeError(f'Type {type(obj)} is not json_serial()-able')
-
-
def time_func(func):
def wrapper(*args, **kwargs):
start = time.perf_counter()
diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py
index 452fee77..e489ff08 100644
--- a/tubesync/sync/fields.py
+++ b/tubesync/sync/fields.py
@@ -63,7 +63,7 @@ class CommaSepChoiceField(models.CharField):
def __init__(self, *args, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, **kwargs):
kwargs.setdefault('max_length', 128)
self.separator = str(separator)
- self.possible_choices = possible_choices or choices
+ self.possible_choices = possible_choices or kwargs.get('choices')
self.selected_choices = list()
self.allow_all = allow_all
self.all_label = all_label
diff --git a/tubesync/sync/filtering.py b/tubesync/sync/filtering.py
index 3f7023c1..ef206696 100644
--- a/tubesync/sync/filtering.py
+++ b/tubesync/sync/filtering.py
@@ -5,7 +5,6 @@
from common.logger import log
from .models import Media
from datetime import datetime
-from django.utils import timezone
from .overrides.custom_filter import filter_custom
diff --git a/tubesync/sync/hooks.py b/tubesync/sync/hooks.py
index 3bb3ce0d..467e2df1 100644
--- a/tubesync/sync/hooks.py
+++ b/tubesync/sync/hooks.py
@@ -1,9 +1,7 @@
import os
-import yt_dlp
from common.logger import log
from common.utils import remove_enclosed
-from django.conf import settings
progress_hook = {
diff --git a/tubesync/sync/management/commands/delete-source.py b/tubesync/sync/management/commands/delete-source.py
index 42f9d5ac..d9f8f204 100644
--- a/tubesync/sync/management/commands/delete-source.py
+++ b/tubesync/sync/management/commands/delete-source.py
@@ -1,16 +1,15 @@
-import os
import uuid
-from django.utils.translation import gettext_lazy as _
from django.core.management.base import BaseCommand, CommandError
from django.db.transaction import atomic
+from django.utils.translation import gettext_lazy as _
from common.logger import log
-from sync.models import Source, Media, MediaServer
+from sync.models import Source
from sync.tasks import schedule_media_servers_update
class Command(BaseCommand):
- help = _('Deletes a source by UUID')
+ help = 'Deletes a source by UUID'
def add_arguments(self, parser):
parser.add_argument('--source', action='store', required=True, help=_('Source UUID'))
diff --git a/tubesync/sync/management/commands/import-existing-media.py b/tubesync/sync/management/commands/import-existing-media.py
index 3813b497..c05c630a 100644
--- a/tubesync/sync/management/commands/import-existing-media.py
+++ b/tubesync/sync/management/commands/import-existing-media.py
@@ -1,6 +1,6 @@
import os
from pathlib import Path
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError # noqa
from common.logger import log
from common.timestamp import timestamp_to_datetime
from sync.choices import FileExtension
@@ -18,7 +18,7 @@ class Command(BaseCommand):
dirmap = {}
for s in Source.objects.all():
dirmap[str(s.directory_path)] = s
- log.info(f'Scanning sources...')
+ log.info('Scanning sources...')
file_extensions = list(FileExtension.values) + self.extra_extensions
for sourceroot, source in dirmap.items():
media = list(Media.objects.filter(source=source, downloaded=False,
diff --git a/tubesync/sync/management/commands/list-sources.py b/tubesync/sync/management/commands/list-sources.py
index 4ee177ae..25eae481 100644
--- a/tubesync/sync/management/commands/list-sources.py
+++ b/tubesync/sync/management/commands/list-sources.py
@@ -1,7 +1,6 @@
-import os
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError # noqa
from common.logger import log
-from sync.models import Source, Media, MediaServer
+from sync.models import Source
class Command(BaseCommand):
diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py
index 55436863..318a5b27 100644
--- a/tubesync/sync/management/commands/reset-tasks.py
+++ b/tubesync/sync/management/commands/reset-tasks.py
@@ -1,14 +1,12 @@
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError # noqa
from django.db.transaction import atomic
from django.utils.translation import gettext_lazy as _
from background_task.models import Task
+from common.logger import log
from sync.models import Source
from sync.tasks import index_source_task, check_source_directory_exists
-from common.logger import log
-
-
class Command(BaseCommand):
help = 'Resets all tasks'
@@ -31,10 +29,10 @@ class Command(BaseCommand):
index_source_task(
str(source.pk),
repeat=source.index_schedule,
+ schedule=source.index_schedule,
verbose_name=verbose_name.format(source.name),
)
- with atomic(durable=True):
- for source in Source.objects.all():
# This also chains down to call each Media objects .save() as well
source.save()
+
log.info('Done')
diff --git a/tubesync/sync/management/commands/sync-missing-metadata.py b/tubesync/sync/management/commands/sync-missing-metadata.py
index 21b25c52..863f83f9 100644
--- a/tubesync/sync/management/commands/sync-missing-metadata.py
+++ b/tubesync/sync/management/commands/sync-missing-metadata.py
@@ -1,6 +1,5 @@
-import os
from shutil import copyfile
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError # noqa
from django.db.models import Q
from common.logger import log
from sync.models import Source, Media
diff --git a/tubesync/sync/management/commands/youtube-dl-info.py b/tubesync/sync/management/commands/youtube-dl-info.py
index 32a47402..c8099765 100644
--- a/tubesync/sync/management/commands/youtube-dl-info.py
+++ b/tubesync/sync/management/commands/youtube-dl-info.py
@@ -1,7 +1,7 @@
import json
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError # noqa
from sync.youtube import get_media_info
-from common.utils import json_serial
+from common.json import JSONEncoder
class Command(BaseCommand):
@@ -15,6 +15,6 @@ class Command(BaseCommand):
url = options['url']
self.stdout.write(f'Showing information for URL: {url}')
info = get_media_info(url)
- d = json.dumps(info, indent=4, sort_keys=True, default=json_serial)
+ d = json.dumps(info, indent=4, sort_keys=True, cls=JSONEncoder)
self.stdout.write(d)
self.stdout.write('Done')
diff --git a/tubesync/sync/mediaservers.py b/tubesync/sync/mediaservers.py
index ceab239f..e141238d 100644
--- a/tubesync/sync/mediaservers.py
+++ b/tubesync/sync/mediaservers.py
@@ -117,9 +117,6 @@ class PlexMediaServer(MediaServer):
raise ValidationError('Plex Media Server "port" must be between 1 '
'and 65535')
options = self.object.options
- if 'token' not in options:
- raise ValidationError('Plex Media Server requires a "token"')
- token = options['token'].strip()
if 'token' not in options:
raise ValidationError('Plex Media Server requires a "token"')
if 'libraries' not in options:
diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654.py b/tubesync/sync/migrations/0011_auto_20220201_1654.py
index 96d9f4a7..51641ece 100644
--- a/tubesync/sync/migrations/0011_auto_20220201_1654.py
+++ b/tubesync/sync/migrations/0011_auto_20220201_1654.py
@@ -1,8 +1,6 @@
# Generated by Django 3.2.11 on 2022-02-01 16:54
-import django.core.files.storage
from django.db import migrations, models
-import sync.models
class Migration(migrations.Migration):
diff --git a/tubesync/sync/migrations/0013_fix_elative_media_file.py b/tubesync/sync/migrations/0013_fix_elative_media_file.py
index c9eee22e..2f1ac385 100644
--- a/tubesync/sync/migrations/0013_fix_elative_media_file.py
+++ b/tubesync/sync/migrations/0013_fix_elative_media_file.py
@@ -1,7 +1,7 @@
# Generated by Django 3.2.12 on 2022-04-06 06:19
from django.conf import settings
-from django.db import migrations, models
+from django.db import migrations
def fix_media_file(apps, schema_editor):
diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py
index d7ed077c..a850a4e0 100644
--- a/tubesync/sync/models/__init__.py
+++ b/tubesync/sync/models/__init__.py
@@ -17,3 +17,9 @@ from .media import Media
from .metadata import Metadata
from .metadata_format import MetadataFormat
+__all__ = [
+ 'get_media_file_path', 'get_media_thumb_path',
+ 'media_file_storage', 'MediaServer', 'Source',
+ 'Media', 'Metadata', 'MetadataFormat',
+]
+
diff --git a/tubesync/sync/models/_private.py b/tubesync/sync/models/_private.py
index 96539dbe..8cf41ce1 100644
--- a/tubesync/sync/models/_private.py
+++ b/tubesync/sync/models/_private.py
@@ -1,4 +1,5 @@
-from ..choices import Val, YouTube_SourceType
+from pathlib import Path
+from ..choices import Val, YouTube_SourceType # noqa
_srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) ))
@@ -10,3 +11,11 @@ def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2
element.tail = tail + (char * indent)
return element
+def directory_and_stem(arg_path, /, all_suffixes=False):
+ filepath = Path(arg_path)
+ stem = Path(filepath.stem)
+ while all_suffixes and stem.suffixes and '' != stem.suffix:
+ stem = Path(stem.stem)
+ stem = str(stem)
+ return (filepath.parent, stem,)
+
diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py
index 6eb0ed76..62f73d5d 100644
--- a/tubesync/sync/models/media.py
+++ b/tubesync/sync/models/media.py
@@ -15,9 +15,9 @@ from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from common.logger import log
from common.errors import NoFormatException
+from common.json import JSONEncoder
from common.utils import (
clean_filename, clean_emoji,
- django_queryset_generator as qs_gen,
)
from ..youtube import (
get_media_info as get_youtube_media_info,
@@ -25,8 +25,7 @@ from ..youtube import (
)
from ..utils import (
seconds_to_timestr, parse_media_format, filter_response,
- write_text_file, mkdir_p, directory_and_stem, glob_quote,
- multi_key_sort,
+ write_text_file, mkdir_p, glob_quote, multi_key_sort,
)
from ..matching import (
get_best_combined_format,
@@ -39,7 +38,7 @@ from ..choices import (
from ._migrations import (
media_file_storage, get_media_thumb_path, get_media_file_path,
)
-from ._private import _srctype_dict, _nfo_element
+from ._private import _srctype_dict, _nfo_element, directory_and_stem
from .media__tasks import (
download_checklist, download_finished, wait_for_premiere,
)
@@ -578,14 +577,13 @@ class Media(models.Model):
def metadata_dumps(self, arg_dict=dict()):
- from common.utils import json_serial
fallback = dict()
try:
fallback.update(self.new_metadata.with_formats)
except ObjectDoesNotExist:
pass
data = arg_dict or fallback
- return json.dumps(data, separators=(',', ':'), default=json_serial)
+ return json.dumps(data, separators=(',', ':'), cls=JSONEncoder)
def metadata_loads(self, arg_str='{}'):
@@ -688,7 +686,7 @@ class Media(models.Model):
pass
setattr(self, '_cached_metadata_dict', data)
return data
- except Exception as e:
+ except Exception:
return {}
@@ -1194,7 +1192,7 @@ class Media(models.Model):
other_path.replace(new_file_path)
for fuzzy_path in fuzzy_paths:
- (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path)
+ (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path, True)
old_file_str = fuzzy_path.name
new_file_str = new_stem + old_file_str[len(fuzzy_stem):]
new_file_path = Path(new_prefix_path / new_file_str)
@@ -1219,7 +1217,7 @@ class Media(models.Model):
parent_dir.rmdir()
log.info(f'Removed empty directory: {parent_dir!s}')
parent_dir = parent_dir.parent
- except OSError as e:
+ except OSError:
pass
diff --git a/tubesync/sync/models/media__tasks.py b/tubesync/sync/models/media__tasks.py
index d63f7b31..e8b33235 100644
--- a/tubesync/sync/models/media__tasks.py
+++ b/tubesync/sync/models/media__tasks.py
@@ -1,9 +1,11 @@
import os
+from pathlib import Path
from common.logger import log
from common.errors import (
NoMetadataException,
)
from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
from ..choices import Val, SourceResolution
@@ -47,6 +49,7 @@ def download_checklist(self, skip_checks=False):
f'the source has a download cap and the media is now too old, '
f'not downloading')
return False
+ return True
def download_finished(self, format_str, container, downloaded_filepath=None):
diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py
index 74f75278..07ade020 100644
--- a/tubesync/sync/models/source.py
+++ b/tubesync/sync/models/source.py
@@ -1,6 +1,7 @@
import os
import re
import uuid
+from collections import deque as queue
from pathlib import Path
from django import db
from django.conf import settings
@@ -510,7 +511,7 @@ class Source(db.models.Model):
def get_example_media_format(self):
try:
return self.media_format.format(**self.example_media_format_dict)
- except Exception as e:
+ except Exception:
return ''
def is_regex_match(self, media_item_title):
@@ -527,23 +528,31 @@ class Source(db.models.Model):
days = timezone.timedelta(seconds=self.download_cap).days
response = indexer(self.get_index_url(type=type), days=days)
if not isinstance(response, dict):
- return []
- entries = response.get('entries', [])
+ return list()
+ entries = response.get('entries', list())
return entries
def index_media(self):
'''
- Index the media source returning a list of media metadata as dicts.
+ Index the media source returning a queue of media metadata as dicts.
'''
- entries = list()
+ entries = queue(list(), getattr(settings, 'MAX_ENTRIES_PROCESSING', 0) or None)
if self.index_videos:
- entries += self.get_index('videos')
+ entries.extend(reversed(self.get_index('videos')))
+
# Playlists do something different that I have yet to figure out
if not self.is_playlist:
if self.index_streams:
- entries += self.get_index('streams')
+ streams = self.get_index('streams')
+ if entries.maxlen is None or 0 == len(entries):
+ entries.extend(reversed(streams))
+ else:
+ # share the queue between streams and videos
+ allowed_streams = max(
+ entries.maxlen // 2,
+ entries.maxlen - len(entries),
+ )
+ entries.extend(reversed(streams[: allowed_streams]))
- if settings.MAX_ENTRIES_PROCESSING:
- entries = entries[:settings.MAX_ENTRIES_PROCESSING]
return entries
diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py
index 790ce1c2..69254146 100644
--- a/tubesync/sync/signals.py
+++ b/tubesync/sync/signals.py
@@ -10,11 +10,11 @@ from django.utils.translation import gettext_lazy as _
from background_task.signals import task_failed
from background_task.models import Task
from common.logger import log
-from .models import Source, Media, MediaServer, Metadata
+from .models import Source, Media, Metadata
from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task,
download_media_thumbnail, download_media_metadata,
map_task_to_instance, check_source_directory_exists,
- download_media, rescan_media_server, download_source_images,
+ download_media, download_source_images,
delete_all_media_for_source, save_all_media_for_source,
rename_media, get_media_metadata_task, get_media_download_task)
from .utils import delete_file, glob_quote, mkdir_p
@@ -270,7 +270,7 @@ def media_post_save(sender, instance, created, **kwargs):
if not (media_file_exists or existing_media_download_task):
# The file was deleted after it was downloaded, skip this media.
if instance.can_download and instance.downloaded:
- skip_changed = True != instance.skip
+ skip_changed = True if not instance.skip else False
instance.skip = True
downloaded = False
if (instance.source.download_media and instance.can_download) and not (
@@ -374,13 +374,13 @@ def media_post_delete(sender, instance, **kwargs):
try:
p.rmdir()
log.info(f'Deleted directory for: {instance} path: {p!s}')
- except OSError as e:
+ except OSError:
pass
# Delete the directory itself
try:
other_path.rmdir()
log.info(f'Deleted directory for: {instance} path: {other_path!s}')
- except OSError as e:
+ except OSError:
pass
# Get all files that start with the bare file path
all_related_files = video_path.parent.glob(f'{glob_quote(video_path.with_suffix("").name)}*')
diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py
index 0467a4fd..bf5e43ed 100644
--- a/tubesync/sync/tasks.py
+++ b/tubesync/sync/tasks.py
@@ -6,7 +6,6 @@
import os
import json
-import math
import random
import requests
import time
@@ -14,9 +13,8 @@ import uuid
from io import BytesIO
from hashlib import sha1
from pathlib import Path
-from datetime import datetime, timedelta
+from datetime import timedelta
from shutil import copyfile, rmtree
-from PIL import Image
from django import db
from django.conf import settings
from django.core.files.base import ContentFile
@@ -30,14 +28,14 @@ from background_task.exceptions import InvalidTaskError
from background_task.models import Task, CompletedTask
from common.logger import log
from common.errors import ( NoFormatException, NoMediaException,
- NoMetadataException, NoThumbnailException,
+ NoThumbnailException,
DownloadFailedException, )
from common.utils import ( django_queryset_generator as qs_gen,
remove_enclosed, )
from .choices import Val, TaskQueue
from .models import Source, Media, MediaServer
-from .utils import ( get_remote_image, resize_image_to_height, delete_file,
- write_text_file, filter_response, )
+from .utils import ( get_remote_image, resize_image_to_height,
+ write_text_file, filter_response, seconds_to_timestr, )
from .youtube import YouTubeError
db_vendor = db.connection.vendor
@@ -122,7 +120,7 @@ def get_error_message(task):
def update_task_status(task, status):
if not task:
return False
- if not task._verbose_name:
+ if not hasattr(task, '_verbose_name'):
task._verbose_name = remove_enclosed(
task.verbose_name, '[', ']', ' ',
)
@@ -226,7 +224,7 @@ def save_model(instance):
@atomic(durable=False)
def schedule_media_servers_update():
# Schedule a task to update media servers
- log.info(f'Scheduling media server updates')
+ log.info('Scheduling media server updates')
verbose_name = _('Request media server rescan for "{}"')
for mediaserver in MediaServer.objects.all():
rescan_media_server(
@@ -235,6 +233,38 @@ def schedule_media_servers_update():
)
+def wait_for_errors(model, /, *, task_name=None):
+ if task_name is None:
+ task_name=tuple((
+ 'sync.tasks.download_media',
+ 'sync.tasks.download_media_metadata',
+ ))
+ elif isinstance(task_name, str):
+ task_name = tuple((task_name,))
+ tasks = list()
+ for tn in task_name:
+ ft = get_first_task(tn, instance=model)
+ if ft:
+ tasks.append(ft)
+ window = timezone.timedelta(hours=3) + timezone.now()
+ tqs = Task.objects.filter(
+ task_name__in=task_name,
+ attempts__gt=0,
+ locked_at__isnull=True,
+ run_at__lte=window,
+ last_error__contains='HTTPError 429: Too Many Requests',
+ )
+ for task in tasks:
+ update_task_status(task, 'paused (429)')
+
+ delay = 10 * tqs.count()
+ time_str = seconds_to_timestr(delay)
+ log.info(f'waiting for errors: 429 ({time_str}): {model}')
+ time.sleep(delay)
+ for task in tasks:
+ update_task_status(task, None)
+
+
def cleanup_old_media():
with atomic():
for source in qs_gen(Source.objects.filter(delete_old_media=True, days_to_keep__gt=0)):
@@ -255,7 +285,7 @@ def cleanup_old_media():
schedule_media_servers_update()
-def cleanup_removed_media(source, videos):
+def cleanup_removed_media(source, video_keys):
if not source.delete_removed_media:
return
log.info(f'Cleaning up media no longer in source: {source}')
@@ -265,8 +295,7 @@ def cleanup_removed_media(source, videos):
source=source,
)
for media in qs_gen(mqs):
- matching_source_item = [video['id'] for video in videos if video['id'] == media.key]
- if not matching_source_item:
+ if media.key not in video_keys:
log.info(f'{media.name} is no longer in source, removing')
with atomic():
media.delete()
@@ -316,12 +345,17 @@ def index_source_task(source_id):
end=task.verbose_name.find('Index'),
)
tvn_format = '{:,}' + f'/{num_videos:,}'
- for vn, video in enumerate(videos, start=1):
+ vn = 0
+ video_keys = set()
+ while len(videos) > 0:
+ vn += 1
+ video = videos.popleft()
# Create or update each video as a Media object
key = video.get(source.key_field, None)
if not key:
# Video has no unique key (ID), it can't be indexed
continue
+ video_keys.add(key)
update_task_status(task, tvn_format.format(vn))
# media, new_media = Media.objects.get_or_create(key=key, source=source)
try:
@@ -376,7 +410,7 @@ def index_source_task(source_id):
# Reset task.verbose_name to the saved value
update_task_status(task, None)
# Cleanup of media no longer available from the source
- cleanup_removed_media(source, videos)
+ cleanup_removed_media(source, video_keys)
videos = video = None
@@ -416,7 +450,7 @@ def download_source_images(source_id):
log.info(f'Thumbnail URL for source with ID: {source_id} / {source} '
f'Avatar: {avatar} '
f'Banner: {banner}')
- if banner != None:
+ if banner is not None:
url = banner
i = get_remote_image(url)
image_file = BytesIO()
@@ -432,7 +466,7 @@ def download_source_images(source_id):
f.write(django_file.read())
i = image_file = None
- if avatar != None:
+ if avatar is not None:
url = avatar
i = get_remote_image(url)
image_file = BytesIO()
@@ -467,6 +501,7 @@ def download_media_metadata(media_id):
log.info(f'Task for ID: {media_id} / {media} skipped, due to task being manually skipped.')
return
source = media.source
+ wait_for_errors(media, task_name='sync.tasks.download_media_metadata')
try:
metadata = media.index_metadata()
except YouTubeError as e:
@@ -615,8 +650,11 @@ def download_media(media_id, override=False):
raise InvalidTaskError(_('no such media')) from e
else:
if not media.download_checklist(override):
+ # any condition that needs to reschedule the task
+ # should raise an exception to avoid this
return
+ wait_for_errors(media, task_name='sync.tasks.download_media')
filepath = media.filepath
container = format_str = None
log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"')
@@ -832,8 +870,7 @@ def wait_for_media_premiere(media_id):
if hours:
task = get_media_premiere_task(media_id)
- if task:
- update_task_status(task, f'available in {hours} hours')
+ update_task_status(task, f'available in {hours} hours')
save_model(media)
@@ -845,7 +882,7 @@ def delete_all_media_for_source(source_id, source_name, source_directory):
assert source_directory
try:
source = Source.objects.get(pk=source_id)
- except Source.DoesNotExist as e:
+ except Source.DoesNotExist:
# Task triggered but the source no longer exists, do nothing
log.warn(f'Task delete_all_media_for_source(pk={source_id}) called but no '
f'source exists with ID: {source_id}')
diff --git a/tubesync/sync/templates/sync/media-item.html b/tubesync/sync/templates/sync/media-item.html
index b70f78c2..074987ea 100644
--- a/tubesync/sync/templates/sync/media-item.html
+++ b/tubesync/sync/templates/sync/media-item.html
@@ -121,7 +121,8 @@
{% if media_file_path == media.filepath %}
(matched)
{% endif %}
-
+
+
| File size |
File size
{{ media.downloaded_filesize|bytesformat }} |
diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py
index 24f0d092..47089673 100644
--- a/tubesync/sync/tests.py
+++ b/tubesync/sync/tests.py
@@ -6,7 +6,6 @@
import logging
-import os
from datetime import datetime, timedelta
from pathlib import Path
from urllib.parse import urlsplit
@@ -1822,13 +1821,13 @@ class TasksTestCase(TestCase):
now = timezone.now()
- m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5))
- m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25))
- m13 = Media.objects.create(source=src1, downloaded=False, key='a13')
+ m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5)) # noqa
+ m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25)) # noqa
+ m13 = Media.objects.create(source=src1, downloaded=False, key='a13') # noqa
- m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5))
+ m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5)) # noqa
m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25))
- m23 = Media.objects.create(source=src2, downloaded=False, key='a23')
+ m23 = Media.objects.create(source=src2, downloaded=False, key='a23') # noqa
self.assertEqual(src1.media_source.all().count(), 3)
self.assertEqual(src2.media_source.all().count(), 3)
diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py
index 5bc90d25..fc7874fd 100644
--- a/tubesync/sync/utils.py
+++ b/tubesync/sync/utils.py
@@ -130,15 +130,6 @@ def file_is_editable(filepath):
return False
-def directory_and_stem(arg_path):
- filepath = Path(arg_path)
- stem = Path(filepath.stem)
- while stem.suffixes and '' != stem.suffix:
- stem = Path(stem.stem)
- stem = str(stem)
- return (filepath.parent, stem,)
-
-
def mkdir_p(arg_path, mode=0o777):
'''
Reminder: mode only affects the last directory
diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 10cfc5db..093cad3f 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -35,7 +35,7 @@ from .tasks import (map_task_to_instance, get_error_message,
from .choices import (Val, MediaServerType, SourceResolution, IndexSchedule,
YouTube_SourceType, youtube_long_source_types,
youtube_help, youtube_validation_urls)
-from . import signals
+from . import signals # noqa
from . import youtube
@@ -258,7 +258,7 @@ class ValidateSourceView(FormView):
self.key = youtube.get_channel_id(
index_url.replace('/channel/', '/')
)
- except youtube.YouTubeError as e:
+ except youtube.YouTubeError:
# It did not work, revert to previous behavior
self.key = old_key
self.source_type = old_source_type
@@ -296,10 +296,13 @@ class EditSourceMixin:
def form_valid(self, form: Form):
# Perform extra validation to make sure the media_format is valid
obj = form.save(commit=False)
- source_type = form.cleaned_data['media_format']
+ # temporarily use media_format from the form
+ saved_media_format = obj.media_format
+ obj.media_format = form.cleaned_data['media_format']
example_media_file = obj.get_example_media_format()
+ obj.media_format = saved_media_format
- if example_media_file == '':
+ if '' == example_media_file:
form.add_error(
'media_format',
ValidationError(self.errors['invalid_media_format'])
@@ -307,12 +310,16 @@ class EditSourceMixin:
# Check for suspicious file path(s)
try:
- targetCheck = form.cleaned_data['directory']+"/.virt"
- newdir = safe_join(settings.DOWNLOAD_ROOT,targetCheck)
+ targetCheck = form.cleaned_data['directory'] + '/.virt'
+ safe_join(settings.DOWNLOAD_ROOT, targetCheck)
except SuspiciousFileOperation:
form.add_error(
'directory',
- ValidationError(self.errors['dir_outside_dlroot'].replace("%BASEDIR%",str(settings.DOWNLOAD_ROOT)))
+ ValidationError(
+ self.errors['dir_outside_dlroot'].replace(
+ "%BASEDIR%", str(settings.DOWNLOAD_ROOT)
+ )
+ ),
)
if form.errors:
@@ -838,9 +845,11 @@ class TasksView(ListView):
data = super().get_context_data(*args, **kwargs)
now = timezone.now()
qs = Task.objects.all()
- errors_qs = qs.filter(attempts__gt=0, locked_by__isnull=True)
running_qs = qs.filter(locked_by__isnull=False)
scheduled_qs = qs.filter(locked_by__isnull=True)
+ errors_qs = scheduled_qs.filter(
+ attempts__gt=0
+ ).exclude(last_error__exact='')
# Add to context data from ListView
data['message'] = self.message
diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py
index 9814db24..7a2ce654 100644
--- a/tubesync/sync/youtube.py
+++ b/tubesync/sync/youtube.py
@@ -6,7 +6,6 @@
import os
-from collections import namedtuple
from common.logger import log
from copy import deepcopy
from pathlib import Path
@@ -102,7 +101,7 @@ def get_channel_image_info(url):
avatar_url = thumbnail['url']
if thumbnail['id'] == 'banner_uncropped':
banner_url = thumbnail['url']
- if banner_url != None and avatar_url != None:
+ if banner_url is not None and avatar_url is not None:
break
return avatar_url, banner_url
@@ -143,7 +142,7 @@ def get_media_info(url, /, *, days=None, info_json=None):
if days is not None:
try:
days = int(str(days), 10)
- except Exception as e:
+ except (TypeError, ValueError):
days = None
start = (
f'yesterday-{days!s}days' if days else None
@@ -326,13 +325,10 @@ def download_media(
final_path = Path(output_file).resolve(strict=False)
expected_file = shell_quote(str(final_path))
cmds = pp_opts.exec_cmd.get('after_move', list())
- # It is important that we use a tuple for strings.
- # Otherwise, list adds each character instead.
- # That last comma is really necessary!
- cmds += (
+ cmds.append(
f'test -f {expected_file} || '
'mv -T -u -- %(filepath,_filename|)q '
- f'{expected_file}',
+ f'{expected_file}'
)
# assignment is the quickest way to cover both 'get' cases
pp_opts.exec_cmd['after_move'] = cmds
@@ -387,7 +383,7 @@ def download_media(
youtube_ea_dict = ytopts['extractor_args'].get('youtube', dict())
formats_list = youtube_ea_dict.get('formats', list())
if 'missing_pot' not in formats_list:
- formats_list += ('missing_pot',)
+ formats_list.append('missing_pot')
youtube_ea_dict.update({
'formats': formats_list,
})
diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py
index ac2ee870..7a824e7c 100644
--- a/tubesync/tubesync/settings.py
+++ b/tubesync/tubesync/settings.py
@@ -198,7 +198,7 @@ RENAME_SOURCES = list()
# You have been warned!
try:
- from .local_settings import *
+ from .local_settings import * # noqa
except ImportError as e:
import sys
sys.stderr.write(f'Unable to import local_settings: {e}\n')
@@ -222,5 +222,5 @@ if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS:
BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS
-from .dbutils import patch_ensure_connection
+from .dbutils import patch_ensure_connection # noqa
patch_ensure_connection()