From c22ce9b67c053719af7e3c92fefc6e20832d1c2a Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 08:42:06 -0400 Subject: [PATCH 01/67] Fixes for `ruff check` in youtube.py --- tubesync/sync/youtube.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 9814db24..022fbcf9 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -6,7 +6,6 @@ import os -from collections import namedtuple from common.logger import log from copy import deepcopy from pathlib import Path @@ -102,7 +101,7 @@ def get_channel_image_info(url): avatar_url = thumbnail['url'] if thumbnail['id'] == 'banner_uncropped': banner_url = thumbnail['url'] - if banner_url != None and avatar_url != None: + if banner_url is not None and avatar_url is not None: break return avatar_url, banner_url @@ -143,7 +142,7 @@ def get_media_info(url, /, *, days=None, info_json=None): if days is not None: try: days = int(str(days), 10) - except Exception as e: + except (TypeError, ValueError): days = None start = ( f'yesterday-{days!s}days' if days else None From 2785c7d3878fee1db60c5b432f432bf735094316 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 09:06:26 -0400 Subject: [PATCH 02/67] Fixes for `ruff check` in views.py --- tubesync/sync/views.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 10cfc5db..2fd4dec8 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -35,7 +35,6 @@ from .tasks import (map_task_to_instance, get_error_message, from .choices import (Val, MediaServerType, SourceResolution, IndexSchedule, YouTube_SourceType, youtube_long_source_types, youtube_help, youtube_validation_urls) -from . import signals from . import youtube @@ -258,7 +257,7 @@ class ValidateSourceView(FormView): self.key = youtube.get_channel_id( index_url.replace('/channel/', '/') ) - except youtube.YouTubeError as e: + except youtube.YouTubeError: # It did not work, revert to previous behavior self.key = old_key self.source_type = old_source_type @@ -296,10 +295,13 @@ class EditSourceMixin: def form_valid(self, form: Form): # Perform extra validation to make sure the media_format is valid obj = form.save(commit=False) - source_type = form.cleaned_data['media_format'] + # temporarily use media_format from the form + saved_media_format = obj.media_format + obj.media_format = form.cleaned_data['media_format'] example_media_file = obj.get_example_media_format() + obj.media_format = saved_media_format - if example_media_file == '': + if '' == example_media_file: form.add_error( 'media_format', ValidationError(self.errors['invalid_media_format']) @@ -307,12 +309,16 @@ class EditSourceMixin: # Check for suspicious file path(s) try: - targetCheck = form.cleaned_data['directory']+"/.virt" - newdir = safe_join(settings.DOWNLOAD_ROOT,targetCheck) + targetCheck = form.cleaned_data['directory'] + '/.virt' + safe_join(settings.DOWNLOAD_ROOT, targetCheck) except SuspiciousFileOperation: form.add_error( 'directory', - ValidationError(self.errors['dir_outside_dlroot'].replace("%BASEDIR%",str(settings.DOWNLOAD_ROOT))) + ValidationError( + self.errors['dir_outside_dlroot'].replace( + "%BASEDIR%", str(settings.DOWNLOAD_ROOT) + ) + ), ) if form.errors: From d956a99110b63f6e139914bb71f10dd815f21a94 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 09:14:24 -0400 Subject: [PATCH 03/67] Fixes for `ruff check` in tasks.py --- tubesync/sync/tasks.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 0467a4fd..1f94e231 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -6,7 +6,6 @@ import os import json -import math import random import requests import time @@ -14,9 +13,8 @@ import uuid from io import BytesIO from hashlib import sha1 from pathlib import Path -from datetime import datetime, timedelta +from datetime import timedelta from shutil import copyfile, rmtree -from PIL import Image from django import db from django.conf import settings from django.core.files.base import ContentFile @@ -36,7 +34,7 @@ from common.utils import ( django_queryset_generator as qs_gen, remove_enclosed, ) from .choices import Val, TaskQueue from .models import Source, Media, MediaServer -from .utils import ( get_remote_image, resize_image_to_height, delete_file, +from .utils import ( get_remote_image, resize_image_to_height, write_text_file, filter_response, ) from .youtube import YouTubeError @@ -226,7 +224,7 @@ def save_model(instance): @atomic(durable=False) def schedule_media_servers_update(): # Schedule a task to update media servers - log.info(f'Scheduling media server updates') + log.info('Scheduling media server updates') verbose_name = _('Request media server rescan for "{}"') for mediaserver in MediaServer.objects.all(): rescan_media_server( @@ -416,7 +414,7 @@ def download_source_images(source_id): log.info(f'Thumbnail URL for source with ID: {source_id} / {source} ' f'Avatar: {avatar} ' f'Banner: {banner}') - if banner != None: + if banner is not None: url = banner i = get_remote_image(url) image_file = BytesIO() @@ -432,7 +430,7 @@ def download_source_images(source_id): f.write(django_file.read()) i = image_file = None - if avatar != None: + if avatar is not None: url = avatar i = get_remote_image(url) image_file = BytesIO() @@ -845,7 +843,7 @@ def delete_all_media_for_source(source_id, source_name, source_directory): assert source_directory try: source = Source.objects.get(pk=source_id) - except Source.DoesNotExist as e: + except Source.DoesNotExist: # Task triggered but the source no longer exists, do nothing log.warn(f'Task delete_all_media_for_source(pk={source_id}) called but no ' f'source exists with ID: {source_id}') From 34dfb52bf08b40d5d5768a0af3233a8de5b24c7b Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 09:43:17 -0400 Subject: [PATCH 04/67] Fixes for `ruff check` in signals.py --- tubesync/sync/signals.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 790ce1c2..69254146 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -10,11 +10,11 @@ from django.utils.translation import gettext_lazy as _ from background_task.signals import task_failed from background_task.models import Task from common.logger import log -from .models import Source, Media, MediaServer, Metadata +from .models import Source, Media, Metadata from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task, download_media_thumbnail, download_media_metadata, map_task_to_instance, check_source_directory_exists, - download_media, rescan_media_server, download_source_images, + download_media, download_source_images, delete_all_media_for_source, save_all_media_for_source, rename_media, get_media_metadata_task, get_media_download_task) from .utils import delete_file, glob_quote, mkdir_p @@ -270,7 +270,7 @@ def media_post_save(sender, instance, created, **kwargs): if not (media_file_exists or existing_media_download_task): # The file was deleted after it was downloaded, skip this media. if instance.can_download and instance.downloaded: - skip_changed = True != instance.skip + skip_changed = True if not instance.skip else False instance.skip = True downloaded = False if (instance.source.download_media and instance.can_download) and not ( @@ -374,13 +374,13 @@ def media_post_delete(sender, instance, **kwargs): try: p.rmdir() log.info(f'Deleted directory for: {instance} path: {p!s}') - except OSError as e: + except OSError: pass # Delete the directory itself try: other_path.rmdir() log.info(f'Deleted directory for: {instance} path: {other_path!s}') - except OSError as e: + except OSError: pass # Get all files that start with the bare file path all_related_files = video_path.parent.glob(f'{glob_quote(video_path.with_suffix("").name)}*') From 565fe79183dee26d855c57b9401e29effa6350ac Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 09:47:57 -0400 Subject: [PATCH 05/67] Fixes for `ruff check` in models/source.py --- tubesync/sync/models/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 74f75278..b7e2b3da 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -510,7 +510,7 @@ class Source(db.models.Model): def get_example_media_format(self): try: return self.media_format.format(**self.example_media_format_dict) - except Exception as e: + except Exception: return '' def is_regex_match(self, media_item_title): From 6f8bc52c0a8e910fa4b1f187e29ffef7db39d1cb Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 09:53:30 -0400 Subject: [PATCH 06/67] Fixes for `ruff check` in models/media.py --- tubesync/sync/models/media.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 6eb0ed76..4af39c7a 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -17,7 +17,6 @@ from common.logger import log from common.errors import NoFormatException from common.utils import ( clean_filename, clean_emoji, - django_queryset_generator as qs_gen, ) from ..youtube import ( get_media_info as get_youtube_media_info, @@ -688,7 +687,7 @@ class Media(models.Model): pass setattr(self, '_cached_metadata_dict', data) return data - except Exception as e: + except Exception: return {} @@ -1219,7 +1218,7 @@ class Media(models.Model): parent_dir.rmdir() log.info(f'Removed empty directory: {parent_dir!s}') parent_dir = parent_dir.parent - except OSError as e: + except OSError: pass From 4ead042235bb73138297df208daa0e60834a3b30 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 10:10:37 -0400 Subject: [PATCH 07/67] Fixes for `ruff check` in models/__init__.py --- tubesync/sync/models/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tubesync/sync/models/__init__.py b/tubesync/sync/models/__init__.py index d7ed077c..a850a4e0 100644 --- a/tubesync/sync/models/__init__.py +++ b/tubesync/sync/models/__init__.py @@ -17,3 +17,9 @@ from .media import Media from .metadata import Metadata from .metadata_format import MetadataFormat +__all__ = [ + 'get_media_file_path', 'get_media_thumb_path', + 'media_file_storage', 'MediaServer', 'Source', + 'Media', 'Metadata', 'MetadataFormat', +] + From 48c9fb1ca32edda29f180fb9ddbc928792958b40 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 10:15:30 -0400 Subject: [PATCH 08/67] Fixes for `ruff check` in mediaservers.py --- tubesync/sync/mediaservers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tubesync/sync/mediaservers.py b/tubesync/sync/mediaservers.py index ceab239f..e141238d 100644 --- a/tubesync/sync/mediaservers.py +++ b/tubesync/sync/mediaservers.py @@ -117,9 +117,6 @@ class PlexMediaServer(MediaServer): raise ValidationError('Plex Media Server "port" must be between 1 ' 'and 65535') options = self.object.options - if 'token' not in options: - raise ValidationError('Plex Media Server requires a "token"') - token = options['token'].strip() if 'token' not in options: raise ValidationError('Plex Media Server requires a "token"') if 'libraries' not in options: From c07ed8eb040ec9ef7064f8b0fa015f40d3546f62 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 10:18:04 -0400 Subject: [PATCH 09/67] Fixes for `ruff check` in hooks.py --- tubesync/sync/hooks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tubesync/sync/hooks.py b/tubesync/sync/hooks.py index 3bb3ce0d..467e2df1 100644 --- a/tubesync/sync/hooks.py +++ b/tubesync/sync/hooks.py @@ -1,9 +1,7 @@ import os -import yt_dlp from common.logger import log from common.utils import remove_enclosed -from django.conf import settings progress_hook = { From 60579ab883d3357e66b2e4db76fbca7103e7f663 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 10:19:48 -0400 Subject: [PATCH 10/67] Fixes for `ruff check` in filtering.py --- tubesync/sync/filtering.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/filtering.py b/tubesync/sync/filtering.py index 3f7023c1..ef206696 100644 --- a/tubesync/sync/filtering.py +++ b/tubesync/sync/filtering.py @@ -5,7 +5,6 @@ from common.logger import log from .models import Media from datetime import datetime -from django.utils import timezone from .overrides.custom_filter import filter_custom From 087e2f2e5d39565e53dccf11b0726613ee194f3e Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 10:22:16 -0400 Subject: [PATCH 11/67] Fixes for `ruff check` in fields.py --- tubesync/sync/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py index 452fee77..648afc63 100644 --- a/tubesync/sync/fields.py +++ b/tubesync/sync/fields.py @@ -63,7 +63,7 @@ class CommaSepChoiceField(models.CharField): def __init__(self, *args, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, **kwargs): kwargs.setdefault('max_length', 128) self.separator = str(separator) - self.possible_choices = possible_choices or choices + self.possible_choices = possible_choices or kwargs['choices'] self.selected_choices = list() self.allow_all = allow_all self.all_label = all_label From 35d2aa7b2d87330c266fb6f7b91386318eb9fff4 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 10:23:48 -0400 Subject: [PATCH 12/67] Fixes for `ruff check` in urls.py --- tubesync/common/urls.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/common/urls.py b/tubesync/common/urls.py index 1f29056d..a3e00a84 100644 --- a/tubesync/common/urls.py +++ b/tubesync/common/urls.py @@ -1,7 +1,6 @@ from django.conf import settings from django.urls import path from django.views.generic.base import RedirectView -from django.views.generic import TemplateView from django.http import HttpResponse from .views import error403, error404, error500, HealthCheckView From 58ed18519afc0d3136cf19d4ff7bf5c7ab941421 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 10:33:42 -0400 Subject: [PATCH 13/67] Fixes for `ruff check` in utils.py --- tubesync/common/utils.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index c4798943..5fc515df 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -84,14 +84,11 @@ def parse_database_connection_string(database_connection_string): f'invalid driver, must be one of {valid_drivers}') django_driver = django_backends.get(driver) host_parts = user_pass_host_port.split('@') - if len(host_parts) != 2: - raise DatabaseConnectionError(f'Database connection string netloc must be in ' - f'the format of user:pass@host') + user_pass_parts = host_parts[0].split(':') + if len(host_parts) != 2 or len(user_pass_parts) != 2: + raise DatabaseConnectionError('Database connection string netloc must be in ' + 'the format of user:pass@host') user_pass, host_port = host_parts - user_pass_parts = user_pass.split(':') - if len(user_pass_parts) != 2: - raise DatabaseConnectionError(f'Database connection string netloc must be in ' - f'the format of user:pass@host') username, password = user_pass_parts host_port_parts = host_port.split(':') if len(host_port_parts) == 1: @@ -113,13 +110,13 @@ def parse_database_connection_string(database_connection_string): f'65535, got {port}') else: # Malformed - raise DatabaseConnectionError(f'Database connection host must be a hostname or ' - f'a hostname:port combination') + raise DatabaseConnectionError('Database connection host must be a hostname or ' + 'a hostname:port combination') if database.startswith('/'): database = database[1:] if not database: - raise DatabaseConnectionError(f'Database connection string path must be a ' - f'string in the format of /databasename') + raise DatabaseConnectionError('Database connection string path must be a ' + 'string in the format of /databasename') if '/' in database: raise DatabaseConnectionError(f'Database connection string path can only ' f'contain a single string name, got: {database}') From 2e6f07652a510e5351d91d86b0cad6de7651cdba Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 19:57:09 -0400 Subject: [PATCH 14/67] Import signals We must import this file to have Django signals used. This should probably be done as part of app configuration instead of in views. --- tubesync/sync/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 2fd4dec8..4f683834 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -35,6 +35,7 @@ from .tasks import (map_task_to_instance, get_error_message, from .choices import (Val, MediaServerType, SourceResolution, IndexSchedule, YouTube_SourceType, youtube_long_source_types, youtube_help, youtube_validation_urls) +from . import signals # noqa from . import youtube From 9e13870e6f3e63c54fc1bc048702e63a59020ef5 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 20:01:51 -0400 Subject: [PATCH 15/67] Use the `dict.get` function instead --- tubesync/sync/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py index 648afc63..e489ff08 100644 --- a/tubesync/sync/fields.py +++ b/tubesync/sync/fields.py @@ -63,7 +63,7 @@ class CommaSepChoiceField(models.CharField): def __init__(self, *args, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, **kwargs): kwargs.setdefault('max_length', 128) self.separator = str(separator) - self.possible_choices = possible_choices or kwargs['choices'] + self.possible_choices = possible_choices or kwargs.get('choices') self.selected_choices = list() self.allow_all = allow_all self.all_label = all_label From c85dd553dc7b8a79d38ba0583add2eb844717c66 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 20:13:44 -0400 Subject: [PATCH 16/67] Remove unused `NoMetadataException` --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 1f94e231..483f6c14 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -28,7 +28,7 @@ from background_task.exceptions import InvalidTaskError from background_task.models import Task, CompletedTask from common.logger import log from common.errors import ( NoFormatException, NoMediaException, - NoMetadataException, NoThumbnailException, + NoThumbnailException, DownloadFailedException, ) from common.utils import ( django_queryset_generator as qs_gen, remove_enclosed, ) From 0fa5c9ef97db27ed55e6b660b1741b93947d2f23 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 17 May 2025 20:17:08 -0400 Subject: [PATCH 17/67] Add missing imports --- tubesync/sync/models/media__tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/models/media__tasks.py b/tubesync/sync/models/media__tasks.py index d63f7b31..24cbb395 100644 --- a/tubesync/sync/models/media__tasks.py +++ b/tubesync/sync/models/media__tasks.py @@ -1,9 +1,11 @@ import os +from pathlib import Path from common.logger import log from common.errors import ( NoMetadataException, ) from django.utils import timezone +from django.utils.translation import gettext_lazy as _ from ..choices import Val, SourceResolution From c69cd7f5823a06a3839dc82568b84a8f13c483cf Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 02:52:15 -0400 Subject: [PATCH 18/67] Fixes for `ruff check` in list-sources.py --- tubesync/sync/management/commands/list-sources.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/management/commands/list-sources.py b/tubesync/sync/management/commands/list-sources.py index 4ee177ae..25eae481 100644 --- a/tubesync/sync/management/commands/list-sources.py +++ b/tubesync/sync/management/commands/list-sources.py @@ -1,7 +1,6 @@ -import os -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand, CommandError # noqa from common.logger import log -from sync.models import Source, Media, MediaServer +from sync.models import Source class Command(BaseCommand): From 2ece2e7f383dd6d9fc684cbc07e2623eecaf242a Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 03:12:12 -0400 Subject: [PATCH 19/67] Fixes for `ruff check` in reset-tasks.py --- tubesync/sync/management/commands/reset-tasks.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py index 55436863..318a5b27 100644 --- a/tubesync/sync/management/commands/reset-tasks.py +++ b/tubesync/sync/management/commands/reset-tasks.py @@ -1,14 +1,12 @@ -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand, CommandError # noqa from django.db.transaction import atomic from django.utils.translation import gettext_lazy as _ from background_task.models import Task +from common.logger import log from sync.models import Source from sync.tasks import index_source_task, check_source_directory_exists -from common.logger import log - - class Command(BaseCommand): help = 'Resets all tasks' @@ -31,10 +29,10 @@ class Command(BaseCommand): index_source_task( str(source.pk), repeat=source.index_schedule, + schedule=source.index_schedule, verbose_name=verbose_name.format(source.name), ) - with atomic(durable=True): - for source in Source.objects.all(): # This also chains down to call each Media objects .save() as well source.save() + log.info('Done') From da182070f87867c33b87e2b98ab4ca2bc41eba75 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 03:23:15 -0400 Subject: [PATCH 20/67] Fixes for `ruff check` in delete-source.py --- tubesync/sync/management/commands/delete-source.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/management/commands/delete-source.py b/tubesync/sync/management/commands/delete-source.py index 42f9d5ac..d9f8f204 100644 --- a/tubesync/sync/management/commands/delete-source.py +++ b/tubesync/sync/management/commands/delete-source.py @@ -1,16 +1,15 @@ -import os import uuid -from django.utils.translation import gettext_lazy as _ from django.core.management.base import BaseCommand, CommandError from django.db.transaction import atomic +from django.utils.translation import gettext_lazy as _ from common.logger import log -from sync.models import Source, Media, MediaServer +from sync.models import Source from sync.tasks import schedule_media_servers_update class Command(BaseCommand): - help = _('Deletes a source by UUID') + help = 'Deletes a source by UUID' def add_arguments(self, parser): parser.add_argument('--source', action='store', required=True, help=_('Source UUID')) From 7a53e363b2ae3fdbe2d753c01d154b97aac16f52 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 05:24:36 -0400 Subject: [PATCH 21/67] Fixes for `ruff check` in sync-missing-metadata.py --- tubesync/sync/management/commands/sync-missing-metadata.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/sync-missing-metadata.py b/tubesync/sync/management/commands/sync-missing-metadata.py index 21b25c52..863f83f9 100644 --- a/tubesync/sync/management/commands/sync-missing-metadata.py +++ b/tubesync/sync/management/commands/sync-missing-metadata.py @@ -1,6 +1,5 @@ -import os from shutil import copyfile -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand, CommandError # noqa from django.db.models import Q from common.logger import log from sync.models import Source, Media From 87bcfb90940c86b077d223ef79c87dc86a005623 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 05:51:31 -0400 Subject: [PATCH 22/67] Fixes for `ruff check` in youtube-dl-info.py --- tubesync/sync/management/commands/youtube-dl-info.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/management/commands/youtube-dl-info.py b/tubesync/sync/management/commands/youtube-dl-info.py index 32a47402..c8099765 100644 --- a/tubesync/sync/management/commands/youtube-dl-info.py +++ b/tubesync/sync/management/commands/youtube-dl-info.py @@ -1,7 +1,7 @@ import json -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand, CommandError # noqa from sync.youtube import get_media_info -from common.utils import json_serial +from common.json import JSONEncoder class Command(BaseCommand): @@ -15,6 +15,6 @@ class Command(BaseCommand): url = options['url'] self.stdout.write(f'Showing information for URL: {url}') info = get_media_info(url) - d = json.dumps(info, indent=4, sort_keys=True, default=json_serial) + d = json.dumps(info, indent=4, sort_keys=True, cls=JSONEncoder) self.stdout.write(d) self.stdout.write('Done') From 9a173dc4775919f2e81f52fbf7f5682966dfa73d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 06:08:49 -0400 Subject: [PATCH 23/67] Use `JSONEncoder` in media.py --- tubesync/sync/models/media.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 4af39c7a..11b30f80 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -15,6 +15,7 @@ from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.logger import log from common.errors import NoFormatException +from common.json import JSONEncoder from common.utils import ( clean_filename, clean_emoji, ) @@ -577,14 +578,13 @@ class Media(models.Model): def metadata_dumps(self, arg_dict=dict()): - from common.utils import json_serial fallback = dict() try: fallback.update(self.new_metadata.with_formats) except ObjectDoesNotExist: pass data = arg_dict or fallback - return json.dumps(data, separators=(',', ':'), default=json_serial) + return json.dumps(data, separators=(',', ':'), cls=JSONEncoder) def metadata_loads(self, arg_str='{}'): From 3abab98f64f113993cd023a9058b57c01f978757 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 06:11:16 -0400 Subject: [PATCH 24/67] Remove `json_serial` from utils.py --- tubesync/common/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 5fc515df..020035b8 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -169,14 +169,6 @@ def clean_emoji(s): return emoji.replace_emoji(s) -def json_serial(obj): - if isinstance(obj, datetime): - return obj.isoformat() - if isinstance(obj, LazyList): - return list(obj) - raise TypeError(f'Type {type(obj)} is not json_serial()-able') - - def time_func(func): def wrapper(*args, **kwargs): start = time.perf_counter() From 1ba4bca58b4563fffdc88cdf199f84aea20a6bc7 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 06:16:06 -0400 Subject: [PATCH 25/67] Add `json_serial` to json.py --- tubesync/common/json.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tubesync/common/json.py b/tubesync/common/json.py index e8a22e1c..5a56a019 100644 --- a/tubesync/common/json.py +++ b/tubesync/common/json.py @@ -1,4 +1,6 @@ +from datetime import datetime from django.core.serializers.json import DjangoJSONEncoder +from yt_dlp.utils import LazyList class JSONEncoder(DjangoJSONEncoder): @@ -14,3 +16,11 @@ class JSONEncoder(DjangoJSONEncoder): return list(iterable) return super().default(obj) + +def json_serial(obj): + if isinstance(obj, datetime): + return obj.isoformat() + if isinstance(obj, LazyList): + return list(obj) + raise TypeError(f'Type {type(obj)} is not json_serial()-able') + From cc0ed6fd473047c2f661dfb50f668dba7103d67d Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 06:18:41 -0400 Subject: [PATCH 26/67] Remove unused imports from utils.py --- tubesync/common/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 020035b8..8f7afc2c 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -6,10 +6,8 @@ import os import pstats import string import time -from datetime import datetime from django.core.paginator import Paginator from urllib.parse import urlunsplit, urlencode, urlparse -from yt_dlp.utils import LazyList from .errors import DatabaseConnectionError From aa8804bb7d58261111b30eef5764f7e4be47bebb Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 06:25:23 -0400 Subject: [PATCH 27/67] Fixes for `ruff check` in import-existing-media.py --- tubesync/sync/management/commands/import-existing-media.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/management/commands/import-existing-media.py b/tubesync/sync/management/commands/import-existing-media.py index 3813b497..c05c630a 100644 --- a/tubesync/sync/management/commands/import-existing-media.py +++ b/tubesync/sync/management/commands/import-existing-media.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand, CommandError # noqa from common.logger import log from common.timestamp import timestamp_to_datetime from sync.choices import FileExtension @@ -18,7 +18,7 @@ class Command(BaseCommand): dirmap = {} for s in Source.objects.all(): dirmap[str(s.directory_path)] = s - log.info(f'Scanning sources...') + log.info('Scanning sources...') file_extensions = list(FileExtension.values) + self.extra_extensions for sourceroot, source in dirmap.items(): media = list(Media.objects.filter(source=source, downloaded=False, From 226a8d6385c1027044935320d3c2ed6fe3792216 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 06:31:24 -0400 Subject: [PATCH 28/67] Fixes for `ruff check` in models/_private.py --- tubesync/sync/models/_private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/_private.py b/tubesync/sync/models/_private.py index 96539dbe..5ec14d7c 100644 --- a/tubesync/sync/models/_private.py +++ b/tubesync/sync/models/_private.py @@ -1,4 +1,4 @@ -from ..choices import Val, YouTube_SourceType +from ..choices import Val, YouTube_SourceType # noqa _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) )) From bb6d5c0a48cbb2cf8e2a26f3a917681467c62cc7 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 06:43:34 -0400 Subject: [PATCH 29/67] Fixes for `ruff check` in tests.py --- tubesync/sync/tests.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 24f0d092..9c510838 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -6,7 +6,6 @@ import logging -import os from datetime import datetime, timedelta from pathlib import Path from urllib.parse import urlsplit @@ -1822,13 +1821,13 @@ class TasksTestCase(TestCase): now = timezone.now() - m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5)) - m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25)) - m13 = Media.objects.create(source=src1, downloaded=False, key='a13') + m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5)) # noqa + m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25)) # noqa + m13 = Media.objects.create(source=src1, downloaded=False, key='a13') # noqa - m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5)) - m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25)) - m23 = Media.objects.create(source=src2, downloaded=False, key='a23') + m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5)) # noqa + m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25)) # noqa + m23 = Media.objects.create(source=src2, downloaded=False, key='a23') # noqa self.assertEqual(src1.media_source.all().count(), 3) self.assertEqual(src2.media_source.all().count(), 3) From 0f27ffe29e8484f6f046d9f5990cf64f864de3b3 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 06:50:13 -0400 Subject: [PATCH 30/67] Fixes for `ruff check` in settings.py --- tubesync/tubesync/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 7f5922ae..37f72de3 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -198,7 +198,7 @@ RENAME_SOURCES = None # You have been warned! try: - from .local_settings import * + from .local_settings import * # noqa except ImportError as e: import sys sys.stderr.write(f'Unable to import local_settings: {e}\n') @@ -222,5 +222,5 @@ if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS: BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS -from .dbutils import patch_ensure_connection +from .dbutils import patch_ensure_connection # noqa patch_ensure_connection() From a4c763d723380764f0c15271b80f9329b6af9b56 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 06:55:12 -0400 Subject: [PATCH 31/67] Fixes for `ruff check` in 0013_fix_elative_media_file.py --- tubesync/sync/migrations/0013_fix_elative_media_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/migrations/0013_fix_elative_media_file.py b/tubesync/sync/migrations/0013_fix_elative_media_file.py index c9eee22e..2f1ac385 100644 --- a/tubesync/sync/migrations/0013_fix_elative_media_file.py +++ b/tubesync/sync/migrations/0013_fix_elative_media_file.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.12 on 2022-04-06 06:19 from django.conf import settings -from django.db import migrations, models +from django.db import migrations def fix_media_file(apps, schema_editor): From 853c4878b5c76e5da6a93be2c6451fad8d348b40 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 07:00:27 -0400 Subject: [PATCH 32/67] Fixes for `ruff check` in 0011_auto_20220201_1654.py --- tubesync/sync/migrations/0011_auto_20220201_1654.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tubesync/sync/migrations/0011_auto_20220201_1654.py b/tubesync/sync/migrations/0011_auto_20220201_1654.py index 96d9f4a7..51641ece 100644 --- a/tubesync/sync/migrations/0011_auto_20220201_1654.py +++ b/tubesync/sync/migrations/0011_auto_20220201_1654.py @@ -1,8 +1,6 @@ # Generated by Django 3.2.11 on 2022-02-01 16:54 -import django.core.files.storage from django.db import migrations, models -import sync.models class Migration(migrations.Migration): From a9508fa2803052cc20e0a21fb4f9d1ba778fd80c Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 07:21:13 -0400 Subject: [PATCH 33/67] Update tests.py --- tubesync/sync/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 9c510838..47089673 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -1826,7 +1826,7 @@ class TasksTestCase(TestCase): m13 = Media.objects.create(source=src1, downloaded=False, key='a13') # noqa m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5)) # noqa - m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25)) # noqa + m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25)) m23 = Media.objects.create(source=src2, downloaded=False, key='a23') # noqa self.assertEqual(src1.media_source.all().count(), 3) From 0e720f90423f5917edc7150b4cbdf139dbdbf96e Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 21:45:24 -0400 Subject: [PATCH 34/67] Use the `list.append` function, which is safe for strings --- tubesync/sync/youtube.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 9814db24..c795261a 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -326,13 +326,10 @@ def download_media( final_path = Path(output_file).resolve(strict=False) expected_file = shell_quote(str(final_path)) cmds = pp_opts.exec_cmd.get('after_move', list()) - # It is important that we use a tuple for strings. - # Otherwise, list adds each character instead. - # That last comma is really necessary! - cmds += ( + cmds.append( f'test -f {expected_file} || ' 'mv -T -u -- %(filepath,_filename|)q ' - f'{expected_file}', + f'{expected_file}' ) # assignment is the quickest way to cover both 'get' cases pp_opts.exec_cmd['after_move'] = cmds @@ -387,7 +384,7 @@ def download_media( youtube_ea_dict = ytopts['extractor_args'].get('youtube', dict()) formats_list = youtube_ea_dict.get('formats', list()) if 'missing_pot' not in formats_list: - formats_list += ('missing_pot',) + formats_list.append('missing_pot') youtube_ea_dict.update({ 'formats': formats_list, }) From 971cea5c0eef12e4baa829985e43ec5b653866f9 Mon Sep 17 00:00:00 2001 From: tcely Date: Sun, 18 May 2025 22:26:06 -0400 Subject: [PATCH 35/67] Use the less ambiguous `list.extend` function --- tubesync/sync/models/source.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 74f75278..063907f3 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -527,8 +527,8 @@ class Source(db.models.Model): days = timezone.timedelta(seconds=self.download_cap).days response = indexer(self.get_index_url(type=type), days=days) if not isinstance(response, dict): - return [] - entries = response.get('entries', []) + return list() + entries = response.get('entries', list()) return entries def index_media(self): @@ -537,11 +537,11 @@ class Source(db.models.Model): ''' entries = list() if self.index_videos: - entries += self.get_index('videos') + entries.extend(self.get_index('videos')) # Playlists do something different that I have yet to figure out if not self.is_playlist: if self.index_streams: - entries += self.get_index('streams') + entries.extend(self.get_index('streams')) if settings.MAX_ENTRIES_PROCESSING: entries = entries[:settings.MAX_ENTRIES_PROCESSING] From 8ba9ce40782e231f377148ae0504e319f77f9e9f Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 00:10:13 -0400 Subject: [PATCH 36/67] Use a queue to avoid list memory performance --- tubesync/sync/models/source.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 063907f3..ebe90767 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -1,6 +1,7 @@ import os import re import uuid +from collections import deque from pathlib import Path from django import db from django.conf import settings @@ -533,17 +534,20 @@ class Source(db.models.Model): def index_media(self): ''' - Index the media source returning a list of media metadata as dicts. + Index the media source returning a queue of media metadata as dicts. ''' - entries = list() + entries = deque(list(), settings.get('MAX_ENTRIES_PROCESSING', 0) or None) if self.index_videos: entries.extend(self.get_index('videos')) + # Playlists do something different that I have yet to figure out if not self.is_playlist: if self.index_streams: - entries.extend(self.get_index('streams')) + streams = self.get_index('streams') + # do not allow streams to consume all of the queue + if entries.maxlen and entries.maxlen <= len(streams): + streams = streams[-1 * ( entries.maxlen // 2 ) :] + entries.extend(reversed(streams)) - if settings.MAX_ENTRIES_PROCESSING: - entries = entries[:settings.MAX_ENTRIES_PROCESSING] return entries From c7c37b12d89a1044665c213b114d55b05fd5e82e Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 00:12:47 -0400 Subject: [PATCH 37/67] Use `queue` for better readability --- tubesync/sync/models/source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index ebe90767..a739936a 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -1,7 +1,7 @@ import os import re import uuid -from collections import deque +from collections import deque as queue from pathlib import Path from django import db from django.conf import settings @@ -536,7 +536,7 @@ class Source(db.models.Model): ''' Index the media source returning a queue of media metadata as dicts. ''' - entries = deque(list(), settings.get('MAX_ENTRIES_PROCESSING', 0) or None) + entries = queue(list(), settings.get('MAX_ENTRIES_PROCESSING', 0) or None) if self.index_videos: entries.extend(self.get_index('videos')) From e84643b95e517a3d28f0020741763442fdcbb4f7 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 01:44:49 -0400 Subject: [PATCH 38/67] Handle unbalanced channels better --- tubesync/sync/models/source.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index a739936a..04fe84d4 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -544,10 +544,15 @@ class Source(db.models.Model): if not self.is_playlist: if self.index_streams: streams = self.get_index('streams') - # do not allow streams to consume all of the queue - if entries.maxlen and entries.maxlen <= len(streams): - streams = streams[-1 * ( entries.maxlen // 2 ) :] - entries.extend(reversed(streams)) + if entries.maxlen is None or 0 == len(entries): + entries.extend(streams) + else: + # share the queue between streams and videos + allowed_streams = max( + entries.maxlen // 2, + entries.maxlen - len(entries), + ) + entries.extend(streams[-1 * allowed_streams :]) return entries From 2483cda1af9db626ba912ea9b4a4a838c5752079 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 05:54:23 -0400 Subject: [PATCH 39/67] Remove the unused `getpot_bgutil_script` plugin --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6ef178c5..c878890e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -410,6 +410,12 @@ RUN --mount=type=tmpfs,target=/cache \ PIPENV_VERBOSITY=64 \ PYTHONPYCACHEPREFIX=/cache/pycache \ pipenv install --system --skip-lock && \ + # remove the getpot_bgutil_script plugin + find /usr/local/lib \ + -name 'getpot_bgutil_script.py' \ + -path '*/yt_dlp_plugins/extractor/getpot_bgutil_script.py' \ + -type f -print -delete \ + && \ # Clean up apt-get -y autoremove --purge \ default-libmysqlclient-dev \ From 01fcf5ec66475238918eacfff7bc0b759d184882 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 07:17:56 -0400 Subject: [PATCH 40/67] Reverse the lists when adding to the queue The default sorting appears to be newest first. We want the oldest to be dropped from the queue, by any limits instead. --- tubesync/sync/models/source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 04fe84d4..776ddfe8 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -538,21 +538,21 @@ class Source(db.models.Model): ''' entries = queue(list(), settings.get('MAX_ENTRIES_PROCESSING', 0) or None) if self.index_videos: - entries.extend(self.get_index('videos')) + entries.extend(reversed(self.get_index('videos'))) # Playlists do something different that I have yet to figure out if not self.is_playlist: if self.index_streams: streams = self.get_index('streams') if entries.maxlen is None or 0 == len(entries): - entries.extend(streams) + entries.extend(reversed(streams)) else: # share the queue between streams and videos allowed_streams = max( entries.maxlen // 2, entries.maxlen - len(entries), ) - entries.extend(streams[-1 * allowed_streams :]) + entries.extend(reversed(streams[-1 * allowed_streams :])) return entries From 4cbcd7688536f6a2e19bfa4d79fcd971e0a232d5 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 07:36:37 -0400 Subject: [PATCH 41/67] Adjust for a queue instead of a list --- tubesync/sync/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 0467a4fd..b192ccfb 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -316,7 +316,10 @@ def index_source_task(source_id): end=task.verbose_name.find('Index'), ) tvn_format = '{:,}' + f'/{num_videos:,}' - for vn, video in enumerate(videos, start=1): + vn = 0 + while len(videos) > 0: + vn += 1 + video = videos.popleft() # Create or update each video as a Media object key = video.get(source.key_field, None) if not key: From 8109d6a836c4b1c0ff16f01d874621105cc5c648 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 08:09:56 -0400 Subject: [PATCH 42/67] Use `getattr` not `dict.get` for `settings` --- tubesync/sync/models/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 776ddfe8..82486521 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -536,7 +536,7 @@ class Source(db.models.Model): ''' Index the media source returning a queue of media metadata as dicts. ''' - entries = queue(list(), settings.get('MAX_ENTRIES_PROCESSING', 0) or None) + entries = queue(list(), getattr(settings, 'MAX_ENTRIES_PROCESSING', 0) or None) if self.index_videos: entries.extend(reversed(self.get_index('videos'))) From ed55710f26e08657f206aef48080eb2f28a5319b Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 08:54:21 -0400 Subject: [PATCH 43/67] Create a set of keys for `cleanup_removed_media` --- tubesync/sync/tasks.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index b192ccfb..6d0727c0 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -255,7 +255,7 @@ def cleanup_old_media(): schedule_media_servers_update() -def cleanup_removed_media(source, videos): +def cleanup_removed_media(source, video_keys): if not source.delete_removed_media: return log.info(f'Cleaning up media no longer in source: {source}') @@ -265,8 +265,7 @@ def cleanup_removed_media(source, videos): source=source, ) for media in qs_gen(mqs): - matching_source_item = [video['id'] for video in videos if video['id'] == media.key] - if not matching_source_item: + if media.key not in video_keys: log.info(f'{media.name} is no longer in source, removing') with atomic(): media.delete() @@ -317,6 +316,7 @@ def index_source_task(source_id): ) tvn_format = '{:,}' + f'/{num_videos:,}' vn = 0 + video_keys = set() while len(videos) > 0: vn += 1 video = videos.popleft() @@ -325,6 +325,7 @@ def index_source_task(source_id): if not key: # Video has no unique key (ID), it can't be indexed continue + video_keys.add(key) update_task_status(task, tvn_format.format(vn)) # media, new_media = Media.objects.get_or_create(key=key, source=source) try: @@ -379,7 +380,7 @@ def index_source_task(source_id): # Reset task.verbose_name to the saved value update_task_status(task, None) # Cleanup of media no longer available from the source - cleanup_removed_media(source, videos) + cleanup_removed_media(source, video_keys) videos = video = None From d05b53919ca1a6f9fbfbeda009eb1db941d07531 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 11:17:39 -0400 Subject: [PATCH 44/67] Wait for HTTP 429 errors --- tubesync/sync/tasks.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 0467a4fd..ba9f0261 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -235,6 +235,21 @@ def schedule_media_servers_update(): ) +def wait_for_errors(media, /, tn='sync.tasks.download_media_metadata'): + window = timezone.timedelta(hours=3) + timezone.now() + tqs = Task.objects.filter( + task_name=tn, + attempts__gt=0, + locked_at__isnull=True, + run_at__lte=window, + last_error__contains='HTTPError 429: Too Many Requests', + ) + task = get_first_task(tn, instance=media) + update_task_status(task, 'paused (429)') + time.sleep(10 * tqs.count()) + update_task_status(task, None) + + def cleanup_old_media(): with atomic(): for source in qs_gen(Source.objects.filter(delete_old_media=True, days_to_keep__gt=0)): @@ -467,6 +482,7 @@ def download_media_metadata(media_id): log.info(f'Task for ID: {media_id} / {media} skipped, due to task being manually skipped.') return source = media.source + wait_for_errors(media) try: metadata = media.index_metadata() except YouTubeError as e: From 1be1e583e56ac16ca3bb18661cadec1276181580 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 11:32:56 -0400 Subject: [PATCH 45/67] Use `hasattr` for `update_task_status` --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index ba9f0261..2942f37e 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -122,7 +122,7 @@ def get_error_message(task): def update_task_status(task, status): if not task: return False - if not task._verbose_name: + if not hasattr(task, '_verbose_name'): task._verbose_name = remove_enclosed( task.verbose_name, '[', ']', ' ', ) From 8080f690769d50816af6aad66adb2d5c0bbf7ae9 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 11:47:52 -0400 Subject: [PATCH 46/67] Remove an unnecessary condition --- tubesync/sync/tasks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 2942f37e..8b39434b 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -848,8 +848,7 @@ def wait_for_media_premiere(media_id): if hours: task = get_media_premiere_task(media_id) - if task: - update_task_status(task, f'available in {hours} hours') + update_task_status(task, f'available in {hours} hours') save_model(media) From be6a51b05273c301a31e34dc77ff8affcd8a37db Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 15:28:27 -0400 Subject: [PATCH 47/67] The check list needs to pass if nothing failed --- tubesync/sync/models/media__tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tubesync/sync/models/media__tasks.py b/tubesync/sync/models/media__tasks.py index d63f7b31..e8b33235 100644 --- a/tubesync/sync/models/media__tasks.py +++ b/tubesync/sync/models/media__tasks.py @@ -1,9 +1,11 @@ import os +from pathlib import Path from common.logger import log from common.errors import ( NoMetadataException, ) from django.utils import timezone +from django.utils.translation import gettext_lazy as _ from ..choices import Val, SourceResolution @@ -47,6 +49,7 @@ def download_checklist(self, skip_checks=False): f'the source has a download cap and the media is now too old, ' f'not downloading') return False + return True def download_finished(self, format_str, container, downloaded_filepath=None): From b1267a3e59981cf3ee32292ff754ff3add572dc2 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 15:34:13 -0400 Subject: [PATCH 48/67] Reschedule the task when the checklist failed --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 0467a4fd..2736fdc7 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -615,7 +615,7 @@ def download_media(media_id, override=False): raise InvalidTaskError(_('no such media')) from e else: if not media.download_checklist(override): - return + raise Exception('media download not ready') filepath = media.filepath container = format_str = None From 0aac8c27ed4cec619acbd10678c3ec05901dfcd0 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 15:40:54 -0400 Subject: [PATCH 49/67] Raising an exception here is not needed --- tubesync/sync/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 2736fdc7..00f4ffd3 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -615,7 +615,9 @@ def download_media(media_id, override=False): raise InvalidTaskError(_('no such media')) from e else: if not media.download_checklist(override): - raise Exception('media download not ready') + # any condition that needs ro reschedule the task + # should raise an exception to avoid this + return filepath = media.filepath container = format_str = None From e5347d485feffa06d1af3dc9e318512798f83eb7 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 19 May 2025 15:43:56 -0400 Subject: [PATCH 50/67] fixup: spelling in the comment --- tubesync/sync/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 00f4ffd3..7ce6df42 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -615,7 +615,7 @@ def download_media(media_id, override=False): raise InvalidTaskError(_('no such media')) from e else: if not media.download_checklist(override): - # any condition that needs ro reschedule the task + # any condition that needs to reschedule the task # should raise an exception to avoid this return From e135da45cab33954ffb86cff19a02887da7c2c1e Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 20 May 2025 02:51:23 -0400 Subject: [PATCH 51/67] This can be any model, not just media --- tubesync/sync/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 61cb52a0..78cc97b1 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -233,7 +233,7 @@ def schedule_media_servers_update(): ) -def wait_for_errors(media, /, tn='sync.tasks.download_media_metadata'): +def wait_for_errors(model, /, tn='sync.tasks.download_media_metadata'): window = timezone.timedelta(hours=3) + timezone.now() tqs = Task.objects.filter( task_name=tn, @@ -242,7 +242,7 @@ def wait_for_errors(media, /, tn='sync.tasks.download_media_metadata'): run_at__lte=window, last_error__contains='HTTPError 429: Too Many Requests', ) - task = get_first_task(tn, instance=media) + task = get_first_task(tn, instance=model) update_task_status(task, 'paused (429)') time.sleep(10 * tqs.count()) update_task_status(task, None) From 3c1cf294ebeec0996909f48697fa4323846608fe Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 20 May 2025 02:57:02 -0400 Subject: [PATCH 52/67] Streams are newest first also Keep the beginning when limited: Newest, ..., Oldest Reverse direction when adding to the queue: Oldest, ..., Newest This way when it is limited by queue size, we keep the newest streams for indexing. --- tubesync/sync/models/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index d85334f0..07ade020 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -552,7 +552,7 @@ class Source(db.models.Model): entries.maxlen // 2, entries.maxlen - len(entries), ) - entries.extend(reversed(streams[-1 * allowed_streams :])) + entries.extend(reversed(streams[: allowed_streams])) return entries From 52a1140f5d944af62a8b0b56b763b4f60556875c Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 20 May 2025 03:25:53 -0400 Subject: [PATCH 53/67] Wait for HTTP 429 for media and metadata download tasks --- tubesync/sync/tasks.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 78cc97b1..52c07533 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -233,10 +233,17 @@ def schedule_media_servers_update(): ) -def wait_for_errors(model, /, tn='sync.tasks.download_media_metadata'): +def wait_for_errors(model, /, *, task_name=None): + if task_name is None: + task_name=tuple(( + 'sync.tasks.download_media', + 'sync.tasks.download_media_metadata', + )) + elif isinstance(task_name, str): + task_name = tuple((task_name,)) window = timezone.timedelta(hours=3) + timezone.now() tqs = Task.objects.filter( - task_name=tn, + task_name__in=task_name, attempts__gt=0, locked_at__isnull=True, run_at__lte=window, @@ -484,7 +491,7 @@ def download_media_metadata(media_id): log.info(f'Task for ID: {media_id} / {media} skipped, due to task being manually skipped.') return source = media.source - wait_for_errors(media) + wait_for_errors(media, task_name='sync.tasks.download_media_metadata') try: metadata = media.index_metadata() except YouTubeError as e: @@ -637,6 +644,7 @@ def download_media(media_id, override=False): # should raise an exception to avoid this return + wait_for_errors(media, task_name='sync.tasks.download_media') filepath = media.filepath container = format_str = None log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"') From c539487b4bd63e57cb709aef42bbe20c9340cb3f Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 20 May 2025 03:43:12 -0400 Subject: [PATCH 54/67] Handle multiple task names --- tubesync/sync/tasks.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 52c07533..466e41d3 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -241,6 +241,11 @@ def wait_for_errors(model, /, *, task_name=None): )) elif isinstance(task_name, str): task_name = tuple((task_name,)) + tasks = list() + for tn in task_name: + ft = get_first_task(tn, instance=model) + if ft: + tasks.append(ft) window = timezone.timedelta(hours=3) + timezone.now() tqs = Task.objects.filter( task_name__in=task_name, @@ -249,10 +254,12 @@ def wait_for_errors(model, /, *, task_name=None): run_at__lte=window, last_error__contains='HTTPError 429: Too Many Requests', ) - task = get_first_task(tn, instance=model) - update_task_status(task, 'paused (429)') + for task in tasks: + update_task_status(task, 'paused (429)') + log.info(f'waiting for errors: 429 ({tqs.count()}): {model}') time.sleep(10 * tqs.count()) - update_task_status(task, None) + for task in tasks: + update_task_status(task, None) def cleanup_old_media(): From 3bc9310c98a47521ca6e01e79dd15b621d3df23e Mon Sep 17 00:00:00 2001 From: tcely Date: Tue, 20 May 2025 03:48:38 -0400 Subject: [PATCH 55/67] Log more human-friendly output --- tubesync/sync/tasks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 466e41d3..bf5e43ed 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -35,7 +35,7 @@ from common.utils import ( django_queryset_generator as qs_gen, from .choices import Val, TaskQueue from .models import Source, Media, MediaServer from .utils import ( get_remote_image, resize_image_to_height, - write_text_file, filter_response, ) + write_text_file, filter_response, seconds_to_timestr, ) from .youtube import YouTubeError db_vendor = db.connection.vendor @@ -256,8 +256,11 @@ def wait_for_errors(model, /, *, task_name=None): ) for task in tasks: update_task_status(task, 'paused (429)') - log.info(f'waiting for errors: 429 ({tqs.count()}): {model}') - time.sleep(10 * tqs.count()) + + delay = 10 * tqs.count() + time_str = seconds_to_timestr(delay) + log.info(f'waiting for errors: 429 ({time_str}): {model}') + time.sleep(delay) for task in tasks: update_task_status(task, None) From d4eeb776e829ee078a76aa0f13e61d9da8e0bee7 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 09:12:35 -0400 Subject: [PATCH 56/67] Close the `tr` tag on the next line --- tubesync/sync/templates/sync/media-item.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/media-item.html b/tubesync/sync/templates/sync/media-item.html index b70f78c2..074987ea 100644 --- a/tubesync/sync/templates/sync/media-item.html +++ b/tubesync/sync/templates/sync/media-item.html @@ -121,7 +121,8 @@ {% if media_file_path == media.filepath %}  (matched) {% endif %} - + + File size File size
{{ media.downloaded_filesize|bytesformat }} From 1415af50015d2cb716548463a6e78475e9b426e4 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 09:53:36 -0400 Subject: [PATCH 57/67] Allow skipping the loop --- tubesync/sync/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 5bc90d25..762e29fe 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -130,10 +130,10 @@ def file_is_editable(filepath): return False -def directory_and_stem(arg_path): +def directory_and_stem(arg_path, /, *, only_once=False): filepath = Path(arg_path) stem = Path(filepath.stem) - while stem.suffixes and '' != stem.suffix: + while not only_once and stem.suffixes and '' != stem.suffix: stem = Path(stem.stem) stem = str(stem) return (filepath.parent, stem,) From 54aa9fd165d522eae32888943e4c820935d2c962 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 09:57:33 -0400 Subject: [PATCH 58/67] The video file should not have multiple extensions --- tubesync/sync/models/media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 11b30f80..4f4a16aa 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -1172,7 +1172,7 @@ class Media(models.Model): self.save(update_fields=('media_file', 'skip')) log.info(f'Updated "media_file" in the database for: {self!s}') - (new_prefix_path, new_stem) = directory_and_stem(new_video_path) + (new_prefix_path, new_stem) = directory_and_stem(new_video_path, only_once=True) # move and change names to match stem for other_path in other_paths: From 15898df3bbb0205d97a9070ef3d0486e9bb997b6 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 10:03:08 -0400 Subject: [PATCH 59/67] The old video file should also have a single extension --- tubesync/sync/models/media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 4f4a16aa..cad249a3 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -1152,7 +1152,7 @@ class Media(models.Model): # collect the list of files to move # this should not include the video we just moved - (old_prefix_path, old_stem) = directory_and_stem(old_video_path) + (old_prefix_path, old_stem) = directory_and_stem(old_video_path, only_once=True) other_paths = list(old_prefix_path.glob(glob_quote(old_stem) + '*')) log.info(f'Collected {len(other_paths)} other paths for: {self!s}') From fc3a68393d946e2369c30173f09feddb903702ea Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 10:08:53 -0400 Subject: [PATCH 60/67] Loop for the fuzzy stem --- tubesync/sync/models/media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index cad249a3..454d82ee 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -1193,7 +1193,7 @@ class Media(models.Model): other_path.replace(new_file_path) for fuzzy_path in fuzzy_paths: - (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path) + (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path, only_once=False) old_file_str = fuzzy_path.name new_file_str = new_stem + old_file_str[len(fuzzy_stem):] new_file_path = Path(new_prefix_path / new_file_str) From 5c2dfca00f4b314e0c824e0727ae9060a80e2ddc Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 10:10:07 -0400 Subject: [PATCH 61/67] Change the default to not loop --- tubesync/sync/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 762e29fe..6cc2e0d3 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -130,7 +130,7 @@ def file_is_editable(filepath): return False -def directory_and_stem(arg_path, /, *, only_once=False): +def directory_and_stem(arg_path, /, *, only_once=True): filepath = Path(arg_path) stem = Path(filepath.stem) while not only_once and stem.suffixes and '' != stem.suffix: From 24105db5cb243a2e5956c6821250b7623f7d16cf Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 10:49:21 -0400 Subject: [PATCH 62/67] Use private `directory_and_stem` function --- tubesync/sync/models/_private.py | 8 ++++++++ tubesync/sync/models/media.py | 11 +++++------ tubesync/sync/utils.py | 9 --------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/tubesync/sync/models/_private.py b/tubesync/sync/models/_private.py index 5ec14d7c..baca9e11 100644 --- a/tubesync/sync/models/_private.py +++ b/tubesync/sync/models/_private.py @@ -10,3 +10,11 @@ def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2 element.tail = tail + (char * indent) return element +def directory_and_stem(arg_path, /, all_suffixes=False): + filepath = Path(arg_path) + stem = Path(filepath.stem) + while all_suffixes and stem.suffixes and '' != stem.suffix: + stem = Path(stem.stem) + stem = str(stem) + return (filepath.parent, stem,) + diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 454d82ee..62f73d5d 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -25,8 +25,7 @@ from ..youtube import ( ) from ..utils import ( seconds_to_timestr, parse_media_format, filter_response, - write_text_file, mkdir_p, directory_and_stem, glob_quote, - multi_key_sort, + write_text_file, mkdir_p, glob_quote, multi_key_sort, ) from ..matching import ( get_best_combined_format, @@ -39,7 +38,7 @@ from ..choices import ( from ._migrations import ( media_file_storage, get_media_thumb_path, get_media_file_path, ) -from ._private import _srctype_dict, _nfo_element +from ._private import _srctype_dict, _nfo_element, directory_and_stem from .media__tasks import ( download_checklist, download_finished, wait_for_premiere, ) @@ -1152,7 +1151,7 @@ class Media(models.Model): # collect the list of files to move # this should not include the video we just moved - (old_prefix_path, old_stem) = directory_and_stem(old_video_path, only_once=True) + (old_prefix_path, old_stem) = directory_and_stem(old_video_path) other_paths = list(old_prefix_path.glob(glob_quote(old_stem) + '*')) log.info(f'Collected {len(other_paths)} other paths for: {self!s}') @@ -1172,7 +1171,7 @@ class Media(models.Model): self.save(update_fields=('media_file', 'skip')) log.info(f'Updated "media_file" in the database for: {self!s}') - (new_prefix_path, new_stem) = directory_and_stem(new_video_path, only_once=True) + (new_prefix_path, new_stem) = directory_and_stem(new_video_path) # move and change names to match stem for other_path in other_paths: @@ -1193,7 +1192,7 @@ class Media(models.Model): other_path.replace(new_file_path) for fuzzy_path in fuzzy_paths: - (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path, only_once=False) + (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path, True) old_file_str = fuzzy_path.name new_file_str = new_stem + old_file_str[len(fuzzy_stem):] new_file_path = Path(new_prefix_path / new_file_str) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 6cc2e0d3..fc7874fd 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -130,15 +130,6 @@ def file_is_editable(filepath): return False -def directory_and_stem(arg_path, /, *, only_once=True): - filepath = Path(arg_path) - stem = Path(filepath.stem) - while not only_once and stem.suffixes and '' != stem.suffix: - stem = Path(stem.stem) - stem = str(stem) - return (filepath.parent, stem,) - - def mkdir_p(arg_path, mode=0o777): ''' Reminder: mode only affects the last directory From 08800a0c71d35ac2fd59a97af44f26ce2b348e70 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 10:52:30 -0400 Subject: [PATCH 63/67] fixup: import Path --- tubesync/sync/models/_private.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/models/_private.py b/tubesync/sync/models/_private.py index baca9e11..8cf41ce1 100644 --- a/tubesync/sync/models/_private.py +++ b/tubesync/sync/models/_private.py @@ -1,3 +1,4 @@ +from pathlib import Path from ..choices import Val, YouTube_SourceType # noqa From 00f2795b086486df8906ca0932758c6f084d3142 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 22 May 2025 10:55:40 -0400 Subject: [PATCH 64/67] Fail when `ruff` finds errors --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 22bd31b2..5ae58c0a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -144,7 +144,7 @@ jobs: done } >> "${GITHUB_STEP_SUMMARY}" uvx --no-config --no-managed-python --no-progress --isolated \ - ruff check --exit-zero \ + ruff check \ --target-version "${target_version}" \ --output-format github \ --ignore "${ignore_csv_list}" From b0721fea80a7c65def8450e45c749fa589fadc17 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 23 May 2025 01:03:30 -0400 Subject: [PATCH 65/67] Tighten `errors_qs` by checking `last_error` --- tubesync/sync/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 4f683834..093cad3f 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -845,9 +845,11 @@ class TasksView(ListView): data = super().get_context_data(*args, **kwargs) now = timezone.now() qs = Task.objects.all() - errors_qs = qs.filter(attempts__gt=0, locked_by__isnull=True) running_qs = qs.filter(locked_by__isnull=False) scheduled_qs = qs.filter(locked_by__isnull=True) + errors_qs = scheduled_qs.filter( + attempts__gt=0 + ).exclude(last_error__exact='') # Add to context data from ListView data['message'] = self.message From 3e310aa9f94eb3a2399165070d54d9756f958267 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 23 May 2025 10:04:50 -0400 Subject: [PATCH 66/67] `yt-dlp-get-pot` was recently superceded by a framework in `yt-dlp` --- Pipfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Pipfile b/Pipfile index 2e3f3acf..49b5127b 100644 --- a/Pipfile +++ b/Pipfile @@ -24,5 +24,4 @@ yt-dlp = {extras = ["default", "curl-cffi"], version = "*"} emoji = "*" brotli = "*" html5lib = "*" -yt-dlp-get-pot = "*" bgutil-ytdlp-pot-provider = "*" From 0830812e2133e99a1adcbc27f53777b25c469967 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 23 May 2025 10:28:56 -0400 Subject: [PATCH 67/67] Bump the version --- tubesync/tubesync/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 3ab7f9ff..cd6d1dca 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -8,7 +8,7 @@ CONFIG_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR -VERSION = '0.15.3' +VERSION = '0.15.4' SECRET_KEY = '' DEBUG = False ALLOWED_HOSTS = []