wimby/tubesync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into patch-14

Browse Source
This commit is contained in:
tcely
2025-05-23 11:02:31 -04:00
committed by GitHub
parent 3effc2ad96 75aef2efdd
commit a7ede7d965
31 changed files with 177 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci.yaml vendored
View File

@@ -144,7 +144,7 @@ jobs:
done
} >> "${GITHUB_STEP_SUMMARY}"
uvx --no-config --no-managed-python --no-progress --isolated \
ruff check --exit-zero \
ruff check \
--target-version "${target_version}" \
--output-format github \
--ignore "${ignore_csv_list}"

6
Dockerfile
View File

@@ -410,6 +410,12 @@ RUN --mount=type=tmpfs,target=/cache \
PIPENV_VERBOSITY=64 \
PYTHONPYCACHEPREFIX=/cache/pycache \
pipenv install --system --skip-lock && \
# remove the getpot_bgutil_script plugin
find /usr/local/lib \
-name 'getpot_bgutil_script.py' \
-path '*/yt_dlp_plugins/extractor/getpot_bgutil_script.py' \
-type f -print -delete \
&& \
# Clean up
apt-get -y autoremove --purge \
default-libmysqlclient-dev \

1
Pipfile
View File

@@ -24,5 +24,4 @@ yt-dlp = {extras = ["default", "curl-cffi"], version = "*"}
emoji = "*"
brotli = "*"
html5lib = "*"
yt-dlp-get-pot = "*"
bgutil-ytdlp-pot-provider = "*"

10
tubesync/common/json.py
View File

@@ -1,4 +1,6 @@
from datetime import datetime
from django.core.serializers.json import DjangoJSONEncoder
from yt_dlp.utils import LazyList
class JSONEncoder(DjangoJSONEncoder):
@@ -14,3 +16,11 @@ class JSONEncoder(DjangoJSONEncoder):
return list(iterable)
return super().default(obj)
def json_serial(obj):
if isinstance(obj, datetime):
return obj.isoformat()
if isinstance(obj, LazyList):
return list(obj)
raise TypeError(f'Type {type(obj)} is not json_serial()-able')

1
tubesync/common/urls.py
View File

@@ -1,7 +1,6 @@
from django.conf import settings
from django.urls import path
from django.views.generic.base import RedirectView
from django.views.generic import TemplateView
from django.http import HttpResponse
from .views import error403, error404, error500, HealthCheckView

29
tubesync/common/utils.py
View File

@@ -6,10 +6,8 @@ import os
import pstats
import string
import time
from datetime import datetime
from django.core.paginator import Paginator
from urllib.parse import urlunsplit, urlencode, urlparse
from yt_dlp.utils import LazyList
from .errors import DatabaseConnectionError
@@ -84,14 +82,11 @@ def parse_database_connection_string(database_connection_string):
f'invalid driver, must be one of {valid_drivers}')
django_driver = django_backends.get(driver)
host_parts = user_pass_host_port.split('@')
if len(host_parts) != 2:
raise DatabaseConnectionError(f'Database connection string netloc must be in '
f'the format of user:pass@host')
user_pass_parts = host_parts[0].split(':')
if len(host_parts) != 2 or len(user_pass_parts) != 2:
raise DatabaseConnectionError('Database connection string netloc must be in '
'the format of user:pass@host')
user_pass, host_port = host_parts
user_pass_parts = user_pass.split(':')
if len(user_pass_parts) != 2:
raise DatabaseConnectionError(f'Database connection string netloc must be in '
f'the format of user:pass@host')
username, password = user_pass_parts
host_port_parts = host_port.split(':')
if len(host_port_parts) == 1:
@@ -113,13 +108,13 @@ def parse_database_connection_string(database_connection_string):
f'65535, got {port}')
else:
# Malformed
raise DatabaseConnectionError(f'Database connection host must be a hostname or '
f'a hostname:port combination')
raise DatabaseConnectionError('Database connection host must be a hostname or '
'a hostname:port combination')
if database.startswith('/'):
database = database[1:]
if not database:
raise DatabaseConnectionError(f'Database connection string path must be a '
f'string in the format of /databasename')
raise DatabaseConnectionError('Database connection string path must be a '
'string in the format of /databasename')
if '/' in database:
raise DatabaseConnectionError(f'Database connection string path can only '
f'contain a single string name, got: {database}')
@@ -172,14 +167,6 @@ def clean_emoji(s):
return emoji.replace_emoji(s)
def json_serial(obj):
if isinstance(obj, datetime):
return obj.isoformat()
if isinstance(obj, LazyList):
return list(obj)
raise TypeError(f'Type {type(obj)} is not json_serial()-able')
def time_func(func):
def wrapper(*args, **kwargs):
start = time.perf_counter()

2
tubesync/sync/fields.py
View File

@@ -63,7 +63,7 @@ class CommaSepChoiceField(models.CharField):
def __init__(self, *args, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, **kwargs):
kwargs.setdefault('max_length', 128)
self.separator = str(separator)
self.possible_choices = possible_choices or choices
self.possible_choices = possible_choices or kwargs.get('choices')
self.selected_choices = list()
self.allow_all = allow_all
self.all_label = all_label

1
tubesync/sync/filtering.py
View File

@@ -5,7 +5,6 @@
from common.logger import log
from .models import Media
from datetime import datetime
from django.utils import timezone
from .overrides.custom_filter import filter_custom

2
tubesync/sync/hooks.py
View File

@@ -1,9 +1,7 @@
import os
import yt_dlp
from common.logger import log
from common.utils import remove_enclosed
from django.conf import settings
progress_hook = {

7
tubesync/sync/management/commands/delete-source.py
View File

@@ -1,16 +1,15 @@
import os
import uuid
from django.utils.translation import gettext_lazy as _
from django.core.management.base import BaseCommand, CommandError
from django.db.transaction import atomic
from django.utils.translation import gettext_lazy as _
from common.logger import log
from sync.models import Source, Media, MediaServer
from sync.models import Source
from sync.tasks import schedule_media_servers_update
class Command(BaseCommand):
help = _('Deletes a source by UUID')
help = 'Deletes a source by UUID'
def add_arguments(self, parser):
parser.add_argument('--source', action='store', required=True, help=_('Source UUID'))

4
tubesync/sync/management/commands/import-existing-media.py
View File

@@ -1,6 +1,6 @@
import os
from pathlib import Path
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand, CommandError # noqa
from common.logger import log
from common.timestamp import timestamp_to_datetime
from sync.choices import FileExtension
@@ -18,7 +18,7 @@ class Command(BaseCommand):
dirmap = {}
for s in Source.objects.all():
dirmap[str(s.directory_path)] = s
log.info(f'Scanning sources...')
log.info('Scanning sources...')
file_extensions = list(FileExtension.values) + self.extra_extensions
for sourceroot, source in dirmap.items():
media = list(Media.objects.filter(source=source, downloaded=False,

5
tubesync/sync/management/commands/list-sources.py
View File

@@ -1,7 +1,6 @@
import os
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand, CommandError # noqa
from common.logger import log
from sync.models import Source, Media, MediaServer
from sync.models import Source
class Command(BaseCommand):

10
tubesync/sync/management/commands/reset-tasks.py
View File

@@ -1,14 +1,12 @@
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand, CommandError # noqa
from django.db.transaction import atomic
from django.utils.translation import gettext_lazy as _
from background_task.models import Task
from common.logger import log
from sync.models import Source
from sync.tasks import index_source_task, check_source_directory_exists
from common.logger import log
class Command(BaseCommand):
help = 'Resets all tasks'
@@ -31,10 +29,10 @@ class Command(BaseCommand):
index_source_task(
str(source.pk),
repeat=source.index_schedule,
schedule=source.index_schedule,
verbose_name=verbose_name.format(source.name),
)
with atomic(durable=True):
for source in Source.objects.all():
# This also chains down to call each Media objects .save() as well
source.save()
log.info('Done')

3
tubesync/sync/management/commands/sync-missing-metadata.py
View File

@@ -1,6 +1,5 @@
import os
from shutil import copyfile
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand, CommandError # noqa
from django.db.models import Q
from common.logger import log
from sync.models import Source, Media

6
tubesync/sync/management/commands/youtube-dl-info.py
View File

@@ -1,7 +1,7 @@
import json
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand, CommandError # noqa
from sync.youtube import get_media_info
from common.utils import json_serial
from common.json import JSONEncoder
class Command(BaseCommand):
@@ -15,6 +15,6 @@ class Command(BaseCommand):
url = options['url']
self.stdout.write(f'Showing information for URL: {url}')
info = get_media_info(url)
d = json.dumps(info, indent=4, sort_keys=True, default=json_serial)
d = json.dumps(info, indent=4, sort_keys=True, cls=JSONEncoder)
self.stdout.write(d)
self.stdout.write('Done')

3
tubesync/sync/mediaservers.py
View File

@@ -117,9 +117,6 @@ class PlexMediaServer(MediaServer):
raise ValidationError('Plex Media Server "port" must be between 1 '
'and 65535')
options = self.object.options
if 'token' not in options:
raise ValidationError('Plex Media Server requires a "token"')
token = options['token'].strip()
if 'token' not in options:
raise ValidationError('Plex Media Server requires a "token"')
if 'libraries' not in options:

2
tubesync/sync/migrations/0011_auto_20220201_1654.py
View File

@@ -1,8 +1,6 @@
# Generated by Django 3.2.11 on 2022-02-01 16:54
import django.core.files.storage
from django.db import migrations, models
import sync.models
class Migration(migrations.Migration):

2
tubesync/sync/migrations/0013_fix_elative_media_file.py
View File

@@ -1,7 +1,7 @@
# Generated by Django 3.2.12 on 2022-04-06 06:19
from django.conf import settings
from django.db import migrations, models
from django.db import migrations
def fix_media_file(apps, schema_editor):

6
tubesync/sync/models/__init__.py
View File

@@ -17,3 +17,9 @@ from .media import Media
from .metadata import Metadata
from .metadata_format import MetadataFormat
__all__ = [
'get_media_file_path', 'get_media_thumb_path',
'media_file_storage', 'MediaServer', 'Source',
'Media', 'Metadata', 'MetadataFormat',
]

11
tubesync/sync/models/_private.py
View File

@@ -1,4 +1,5 @@
from ..choices import Val, YouTube_SourceType
from pathlib import Path
from ..choices import Val, YouTube_SourceType # noqa
_srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) ))
@@ -10,3 +11,11 @@ def _nfo_element(nfo, label, text, /, *, attrs={}, tail='\n', char=' ', indent=2
element.tail = tail + (char * indent)
return element
def directory_and_stem(arg_path, /, all_suffixes=False):
filepath = Path(arg_path)
stem = Path(filepath.stem)
while all_suffixes and stem.suffixes and '' != stem.suffix:
stem = Path(stem.stem)
stem = str(stem)
return (filepath.parent, stem,)

16
tubesync/sync/models/media.py
View File

@@ -15,9 +15,9 @@ from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from common.logger import log
from common.errors import NoFormatException
from common.json import JSONEncoder
from common.utils import (
clean_filename, clean_emoji,
django_queryset_generator as qs_gen,
)
from ..youtube import (
get_media_info as get_youtube_media_info,
@@ -25,8 +25,7 @@ from ..youtube import (
)
from ..utils import (
seconds_to_timestr, parse_media_format, filter_response,
write_text_file, mkdir_p, directory_and_stem, glob_quote,
multi_key_sort,
write_text_file, mkdir_p, glob_quote, multi_key_sort,
)
from ..matching import (
get_best_combined_format,
@@ -39,7 +38,7 @@ from ..choices import (
from ._migrations import (
media_file_storage, get_media_thumb_path, get_media_file_path,
)
from ._private import _srctype_dict, _nfo_element
from ._private import _srctype_dict, _nfo_element, directory_and_stem
from .media__tasks import (
download_checklist, download_finished, wait_for_premiere,
)
@@ -578,14 +577,13 @@ class Media(models.Model):
def metadata_dumps(self, arg_dict=dict()):
from common.utils import json_serial
fallback = dict()
try:
fallback.update(self.new_metadata.with_formats)
except ObjectDoesNotExist:
pass
data = arg_dict or fallback
return json.dumps(data, separators=(',', ':'), default=json_serial)
return json.dumps(data, separators=(',', ':'), cls=JSONEncoder)
def metadata_loads(self, arg_str='{}'):
@@ -688,7 +686,7 @@ class Media(models.Model):
pass
setattr(self, '_cached_metadata_dict', data)
return data
except Exception as e:
except Exception:
return {}
@@ -1194,7 +1192,7 @@ class Media(models.Model):
other_path.replace(new_file_path)
for fuzzy_path in fuzzy_paths:
(fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path)
(fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path, True)
old_file_str = fuzzy_path.name
new_file_str = new_stem + old_file_str[len(fuzzy_stem):]
new_file_path = Path(new_prefix_path / new_file_str)
@@ -1219,7 +1217,7 @@ class Media(models.Model):
parent_dir.rmdir()
log.info(f'Removed empty directory: {parent_dir!s}')
parent_dir = parent_dir.parent
except OSError as e:
except OSError:
pass

3
tubesync/sync/models/media__tasks.py
View File

@@ -1,9 +1,11 @@
import os
from pathlib import Path
from common.logger import log
from common.errors import (
NoMetadataException,
)
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from ..choices import Val, SourceResolution
@@ -47,6 +49,7 @@ def download_checklist(self, skip_checks=False):
f'the source has a download cap and the media is now too old, '
f'not downloading')
return False
return True
def download_finished(self, format_str, container, downloaded_filepath=None):

27
tubesync/sync/models/source.py
View File

@@ -1,6 +1,7 @@
import os
import re
import uuid
from collections import deque as queue
from pathlib import Path
from django import db
from django.conf import settings
@@ -510,7 +511,7 @@ class Source(db.models.Model):
def get_example_media_format(self):
try:
return self.media_format.format(**self.example_media_format_dict)
except Exception as e:
except Exception:
return ''
def is_regex_match(self, media_item_title):
@@ -527,23 +528,31 @@ class Source(db.models.Model):
days = timezone.timedelta(seconds=self.download_cap).days
response = indexer(self.get_index_url(type=type), days=days)
if not isinstance(response, dict):
return []
entries = response.get('entries', [])
return list()
entries = response.get('entries', list())
return entries
def index_media(self):
'''
Index the media source returning a list of media metadata as dicts.
Index the media source returning a queue of media metadata as dicts.
'''
entries = list()
entries = queue(list(), getattr(settings, 'MAX_ENTRIES_PROCESSING', 0) or None)
if self.index_videos:
entries += self.get_index('videos')
entries.extend(reversed(self.get_index('videos')))
# Playlists do something different that I have yet to figure out
if not self.is_playlist:
if self.index_streams:
entries += self.get_index('streams')
streams = self.get_index('streams')
if entries.maxlen is None or 0 == len(entries):
entries.extend(reversed(streams))
else:
# share the queue between streams and videos
allowed_streams = max(
entries.maxlen // 2,
entries.maxlen - len(entries),
)
entries.extend(reversed(streams[: allowed_streams]))
if settings.MAX_ENTRIES_PROCESSING:
entries = entries[:settings.MAX_ENTRIES_PROCESSING]
return entries

10
tubesync/sync/signals.py
View File

@@ -10,11 +10,11 @@ from django.utils.translation import gettext_lazy as _
from background_task.signals import task_failed
from background_task.models import Task
from common.logger import log
from .models import Source, Media, MediaServer, Metadata
from .models import Source, Media, Metadata
from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task,
download_media_thumbnail, download_media_metadata,
map_task_to_instance, check_source_directory_exists,
download_media, rescan_media_server, download_source_images,
download_media, download_source_images,
delete_all_media_for_source, save_all_media_for_source,
rename_media, get_media_metadata_task, get_media_download_task)
from .utils import delete_file, glob_quote, mkdir_p
@@ -270,7 +270,7 @@ def media_post_save(sender, instance, created, **kwargs):
if not (media_file_exists or existing_media_download_task):
# The file was deleted after it was downloaded, skip this media.
if instance.can_download and instance.downloaded:
skip_changed = True != instance.skip
skip_changed = True if not instance.skip else False
instance.skip = True
downloaded = False
if (instance.source.download_media and instance.can_download) and not (
@@ -374,13 +374,13 @@ def media_post_delete(sender, instance, **kwargs):
try:
p.rmdir()
log.info(f'Deleted directory for: {instance} path: {p!s}')
except OSError as e:
except OSError:
pass
# Delete the directory itself
try:
other_path.rmdir()
log.info(f'Deleted directory for: {instance} path: {other_path!s}')
except OSError as e:
except OSError:
pass
# Get all files that start with the bare file path
all_related_files = video_path.parent.glob(f'{glob_quote(video_path.with_suffix("").name)}*')

73
tubesync/sync/tasks.py
View File

@@ -6,7 +6,6 @@
import os
import json
import math
import random
import requests
import time
@@ -14,9 +13,8 @@ import uuid
from io import BytesIO
from hashlib import sha1
from pathlib import Path
from datetime import datetime, timedelta
from datetime import timedelta
from shutil import copyfile, rmtree
from PIL import Image
from django import db
from django.conf import settings
from django.core.files.base import ContentFile
@@ -30,14 +28,14 @@ from background_task.exceptions import InvalidTaskError
from background_task.models import Task, CompletedTask
from common.logger import log
from common.errors import ( NoFormatException, NoMediaException,
NoMetadataException, NoThumbnailException,
NoThumbnailException,
DownloadFailedException, )
from common.utils import ( django_queryset_generator as qs_gen,
remove_enclosed, )
from .choices import Val, TaskQueue
from .models import Source, Media, MediaServer
from .utils import ( get_remote_image, resize_image_to_height, delete_file,
write_text_file, filter_response, )
from .utils import ( get_remote_image, resize_image_to_height,
write_text_file, filter_response, seconds_to_timestr, )
from .youtube import YouTubeError
db_vendor = db.connection.vendor
@@ -122,7 +120,7 @@ def get_error_message(task):
def update_task_status(task, status):
if not task:
return False
if not task._verbose_name:
if not hasattr(task, '_verbose_name'):
task._verbose_name = remove_enclosed(
task.verbose_name, '[', ']', ' ',
)
@@ -226,7 +224,7 @@ def save_model(instance):
@atomic(durable=False)
def schedule_media_servers_update():
# Schedule a task to update media servers
log.info(f'Scheduling media server updates')
log.info('Scheduling media server updates')
verbose_name = _('Request media server rescan for "{}"')
for mediaserver in MediaServer.objects.all():
rescan_media_server(
@@ -235,6 +233,38 @@ def schedule_media_servers_update():
)
def wait_for_errors(model, /, *, task_name=None):
if task_name is None:
task_name=tuple((
'sync.tasks.download_media',
'sync.tasks.download_media_metadata',
))
elif isinstance(task_name, str):
task_name = tuple((task_name,))
tasks = list()
for tn in task_name:
ft = get_first_task(tn, instance=model)
if ft:
tasks.append(ft)
window = timezone.timedelta(hours=3) + timezone.now()
tqs = Task.objects.filter(
task_name__in=task_name,
attempts__gt=0,
locked_at__isnull=True,
run_at__lte=window,
last_error__contains='HTTPError 429: Too Many Requests',
)
for task in tasks:
update_task_status(task, 'paused (429)')
delay = 10 * tqs.count()
time_str = seconds_to_timestr(delay)
log.info(f'waiting for errors: 429 ({time_str}): {model}')
time.sleep(delay)
for task in tasks:
update_task_status(task, None)
def cleanup_old_media():
with atomic():
for source in qs_gen(Source.objects.filter(delete_old_media=True, days_to_keep__gt=0)):
@@ -255,7 +285,7 @@ def cleanup_old_media():
schedule_media_servers_update()
def cleanup_removed_media(source, videos):
def cleanup_removed_media(source, video_keys):
if not source.delete_removed_media:
return
log.info(f'Cleaning up media no longer in source: {source}')
@@ -265,8 +295,7 @@ def cleanup_removed_media(source, videos):
source=source,
)
for media in qs_gen(mqs):
matching_source_item = [video['id'] for video in videos if video['id'] == media.key]
if not matching_source_item:
if media.key not in video_keys:
log.info(f'{media.name} is no longer in source, removing')
with atomic():
media.delete()
@@ -316,12 +345,17 @@ def index_source_task(source_id):
end=task.verbose_name.find('Index'),
)
tvn_format = '{:,}' + f'/{num_videos:,}'
for vn, video in enumerate(videos, start=1):
vn = 0
video_keys = set()
while len(videos) > 0:
vn += 1
video = videos.popleft()
# Create or update each video as a Media object
key = video.get(source.key_field, None)
if not key:
# Video has no unique key (ID), it can't be indexed
continue
video_keys.add(key)
update_task_status(task, tvn_format.format(vn))
# media, new_media = Media.objects.get_or_create(key=key, source=source)
try:
@@ -376,7 +410,7 @@ def index_source_task(source_id):
# Reset task.verbose_name to the saved value
update_task_status(task, None)
# Cleanup of media no longer available from the source
cleanup_removed_media(source, videos)
cleanup_removed_media(source, video_keys)
videos = video = None
@@ -416,7 +450,7 @@ def download_source_images(source_id):
log.info(f'Thumbnail URL for source with ID: {source_id} / {source} '
f'Avatar: {avatar} '
f'Banner: {banner}')
if banner != None:
if banner is not None:
url = banner
i = get_remote_image(url)
image_file = BytesIO()
@@ -432,7 +466,7 @@ def download_source_images(source_id):
f.write(django_file.read())
i = image_file = None
if avatar != None:
if avatar is not None:
url = avatar
i = get_remote_image(url)
image_file = BytesIO()
@@ -467,6 +501,7 @@ def download_media_metadata(media_id):
log.info(f'Task for ID: {media_id} / {media} skipped, due to task being manually skipped.')
return
source = media.source
wait_for_errors(media, task_name='sync.tasks.download_media_metadata')
try:
metadata = media.index_metadata()
except YouTubeError as e:
@@ -615,8 +650,11 @@ def download_media(media_id, override=False):
raise InvalidTaskError(_('no such media')) from e
else:
if not media.download_checklist(override):
# any condition that needs to reschedule the task
# should raise an exception to avoid this
return
wait_for_errors(media, task_name='sync.tasks.download_media')
filepath = media.filepath
container = format_str = None
log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"')
@@ -832,8 +870,7 @@ def wait_for_media_premiere(media_id):
if hours:
task = get_media_premiere_task(media_id)
if task:
update_task_status(task, f'available in {hours} hours')
update_task_status(task, f'available in {hours} hours')
save_model(media)
@@ -845,7 +882,7 @@ def delete_all_media_for_source(source_id, source_name, source_directory):
assert source_directory
try:
source = Source.objects.get(pk=source_id)
except Source.DoesNotExist as e:
except Source.DoesNotExist:
# Task triggered but the source no longer exists, do nothing
log.warn(f'Task delete_all_media_for_source(pk={source_id}) called but no '
f'source exists with ID: {source_id}')

3
tubesync/sync/templates/sync/media-item.html
View File

@@ -121,7 +121,8 @@
{% if media_file_path == media.filepath %}
<span class="green-text">&nbsp;(matched)</span>
{% endif %}
</td> </tr>
</td>
</tr>
<tr title="Size of the file on disk">
<td class="hide-on-small-only">File size</td>
<td><span class="hide-on-med-and-up">File size<br></span><strong>{{ media.downloaded_filesize|bytesformat }}</strong></td>

11
tubesync/sync/tests.py
View File

@@ -6,7 +6,6 @@
import logging
import os
from datetime import datetime, timedelta
from pathlib import Path
from urllib.parse import urlsplit
@@ -1822,13 +1821,13 @@ class TasksTestCase(TestCase):
now = timezone.now()
m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5))
m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25))
m13 = Media.objects.create(source=src1, downloaded=False, key='a13')
m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5)) # noqa
m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25)) # noqa
m13 = Media.objects.create(source=src1, downloaded=False, key='a13') # noqa
m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5))
m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5)) # noqa
m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25))
m23 = Media.objects.create(source=src2, downloaded=False, key='a23')
m23 = Media.objects.create(source=src2, downloaded=False, key='a23') # noqa
self.assertEqual(src1.media_source.all().count(), 3)
self.assertEqual(src2.media_source.all().count(), 3)

9
tubesync/sync/utils.py
View File

@@ -130,15 +130,6 @@ def file_is_editable(filepath):
return False
def directory_and_stem(arg_path):
filepath = Path(arg_path)
stem = Path(filepath.stem)
while stem.suffixes and '' != stem.suffix:
stem = Path(stem.stem)
stem = str(stem)
return (filepath.parent, stem,)
def mkdir_p(arg_path, mode=0o777):
'''
Reminder: mode only affects the last directory

25
tubesync/sync/views.py
View File

@@ -35,7 +35,7 @@ from .tasks import (map_task_to_instance, get_error_message,
from .choices import (Val, MediaServerType, SourceResolution, IndexSchedule,
YouTube_SourceType, youtube_long_source_types,
youtube_help, youtube_validation_urls)
from . import signals
from . import signals # noqa
from . import youtube
@@ -258,7 +258,7 @@ class ValidateSourceView(FormView):
self.key = youtube.get_channel_id(
index_url.replace('/channel/', '/')
)
except youtube.YouTubeError as e:
except youtube.YouTubeError:
# It did not work, revert to previous behavior
self.key = old_key
self.source_type = old_source_type
@@ -296,10 +296,13 @@ class EditSourceMixin:
def form_valid(self, form: Form):
# Perform extra validation to make sure the media_format is valid
obj = form.save(commit=False)
source_type = form.cleaned_data['media_format']
# temporarily use media_format from the form
saved_media_format = obj.media_format
obj.media_format = form.cleaned_data['media_format']
example_media_file = obj.get_example_media_format()
obj.media_format = saved_media_format
if example_media_file == '':
if '' == example_media_file:
form.add_error(
'media_format',
ValidationError(self.errors['invalid_media_format'])
@@ -307,12 +310,16 @@ class EditSourceMixin:
# Check for suspicious file path(s)
try:
targetCheck = form.cleaned_data['directory']+"/.virt"
newdir = safe_join(settings.DOWNLOAD_ROOT,targetCheck)
targetCheck = form.cleaned_data['directory'] + '/.virt'
safe_join(settings.DOWNLOAD_ROOT, targetCheck)
except SuspiciousFileOperation:
form.add_error(
'directory',
ValidationError(self.errors['dir_outside_dlroot'].replace("%BASEDIR%",str(settings.DOWNLOAD_ROOT)))
ValidationError(
self.errors['dir_outside_dlroot'].replace(
"%BASEDIR%", str(settings.DOWNLOAD_ROOT)
)
),
)
if form.errors:
@@ -838,9 +845,11 @@ class TasksView(ListView):
data = super().get_context_data(*args, **kwargs)
now = timezone.now()
qs = Task.objects.all()
errors_qs = qs.filter(attempts__gt=0, locked_by__isnull=True)
running_qs = qs.filter(locked_by__isnull=False)
scheduled_qs = qs.filter(locked_by__isnull=True)
errors_qs = scheduled_qs.filter(
attempts__gt=0
).exclude(last_error__exact='')
# Add to context data from ListView
data['message'] = self.message

14
tubesync/sync/youtube.py
View File

@@ -6,7 +6,6 @@
import os
from collections import namedtuple
from common.logger import log
from copy import deepcopy
from pathlib import Path
@@ -102,7 +101,7 @@ def get_channel_image_info(url):
avatar_url = thumbnail['url']
if thumbnail['id'] == 'banner_uncropped':
banner_url = thumbnail['url']
if banner_url != None and avatar_url != None:
if banner_url is not None and avatar_url is not None:
break
return avatar_url, banner_url
@@ -143,7 +142,7 @@ def get_media_info(url, /, *, days=None, info_json=None):
if days is not None:
try:
days = int(str(days), 10)
except Exception as e:
except (TypeError, ValueError):
days = None
start = (
f'yesterday-{days!s}days' if days else None
@@ -326,13 +325,10 @@ def download_media(
final_path = Path(output_file).resolve(strict=False)
expected_file = shell_quote(str(final_path))
cmds = pp_opts.exec_cmd.get('after_move', list())
# It is important that we use a tuple for strings.
# Otherwise, list adds each character instead.
# That last comma is really necessary!
cmds += (
cmds.append(
f'test -f {expected_file} || '
'mv -T -u -- %(filepath,_filename|)q '
f'{expected_file}',
f'{expected_file}'
)
# assignment is the quickest way to cover both 'get' cases
pp_opts.exec_cmd['after_move'] = cmds
@@ -387,7 +383,7 @@ def download_media(
youtube_ea_dict = ytopts['extractor_args'].get('youtube', dict())
formats_list = youtube_ea_dict.get('formats', list())
if 'missing_pot' not in formats_list:
formats_list += ('missing_pot',)
formats_list.append('missing_pot')
youtube_ea_dict.update({
'formats': formats_list,
})

4
tubesync/tubesync/settings.py
View File

@@ -198,7 +198,7 @@ RENAME_SOURCES = list()
# You have been warned!
try:
from .local_settings import *
from .local_settings import * # noqa
except ImportError as e:
import sys
sys.stderr.write(f'Unable to import local_settings: {e}\n')
@@ -222,5 +222,5 @@ if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS:
BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS
from .dbutils import patch_ensure_connection
from .dbutils import patch_ensure_connection # noqa
patch_ensure_connection()