Merge branch 'main' into patch-14

Dockerfile

@@ -58,6 +58,8 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
     apt-get -y autoclean && \
     rm -f /var/cache/debconf/*.dat-old
 
+FROM ghcr.io/astral-sh/uv:latest AS uv-binaries
+
 FROM alpine:${ALPINE_VERSION} AS openresty-debian
 ARG TARGETARCH
 ARG DEBIAN_VERSION
@@ -279,6 +281,9 @@ RUN set -eu ; \
 FROM scratch AS s6-overlay
 COPY --from=s6-overlay-extracted /s6-overlay-rootfs /
 
+FROM tubesync-base AS tubesync-uv
+COPY --from=uv-binaries /uv /uvx /usr/local/bin/
+
 FROM tubesync-base AS tubesync-openresty
 
 COPY --from=openresty-debian \
@@ -290,7 +295,10 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
     --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \
   set -x && \
   apt-get update && \
-  apt-get -y --no-install-recommends install nginx-common openresty && \
+  apt-get -y --no-install-recommends install \
+    nginx-common \
+    openresty \
+  && \
   # Clean up
   apt-get -y autopurge && \
   apt-get -y autoclean && \
@@ -302,7 +310,10 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
     --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \
   set -x && \
   apt-get update && \
-  apt-get -y --no-install-recommends install nginx-light && \
+  apt-get -y --no-install-recommends install \
+    nginx-light \
+    libnginx-mod-http-lua \
+  && \
   # openresty binary should still work
   ln -v -s -T ../sbin/nginx /usr/bin/openresty && \
   # Clean up
@@ -335,12 +346,11 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
   libmariadb3 \
   libpq5 \
   libwebp7 \
-  pipenv \
   pkgconf \
   python3 \
   python3-libsass \
+  python3-pip-whl \
   python3-socks \
-  python3-wheel \
   curl \
   less \
   && \
@@ -383,9 +393,11 @@ ARG YTDLP_DATE
 
 # Set up the app
 RUN --mount=type=tmpfs,target=/cache \
+    --mount=type=cache,id=uv-cache,sharing=locked,target=/cache/uv \
     --mount=type=cache,id=pipenv-cache,sharing=locked,target=/cache/pipenv \
     --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/var/lib/apt \
     --mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \
+    --mount=type=bind,source=/uv,target=/usr/local/bin/uv,from=uv-binaries \
     --mount=type=bind,source=Pipfile,target=/app/Pipfile \
   set -x && \
   apt-get update && \
@@ -400,16 +412,32 @@ RUN --mount=type=tmpfs,target=/cache \
   make \
   postgresql-common \
   python3-dev \
-  python3-pip \
   zlib1g-dev \
   && \
   # Install non-distro packages
-  cp -at /tmp/ "${HOME}" && \
+  mkdir -v -p /cache/.home-directories && \
-  HOME="/tmp/${HOME#/}" \
+  cp -at /cache/.home-directories/ "${HOME}" && \
+  HOME="/cache/.home-directories/${HOME#/}" \
   XDG_CACHE_HOME='/cache' \
   PIPENV_VERBOSITY=64 \
   PYTHONPYCACHEPREFIX=/cache/pycache \
-    pipenv install --system --skip-lock && \
+  uv tool run --no-config --no-progress --no-managed-python -- \
+    pipenv lock && \
+  HOME="/cache/.home-directories/${HOME#/}" \
+  XDG_CACHE_HOME='/cache' \
+  PIPENV_VERBOSITY=1 \
+  PYTHONPYCACHEPREFIX=/cache/pycache \
+  uv tool run --no-config --no-progress --no-managed-python -- \
+    pipenv requirements --from-pipfile --hash >| /cache/requirements.txt && \
+  rm -v Pipfile.lock && \
+  cat -v /cache/requirements.txt && \
+  HOME="/cache/.home-directories/${HOME#/}" \
+  UV_LINK_MODE='copy' \
+  XDG_CACHE_HOME='/cache' \
+  PYTHONPYCACHEPREFIX=/cache/pycache \
+    uv --no-config --no-progress --no-managed-python \
+    pip install --strict --system --break-system-packages \
+    --requirements /cache/requirements.txt && \
   # remove the getpot_bgutil_script plugin
   find /usr/local/lib \
   -name 'getpot_bgutil_script.py' \
@@ -427,7 +455,6 @@ RUN --mount=type=tmpfs,target=/cache \
   make \
   postgresql-common \
   python3-dev \
-  python3-pip \
   zlib1g-dev \
   && \
   apt-get -y autopurge && \

config/root/etc/nginx/nginx.conf

@@ -1,3 +1,7 @@
+# load_module snippets for installed modules
+include /etc/nginx/modules-enabled/*.conf;
+# a very silly syntax rendering bug needs this */
+
 daemon off;
 
 user app;

tubesync/sync/choices.py

@@ -167,6 +167,28 @@ class TaskQueue(models.TextChoices):
     NET = 'network', _('Networking')
 
 
+class WeekDay(models.IntegerChoices):
+    MON = 0, _('Monday')
+    TUE = 1, _('Tuesday')
+    WED = 2, _('Wednesday')
+    THU = 3, _('Thursday')
+    FRI = 4, _('Friday')
+    SAT = 5, _('Saturday')
+    SUN = 6, _('Sunday')
+
+    @classmethod
+    def get(cls, wdn, /):
+        return cls[cls.names[wdn]]
+
+    @classmethod
+    def _from_iso(cls, wdn, /):
+        return cls[cls.names[wdn - 1]]
+
+    @classmethod
+    def _to_iso(cls, choice, /):
+        return 1 + choice.value
+
+
 class YouTube_SourceType(models.TextChoices):
     CHANNEL = 'c', _('YouTube channel')
     CHANNEL_ID = 'i', _('YouTube channel by ID')

tubesync/sync/forms.py

@@ -2,13 +2,33 @@
 from django import forms, VERSION as DJANGO_VERSION
 from django.utils.translation import gettext_lazy as _
 
+from .models import Source
+
 
 if DJANGO_VERSION[0:3] < (5, 0, 0):
     _assume_scheme = dict()
 else:
     # Silence RemovedInDjango60Warning
     _assume_scheme = dict(assume_scheme='http')
-    
+
+SourceForm = forms.modelform_factory(
+    Source,
+    # manual ordering
+    fields = (
+        'source_type', 'key', 'name', 'directory', 'filter_text', 'filter_text_invert', 'filter_seconds', 'filter_seconds_min',
+        'media_format', 'target_schedule', 'index_schedule', 'index_videos', 'index_streams', 'download_media',
+        'download_cap', 'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
+        'prefer_60fps', 'prefer_hdr', 'fallback', 'delete_removed_media', 'delete_files_on_disk', 'copy_channel_images',
+        'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
+        'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs',
+    ),
+    widgets = {
+        'target_schedule': forms.DateTimeInput(
+            attrs={'type': 'datetime-local'},
+        ),
+    },
+)
+
 class ValidateSourceForm(forms.Form):
 
     source_type = forms.CharField(

tubesync/sync/hooks.py

@@ -2,6 +2,7 @@ import os
 
 from common.logger import log
 from common.utils import remove_enclosed
+from sync.utils import filter_response
 
 
 progress_hook = {
@@ -212,11 +213,15 @@ def yt_dlp_postprocessor_hook(event):
     status.media_name = name
 
     if 'started' == event['status']:
-        if 'formats' in event['info_dict']:
+        remove_keys = frozenset({
-            del event['info_dict']['formats']
+            'formats',
-        if 'automatic_captions' in event['info_dict']:
+            'automatic_captions',
-            del event['info_dict']['automatic_captions']
+        })
-        log.debug(repr(event['info_dict']))
+        logged_dict = filter_response(event['info_dict'], True)
+        found_keys = set(logged_dict.keys()).intersection(remove_keys)
+        for key in found_keys:
+            del logged_dict[key]
+        log.debug(repr(logged_dict))
 
     if 'Unknown' != key:
         status.media_key = key
@@ -224,6 +229,19 @@ def yt_dlp_postprocessor_hook(event):
     status.update_task()
 
     log.info(f'[{event["postprocessor"]}] {event["status"]} for: {name}')
+    if 'started' == event['status'] and 'Merger' == event['postprocessor']:
+        files_to_merge = event['info_dict'].get('__files_to_merge') or list()
+        log.info(f'[{event["postprocessor"]}] Files to merge: {files_to_merge}')
+        from .models import Media
+        try:
+            media = Media.objects.get(pk=status.media_uuid)
+            media.new_metadata.value['requested_formats'] = event['info_dict'].get('requested_formats')
+            media.new_metadata.value['requested_subtitles'] = event['info_dict'].get('requested_subtitles')
+            media.new_metadata.save()
+        except Media.DoesNotExist:
+            pass
+        except Exception as e:
+            log.exception(e)
     if 'finished' == event['status']:
         status.cleanup()
 

tubesync/sync/management/commands/reset-tasks.py

@@ -29,7 +29,7 @@ class Command(BaseCommand):
                 index_source_task(
                     str(source.pk),
                     repeat=source.index_schedule,
-                    schedule=source.index_schedule,
+                    schedule=source.task_run_at_dt,
                     verbose_name=verbose_name.format(source.name),
                 )
                 # This also chains down to call each Media objects .save() as well

tubesync/sync/migrations/0034_source_target_schedule_and_more.py

@@ -0,0 +1,38 @@
+# Generated by Django 5.2.1 on 2025-05-26 04:43
+
+import django.utils.timezone
+import sync.fields
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0033_alter_mediaserver_options_alter_source_source_acodec_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='source',
+            name='target_schedule',
+            field=models.DateTimeField(
+                blank=True, db_index=True, default=django.utils.timezone.now,
+                help_text='Date and time when the task to index the source should begin',
+                verbose_name='target schedule',
+            ),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='sponsorblock_categories',
+            field=sync.fields.CommaSepChoiceField(
+                all_choice='all', all_label='(All Categories)', allow_all=True, default='all',
+                help_text='Select the SponsorBlock categories that you wish to be removed from downloaded videos.',
+                max_length=128, possible_choices=[
+                    ('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'),
+                    ('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'),
+                    ('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'),
+                ],
+                verbose_name='removed categories',
+            ),
+        ),
+    ]
+

tubesync/sync/models/metadata.py

@@ -134,9 +134,9 @@ class Metadata(db.models.Model):
                     ('release_timestamp', 'timestamp',),
                     arg_dict=data,
                 )
-            ) or self.media.published
+            ) or self.media.published or self.retrieved
         except AssertionError:
-            self.published = self.media.published
+            self.published = self.media.published or self.retrieved
 
         self.value = data.copy() # try not to have side-effects for the caller
         formats_key = self.media.get_metadata_field('formats')

tubesync/sync/models/source.py

@@ -31,16 +31,6 @@ class Source(db.models.Model):
         or a YouTube playlist.
     '''
 
-    sponsorblock_categories = CommaSepChoiceField(
-        _(''),
-        max_length=128,
-        possible_choices=SponsorBlock_Category.choices,
-        all_choice='all',
-        allow_all=True,
-        all_label='(All Categories)',
-        default='all',
-        help_text=_('Select the SponsorBlock categories that you wish to be removed from downloaded videos.'),
-    )
     embed_metadata = db.models.BooleanField(
         _('embed metadata'),
         default=False,
@@ -51,11 +41,6 @@ class Source(db.models.Model):
         default=False,
         help_text=_('Embed thumbnail into the file'),
     )
-    enable_sponsorblock = db.models.BooleanField(
-        _('enable sponsorblock'),
-        default=True,
-        help_text=_('Use SponsorBlock?'),
-    )
 
     # Fontawesome icons used for the source on the front end
     ICONS = _srctype_dict('<i class="fab fa-youtube"></i>')
@@ -141,6 +126,13 @@ class Source(db.models.Model):
         default=settings.MEDIA_FORMATSTR_DEFAULT,
         help_text=_('File format to use for saving files, detailed options at bottom of page.'),
     )
+    target_schedule = db.models.DateTimeField(
+        _('target schedule'),
+        blank=True,
+        db_index=True,
+        default=timezone.now,
+        help_text=_('Date and time when the task to index the source should begin'),
+    )
     index_schedule = db.models.IntegerField(
         _('index schedule'),
         choices=IndexSchedule.choices,
@@ -310,6 +302,21 @@ class Source(db.models.Model):
             ),
         ],
     )
+    enable_sponsorblock = db.models.BooleanField(
+        _('enable sponsorblock'),
+        default=True,
+        help_text=_('Use SponsorBlock?'),
+    )
+    sponsorblock_categories = CommaSepChoiceField(
+        _('removed categories'),
+        max_length=128,
+        possible_choices=SponsorBlock_Category.choices,
+        all_choice='all',
+        allow_all=True,
+        all_label='(All Categories)',
+        default='all',
+        help_text=_('Select the SponsorBlock categories that you wish to be removed from downloaded videos.'),
+    )
 
     def __str__(self):
         return self.name
@@ -376,6 +383,39 @@ class Source(db.models.Model):
         else:
             return False
 
+    @property
+    def task_run_at_dt(self):
+        now = timezone.now()
+        when = now.replace(minute=0, second=0, microsecond=0)
+
+        def advance_hour(arg_dt, target_hour, /):
+            delta_hours = ((24 + target_hour) - arg_dt.hour) % 24
+            return arg_dt + timezone.timedelta(hours=delta_hours)
+
+        def advance_day(arg_dt, target_weekday, /):
+            delta_days = ((7 + target_weekday) - arg_dt.weekday) % 7
+            return arg_dt + timezone.timedelta(days=delta_days)
+
+        if self.target_schedule is None:
+            self.target_schedule = when
+        if Val(IndexSchedule.EVERY_24_HOURS) > self.index_schedule:
+            self.target_schedule = now + timezone.timedelta(
+                seconds=1+self.index_schedule,
+            )
+        elif Val(IndexSchedule.EVERY_7_DAYS) > self.index_schedule:
+            self.target_schedule = advance_hour(
+                when.replace(hour=1+when.hour),
+                self.target_schedule.hour,
+            )
+
+        if now < self.target_schedule:
+            return self.target_schedule
+
+        when = advance_hour(when, self.target_schedule.hour)
+        when = advance_day(when, self.target_schedule.weekday)
+        self.target_schedule = when
+        return when
+
     @property
     def extension(self):
         '''

tubesync/sync/signals.py

@@ -95,7 +95,7 @@ def source_pre_save(sender, instance, **kwargs):
         index_source_task(
             str(instance.pk),
             repeat=instance.index_schedule,
-            schedule=instance.index_schedule,
+            schedule=instance.task_run_at_dt,
             verbose_name=verbose_name.format(instance.name),
         )
 

tubesync/sync/tasks.py

@@ -319,6 +319,8 @@ def index_source_task(source_id):
     # An inactive Source would return an empty list for videos anyway
     if not source.is_active:
         return
+    # update the target schedule column
+    source.task_run_at_dt
     # Reset any errors
     # TODO: determine if this affects anything
     source.has_failed = False

tubesync/sync/templates/sync/source.html

@@ -93,6 +93,10 @@
         <td class="hide-on-small-only">Last crawl</td>
         <td><span class="hide-on-med-and-up">Last crawl<br></span><strong>{% if source.last_crawl %}{{ source.last_crawl|date:'Y-m-d H:i:s' }}{% else %}Never{% endif %}</strong></td>
       </tr>
+      <tr title="When the source should be next checked for available media">
+        <td class="hide-on-small-only">Target schedule</td>
+        <td><span class="hide-on-med-and-up">Target schedule<br></span><strong>{% if source.target_schedule %}{{ source.target_schedule|date:'l, h:00 A (c)' }}{% else %}Not set{% endif %}</strong></td>
+      </tr>
       <tr title="Quality and type of media the source will attempt to sync">
         <td class="hide-on-small-only">Source resolution</td>
         <td><span class="hide-on-med-and-up">Source resolution<br></span><strong>{{ source.get_source_resolution_display }}</strong></td>

tubesync/sync/views.py

@@ -25,7 +25,7 @@ from background_task.models import Task, CompletedTask
 from .models import Source, Media, MediaServer
 from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm,
                     SkipMediaForm, EnableMediaForm, ResetTasksForm, ScheduleTaskForm,
-                    ConfirmDeleteMediaServerForm)
+                    ConfirmDeleteMediaServerForm, SourceForm)
 from .utils import validate_url, delete_file, multi_key_sort, mkdir_p
 from .tasks import (map_task_to_instance, get_error_message,
                     get_source_completed_tasks, get_media_download_task,
@@ -276,15 +276,7 @@ class ValidateSourceView(FormView):
 
 class EditSourceMixin:
     model = Source
-    # manual ordering
+    form_class = SourceForm
-    fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'filter_text_invert', 'filter_seconds', 'filter_seconds_min',
-              'media_format', 'index_schedule', 'index_videos', 'index_streams', 'download_media', 'download_cap', 'delete_old_media',
-              'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
-              'prefer_60fps', 'prefer_hdr', 'fallback',
-              'delete_removed_media', 'delete_files_on_disk', 'copy_channel_images',
-              'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
-              'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
-              'auto_subtitles', 'sub_langs')
     errors = {
         'invalid_media_format': _('Invalid media format, the media format contains '
                                   'errors or is empty. Check the table at the end of '
@@ -357,6 +349,9 @@ class AddSourceView(EditSourceMixin, CreateView):
 
     def get_initial(self):
         initial = super().get_initial()
+        initial['target_schedule'] = timezone.now().replace(
+            second=0, microsecond=0,
+        )
         for k, v in self.prepopulated_data.items():
             initial[k] = v
         return initial
@@ -403,6 +398,12 @@ class UpdateSourceView(EditSourceMixin, UpdateView):
 
     template_name = 'sync/source-update.html'
 
+    def get_initial(self):
+        initial = super().get_initial()
+        when = getattr(self.object, 'target_schedule') or timezone.now()
+        initial['target_schedule'] = when.replace(second=0, microsecond=0)
+        return initial
+
     def get_success_url(self):
         url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
         return append_uri_params(url, {'message': 'source-updated'})
@@ -1003,6 +1004,7 @@ class ResetTasks(FormView):
             index_source_task(
                 str(source.pk),
                 repeat=source.index_schedule,
+                schedule=source.task_run_at_dt,
                 verbose_name=verbose_name.format(source.name)
             )
             # This also chains down to call each Media objects .save() as well

tubesync/sync/youtube.py

@@ -179,7 +179,7 @@ def get_media_info(url, /, *, days=None, info_json=None):
     playlist_infojson = 'postprocessor_[%(id)s]_%(n_entries)d_%(playlist_count)d_temp'
     outtmpl = dict(
         default='',
-        infojson='%(extractor)s/%(id)s.%(ext)s' if paths.get('infojson') else '',
+        infojson='%(extractor_key)s/%(id)s.%(ext)s' if paths.get('infojson') else '',
         pl_infojson=f'{cache_directory_path}/infojson/playlist/{playlist_infojson}.%(ext)s',
     )
     for k in OUTTMPL_TYPES.keys():