wimby/tubesync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into patch-14

Browse Source
This commit is contained in:
tcely
2025-05-29 00:47:27 -04:00
committed by GitHub
parent a7ede7d965 c0f9b05eb4
commit 2120776581
14 changed files with 222 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
Dockerfile
View File

@@ -58,6 +58,8 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
apt-get -y autoclean && \
rm -f /var/cache/debconf/*.dat-old
FROM ghcr.io/astral-sh/uv:latest AS uv-binaries
FROM alpine:${ALPINE_VERSION} AS openresty-debian
ARG TARGETARCH
ARG DEBIAN_VERSION
@@ -279,6 +281,9 @@ RUN set -eu ; \
FROM scratch AS s6-overlay
COPY --from=s6-overlay-extracted /s6-overlay-rootfs /
FROM tubesync-base AS tubesync-uv
COPY --from=uv-binaries /uv /uvx /usr/local/bin/
FROM tubesync-base AS tubesync-openresty
COPY --from=openresty-debian \
@@ -290,7 +295,10 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
--mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \
set -x && \
apt-get update && \
apt-get -y --no-install-recommends install nginx-common openresty && \
apt-get -y --no-install-recommends install \
nginx-common \
openresty \
&& \
# Clean up
apt-get -y autopurge && \
apt-get -y autoclean && \
@@ -302,7 +310,10 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
--mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \
set -x && \
apt-get update && \
apt-get -y --no-install-recommends install nginx-light && \
apt-get -y --no-install-recommends install \
nginx-light \
libnginx-mod-http-lua \
&& \
# openresty binary should still work
ln -v -s -T ../sbin/nginx /usr/bin/openresty && \
# Clean up
@@ -335,12 +346,11 @@ RUN --mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/va
libmariadb3 \
libpq5 \
libwebp7 \
pipenv \
pkgconf \
python3 \
python3-libsass \
python3-pip-whl \
python3-socks \
python3-wheel \
curl \
less \
&& \
@@ -383,9 +393,11 @@ ARG YTDLP_DATE
# Set up the app
RUN --mount=type=tmpfs,target=/cache \
--mount=type=cache,id=uv-cache,sharing=locked,target=/cache/uv \
--mount=type=cache,id=pipenv-cache,sharing=locked,target=/cache/pipenv \
--mount=type=cache,id=apt-lib-cache-${TARGETARCH},sharing=private,target=/var/lib/apt \
--mount=type=cache,id=apt-cache-cache,sharing=private,target=/var/cache/apt \
--mount=type=bind,source=/uv,target=/usr/local/bin/uv,from=uv-binaries \
--mount=type=bind,source=Pipfile,target=/app/Pipfile \
set -x && \
apt-get update && \
@@ -400,16 +412,32 @@ RUN --mount=type=tmpfs,target=/cache \
make \
postgresql-common \
python3-dev \
python3-pip \
zlib1g-dev \
&& \
# Install non-distro packages
cp -at /tmp/ "${HOME}" && \
HOME="/tmp/${HOME#/}" \
mkdir -v -p /cache/.home-directories && \
cp -at /cache/.home-directories/ "${HOME}" && \
HOME="/cache/.home-directories/${HOME#/}" \
XDG_CACHE_HOME='/cache' \
PIPENV_VERBOSITY=64 \
PYTHONPYCACHEPREFIX=/cache/pycache \
pipenv install --system --skip-lock && \
uv tool run --no-config --no-progress --no-managed-python -- \
pipenv lock && \
HOME="/cache/.home-directories/${HOME#/}" \
XDG_CACHE_HOME='/cache' \
PIPENV_VERBOSITY=1 \
PYTHONPYCACHEPREFIX=/cache/pycache \
uv tool run --no-config --no-progress --no-managed-python -- \
pipenv requirements --from-pipfile --hash >| /cache/requirements.txt && \
rm -v Pipfile.lock && \
cat -v /cache/requirements.txt && \
HOME="/cache/.home-directories/${HOME#/}" \
UV_LINK_MODE='copy' \
XDG_CACHE_HOME='/cache' \
PYTHONPYCACHEPREFIX=/cache/pycache \
uv --no-config --no-progress --no-managed-python \
pip install --strict --system --break-system-packages \
--requirements /cache/requirements.txt && \
# remove the getpot_bgutil_script plugin
find /usr/local/lib \
-name 'getpot_bgutil_script.py' \
@@ -427,7 +455,6 @@ RUN --mount=type=tmpfs,target=/cache \
make \
postgresql-common \
python3-dev \
python3-pip \
zlib1g-dev \
&& \
apt-get -y autopurge && \

4
config/root/etc/nginx/nginx.conf
View File

@@ -1,3 +1,7 @@
# load_module snippets for installed modules
include /etc/nginx/modules-enabled/*.conf;
# a very silly syntax rendering bug needs this */
daemon off;
user app;

22
tubesync/sync/choices.py
View File

@@ -167,6 +167,28 @@ class TaskQueue(models.TextChoices):
NET = 'network', _('Networking')
class WeekDay(models.IntegerChoices):
MON = 0, _('Monday')
TUE = 1, _('Tuesday')
WED = 2, _('Wednesday')
THU = 3, _('Thursday')
FRI = 4, _('Friday')
SAT = 5, _('Saturday')
SUN = 6, _('Sunday')
@classmethod
def get(cls, wdn, /):
return cls[cls.names[wdn]]
@classmethod
def _from_iso(cls, wdn, /):
return cls[cls.names[wdn - 1]]
@classmethod
def _to_iso(cls, choice, /):
return 1 + choice.value
class YouTube_SourceType(models.TextChoices):
CHANNEL = 'c', _('YouTube channel')
CHANNEL_ID = 'i', _('YouTube channel by ID')

22
tubesync/sync/forms.py
View File

@@ -2,13 +2,33 @@
from django import forms, VERSION as DJANGO_VERSION
from django.utils.translation import gettext_lazy as _
from .models import Source
if DJANGO_VERSION[0:3] < (5, 0, 0):
_assume_scheme = dict()
else:
# Silence RemovedInDjango60Warning
_assume_scheme = dict(assume_scheme='http')
SourceForm = forms.modelform_factory(
Source,
# manual ordering
fields = (
'source_type', 'key', 'name', 'directory', 'filter_text', 'filter_text_invert', 'filter_seconds', 'filter_seconds_min',
'media_format', 'target_schedule', 'index_schedule', 'index_videos', 'index_streams', 'download_media',
'download_cap', 'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
'prefer_60fps', 'prefer_hdr', 'fallback', 'delete_removed_media', 'delete_files_on_disk', 'copy_channel_images',
'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles', 'auto_subtitles', 'sub_langs',
),
widgets = {
'target_schedule': forms.DateTimeInput(
attrs={'type': 'datetime-local'},
),
},
)
class ValidateSourceForm(forms.Form):
source_type = forms.CharField(

28
tubesync/sync/hooks.py
View File

@@ -2,6 +2,7 @@ import os
from common.logger import log
from common.utils import remove_enclosed
from sync.utils import filter_response
progress_hook = {
@@ -212,11 +213,15 @@ def yt_dlp_postprocessor_hook(event):
status.media_name = name
if 'started' == event['status']:
if 'formats' in event['info_dict']:
del event['info_dict']['formats']
if 'automatic_captions' in event['info_dict']:
del event['info_dict']['automatic_captions']
log.debug(repr(event['info_dict']))
remove_keys = frozenset({
'formats',
'automatic_captions',
})
logged_dict = filter_response(event['info_dict'], True)
found_keys = set(logged_dict.keys()).intersection(remove_keys)
for key in found_keys:
del logged_dict[key]
log.debug(repr(logged_dict))
if 'Unknown' != key:
status.media_key = key
@@ -224,6 +229,19 @@ def yt_dlp_postprocessor_hook(event):
status.update_task()
log.info(f'[{event["postprocessor"]}] {event["status"]} for: {name}')
if 'started' == event['status'] and 'Merger' == event['postprocessor']:
files_to_merge = event['info_dict'].get('__files_to_merge') or list()
log.info(f'[{event["postprocessor"]}] Files to merge: {files_to_merge}')
from .models import Media
try:
media = Media.objects.get(pk=status.media_uuid)
media.new_metadata.value['requested_formats'] = event['info_dict'].get('requested_formats')
media.new_metadata.value['requested_subtitles'] = event['info_dict'].get('requested_subtitles')
media.new_metadata.save()
except Media.DoesNotExist:
pass
except Exception as e:
log.exception(e)
if 'finished' == event['status']:
status.cleanup()

2
tubesync/sync/management/commands/reset-tasks.py
View File

@@ -29,7 +29,7 @@ class Command(BaseCommand):
index_source_task(
str(source.pk),
repeat=source.index_schedule,
schedule=source.index_schedule,
schedule=source.task_run_at_dt,
verbose_name=verbose_name.format(source.name),
)
# This also chains down to call each Media objects .save() as well

38
tubesync/sync/migrations/0034_source_target_schedule_and_more.py Normal file
View File

@@ -0,0 +1,38 @@
# Generated by Django 5.2.1 on 2025-05-26 04:43
import django.utils.timezone
import sync.fields
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sync', '0033_alter_mediaserver_options_alter_source_source_acodec_and_more'),
]
operations = [
migrations.AddField(
model_name='source',
name='target_schedule',
field=models.DateTimeField(
blank=True, db_index=True, default=django.utils.timezone.now,
help_text='Date and time when the task to index the source should begin',
verbose_name='target schedule',
),
),
migrations.AlterField(
model_name='source',
name='sponsorblock_categories',
field=sync.fields.CommaSepChoiceField(
all_choice='all', all_label='(All Categories)', allow_all=True, default='all',
help_text='Select the SponsorBlock categories that you wish to be removed from downloaded videos.',
max_length=128, possible_choices=[
('sponsor', 'Sponsor'), ('intro', 'Intermission/Intro Animation'), ('outro', 'Endcards/Credits'),
('selfpromo', 'Unpaid/Self Promotion'), ('preview', 'Preview/Recap'), ('filler', 'Filler Tangent'),
('interaction', 'Interaction Reminder'), ('music_offtopic', 'Non-Music Section'),
],
verbose_name='removed categories',
),
),
]

4
tubesync/sync/models/metadata.py
View File

@@ -134,9 +134,9 @@ class Metadata(db.models.Model):
('release_timestamp', 'timestamp',),
arg_dict=data,
)
) or self.media.published
) or self.media.published or self.retrieved
except AssertionError:
self.published = self.media.published
self.published = self.media.published or self.retrieved
self.value = data.copy() # try not to have side-effects for the caller
formats_key = self.media.get_metadata_field('formats')

70
tubesync/sync/models/source.py
View File

@@ -31,16 +31,6 @@ class Source(db.models.Model):
or a YouTube playlist.
'''
sponsorblock_categories = CommaSepChoiceField(
_(''),
max_length=128,
possible_choices=SponsorBlock_Category.choices,
all_choice='all',
allow_all=True,
all_label='(All Categories)',
default='all',
help_text=_('Select the SponsorBlock categories that you wish to be removed from downloaded videos.'),
)
embed_metadata = db.models.BooleanField(
_('embed metadata'),
default=False,
@@ -51,11 +41,6 @@ class Source(db.models.Model):
default=False,
help_text=_('Embed thumbnail into the file'),
)
enable_sponsorblock = db.models.BooleanField(
_('enable sponsorblock'),
default=True,
help_text=_('Use SponsorBlock?'),
)
# Fontawesome icons used for the source on the front end
ICONS = _srctype_dict('<i class="fab fa-youtube"></i>')
@@ -141,6 +126,13 @@ class Source(db.models.Model):
default=settings.MEDIA_FORMATSTR_DEFAULT,
help_text=_('File format to use for saving files, detailed options at bottom of page.'),
)
target_schedule = db.models.DateTimeField(
_('target schedule'),
blank=True,
db_index=True,
default=timezone.now,
help_text=_('Date and time when the task to index the source should begin'),
)
index_schedule = db.models.IntegerField(
_('index schedule'),
choices=IndexSchedule.choices,
@@ -310,6 +302,21 @@ class Source(db.models.Model):
),
],
)
enable_sponsorblock = db.models.BooleanField(
_('enable sponsorblock'),
default=True,
help_text=_('Use SponsorBlock?'),
)
sponsorblock_categories = CommaSepChoiceField(
_('removed categories'),
max_length=128,
possible_choices=SponsorBlock_Category.choices,
all_choice='all',
allow_all=True,
all_label='(All Categories)',
default='all',
help_text=_('Select the SponsorBlock categories that you wish to be removed from downloaded videos.'),
)
def __str__(self):
return self.name
@@ -376,6 +383,39 @@ class Source(db.models.Model):
else:
return False
@property
def task_run_at_dt(self):
now = timezone.now()
when = now.replace(minute=0, second=0, microsecond=0)
def advance_hour(arg_dt, target_hour, /):
delta_hours = ((24 + target_hour) - arg_dt.hour) % 24
return arg_dt + timezone.timedelta(hours=delta_hours)
def advance_day(arg_dt, target_weekday, /):
delta_days = ((7 + target_weekday) - arg_dt.weekday) % 7
return arg_dt + timezone.timedelta(days=delta_days)
if self.target_schedule is None:
self.target_schedule = when
if Val(IndexSchedule.EVERY_24_HOURS) > self.index_schedule:
self.target_schedule = now + timezone.timedelta(
seconds=1+self.index_schedule,
)
elif Val(IndexSchedule.EVERY_7_DAYS) > self.index_schedule:
self.target_schedule = advance_hour(
when.replace(hour=1+when.hour),
self.target_schedule.hour,
)
if now < self.target_schedule:
return self.target_schedule
when = advance_hour(when, self.target_schedule.hour)
when = advance_day(when, self.target_schedule.weekday)
self.target_schedule = when
return when
@property
def extension(self):
'''

2
tubesync/sync/signals.py
View File

@@ -95,7 +95,7 @@ def source_pre_save(sender, instance, **kwargs):
index_source_task(
str(instance.pk),
repeat=instance.index_schedule,
schedule=instance.index_schedule,
schedule=instance.task_run_at_dt,
verbose_name=verbose_name.format(instance.name),
)

2
tubesync/sync/tasks.py
View File

@@ -319,6 +319,8 @@ def index_source_task(source_id):
# An inactive Source would return an empty list for videos anyway
if not source.is_active:
return
# update the target schedule column
source.task_run_at_dt
# Reset any errors
# TODO: determine if this affects anything
source.has_failed = False

4
tubesync/sync/templates/sync/source.html
View File

@@ -93,6 +93,10 @@
<td class="hide-on-small-only">Last crawl</td>
<td><span class="hide-on-med-and-up">Last crawl<br></span><strong>{% if source.last_crawl %}{{ source.last_crawl|date:'Y-m-d H:i:s' }}{% else %}Never{% endif %}</strong></td>
</tr>
<tr title="When the source should be next checked for available media">
<td class="hide-on-small-only">Target schedule</td>
<td><span class="hide-on-med-and-up">Target schedule<br></span><strong>{% if source.target_schedule %}{{ source.target_schedule|date:'l, h:00 A (c)' }}{% else %}Not set{% endif %}</strong></td>
</tr>
<tr title="Quality and type of media the source will attempt to sync">
<td class="hide-on-small-only">Source resolution</td>
<td><span class="hide-on-med-and-up">Source resolution<br></span><strong>{{ source.get_source_resolution_display }}</strong></td>

22
tubesync/sync/views.py
View File

@@ -25,7 +25,7 @@ from background_task.models import Task, CompletedTask
from .models import Source, Media, MediaServer
from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm,
SkipMediaForm, EnableMediaForm, ResetTasksForm, ScheduleTaskForm,
ConfirmDeleteMediaServerForm)
ConfirmDeleteMediaServerForm, SourceForm)
from .utils import validate_url, delete_file, multi_key_sort, mkdir_p
from .tasks import (map_task_to_instance, get_error_message,
get_source_completed_tasks, get_media_download_task,
@@ -276,15 +276,7 @@ class ValidateSourceView(FormView):
class EditSourceMixin:
model = Source
# manual ordering
fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'filter_text_invert', 'filter_seconds', 'filter_seconds_min',
'media_format', 'index_schedule', 'index_videos', 'index_streams', 'download_media', 'download_cap', 'delete_old_media',
'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec',
'prefer_60fps', 'prefer_hdr', 'fallback',
'delete_removed_media', 'delete_files_on_disk', 'copy_channel_images',
'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail',
'enable_sponsorblock', 'sponsorblock_categories', 'write_subtitles',
'auto_subtitles', 'sub_langs')
form_class = SourceForm
errors = {
'invalid_media_format': _('Invalid media format, the media format contains '
'errors or is empty. Check the table at the end of '
@@ -357,6 +349,9 @@ class AddSourceView(EditSourceMixin, CreateView):
def get_initial(self):
initial = super().get_initial()
initial['target_schedule'] = timezone.now().replace(
second=0, microsecond=0,
)
for k, v in self.prepopulated_data.items():
initial[k] = v
return initial
@@ -403,6 +398,12 @@ class UpdateSourceView(EditSourceMixin, UpdateView):
template_name = 'sync/source-update.html'
def get_initial(self):
initial = super().get_initial()
when = getattr(self.object, 'target_schedule') or timezone.now()
initial['target_schedule'] = when.replace(second=0, microsecond=0)
return initial
def get_success_url(self):
url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
return append_uri_params(url, {'message': 'source-updated'})
@@ -1003,6 +1004,7 @@ class ResetTasks(FormView):
index_source_task(
str(source.pk),
repeat=source.index_schedule,
schedule=source.task_run_at_dt,
verbose_name=verbose_name.format(source.name)
)
# This also chains down to call each Media objects .save() as well

2
tubesync/sync/youtube.py
View File

@@ -179,7 +179,7 @@ def get_media_info(url, /, *, days=None, info_json=None):
playlist_infojson = 'postprocessor_[%(id)s]_%(n_entries)d_%(playlist_count)d_temp'
outtmpl = dict(
default='',
infojson='%(extractor)s/%(id)s.%(ext)s' if paths.get('infojson') else '',
infojson='%(extractor_key)s/%(id)s.%(ext)s' if paths.get('infojson') else '',
pl_infojson=f'{cache_directory_path}/infojson/playlist/{playlist_infojson}.%(ext)s',
)
for k in OUTTMPL_TYPES.keys():