wimby/tubesync
1
0 Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into patch-2

Browse Source
This commit is contained in:
tcely authored and GitHub committed 2025-05-20 04:09:47 -04:00
2 parents aedaa6f7b7 c434eb2487
commit e46beb39c6
34 files changed
+500 -207
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/actions/get-tag/action.yml
+41
View File
@@ -0,0 +1,41 @@
name: Get tag
description: Get tag name from GITHUB_REF environment variable
inputs:
strip_v:
required: false
default: false
description: Whether to strip "v" from the tag or not
outputs:
tag:
value: ${{ steps.set.outputs.tag }}
description: Git tag name
runs:
using: 'composite'
steps:
- name: Set outputs
id: 'set'
env:
INPUT_STRIP_V: '${{ inputs.strip_v }}'
shell: 'bash'
run: |
tag="${GITHUB_REF}"
printf -- 'Manipulating string: %s\n' "${tag}"
test -n "${tag}" || exit 1
case "${tag}" in
(refs/tags/*) tag="${tag#refs/tags/}" ;;
(*) printf -- 'Not a tag ref\n' ; exit 2 ;;
esac
if [ 'true' = "${INPUT_STRIP_V,,}" ]
then
tag="${tag#[Vv]}"
fi
set_sl_var() { local f='%s=%s\n' ; printf -- "${f}" "$@" ; } ;
set_sl_var tag "${tag}" >> "${GITHUB_OUTPUT}"
set_sl_var 'tag ' " ${tag}"
.github/workflows/ci.yaml
+49 -3
View File
@@ -93,15 +93,61 @@ jobs:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pipenv
PIPENV_VERBOSITY=64 pipenv install --system --skip-lock
python -m pip install uv
uv --no-config --no-managed-python --no-progress \
pip install --system --strict pipenv
pipenv lock
pipenv requirements | tee requirements.txt
#PIPENV_VERBOSITY=64 pipenv install --system --skip-lock
uv --no-config --no-managed-python --no-progress \
pip install --system --strict --requirements requirements.txt
- name: Set up Django environment
run: |
cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py
cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/*
cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/*
cd tubesync && python3 -B manage.py collectstatic --no-input --link
- name: Check with ruff
continue-on-error: false
run: |
target_version='py310'
ignore_csv_list='E701,E722,E731'
cd tubesync
# output formats:
# "full" | "concise" | "grouped" |
# "json" | "junit" | "github" | "gitlab" |
# "pylint" | "azure"
{
echo '## Output from `ruff check` for `tubesync`'
echo ''
echo '### Formats'
echo ''
for fmt in full concise grouped pylint
do
echo '<details>'
echo '<summary>'"${fmt}"'</summary>'
echo ''
echo '#### '"${fmt}"' output format'
echo ''
echo '```'
uvx --no-config --no-managed-python --no-progress --isolated \
ruff check --exit-zero \
--target-version "${target_version}" \
--output-format "${fmt}" \
--extend-select RUF100 \
--ignore "${ignore_csv_list}"
echo ''
echo '```'
echo ''
echo '</details>'
echo ''
done
} >> "${GITHUB_STEP_SUMMARY}"
uvx --no-config --no-managed-python --no-progress --isolated \
ruff check --exit-zero \
--target-version "${target_version}" \
--output-format github \
--ignore "${ignore_csv_list}"
- name: Run Django tests
run: cd tubesync && python3 -B -W default manage.py test --verbosity=2
.github/workflows/release.yaml
+73 -15
View File
@@ -8,31 +8,89 @@ on:
types: [published]
jobs:
info:
runs-on: ubuntu-latest
outputs:
ffmpeg-date: ${{ steps.jq.outputs.FFMPEG_DATE }}
ffmpeg-releases: ${{ steps.ffmpeg.outputs.releases }}
ffmpeg-version: ${{ steps.jq.outputs.FFMPEG_VERSION }}
lowercase-github-actor: ${{ steps.github-actor.outputs.lowercase }}
lowercase-github-repository_owner: ${{ steps.github-repository_owner.outputs.lowercase }}
tag: ${{ steps.tag.outputs.tag }}
ytdlp-latest-release: ${{ steps.yt-dlp.outputs.latest-release }}
ytdlp-releases: ${{ steps.yt-dlp.outputs.releases }}
steps:
- uses: actions/checkout@v4
- name: Get tag
id: tag
uses: ./.github/actions/get-tag
- name: Lowercase github username
id: github-actor
uses: ./.github/actions/string-case
with:
string: ${{ github.actor }}
- name: Lowercase github repository owner
id: github-repository_owner
uses: ./.github/actions/string-case
with:
string: ${{ github.repository_owner }}
- name: Retrieve yt-dlp/FFmpeg-Builds releases with GitHub CLI
id: ffmpeg
uses: ./.github/actions/FFmpeg
- name: Retrieve yt-dlp/yt-dlp releases with GitHub CLI
id: yt-dlp
uses: ./.github/actions/yt-dlp
- name: Set outputs with jq
id: jq
run: |
cat >| .ffmpeg.releases.json <<'EOF'
${{ steps.ffmpeg.outputs.releases }}
EOF
mk_delim() { local f='%s_EOF_%d_' ; printf -- "${f}" "$1" "${RANDOM}" ; } ;
open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ;
close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ;
{
var='FFMPEG_DATE' ;
delim="$(mk_delim "${var}")" ;
open_ml_var "${delim}" "${var}" ;
jq_arg='[foreach .[] as $release ([{}, []]; [ .[0] + {($release.commit): ([ $release.date ] + (.[0][($release.commit)] // []) ) }, [ .[1][0] // $release.commit ] ] ; .[0][(.[1][0])] ) ][-1][0]' ;
jq -r "${jq_arg}" -- .ffmpeg.releases.json ;
close_ml_var "${delim}" "${var}" ;
ffmpeg_date="$( jq -r "${jq_arg}" -- .ffmpeg.releases.json )"
var='FFMPEG_VERSION' ;
delim="$(mk_delim "${var}")" ;
open_ml_var "${delim}" "${var}" ;
jq_arg='.[]|select(.date == $date)|.versions[]|select(startswith("N-"))' ;
jq -r --arg date "${ffmpeg_date}" "${jq_arg}" -- .ffmpeg.releases.json ;
close_ml_var "${delim}" "${var}" ;
unset -v delim jq_arg var ;
} >> "${GITHUB_OUTPUT}"
cat -v "${GITHUB_OUTPUT}"
rm -v -f .ffmpeg.releases.json
containerise:
needs: ['info']
runs-on: ubuntu-latest
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Get tag
id: tag
uses: dawidd6/action-get-tag@v1
- uses: docker/build-push-action@v2
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@v3
- name: Log into GitHub Container Registry
run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
- name: Lowercase github username for ghcr
id: string
uses: ASzc/change-string-case-action@v1
with:
string: ${{ github.actor }}
- name: Build and push
uses: docker/build-push-action@v2
uses: docker/build-push-action@v6
with:
platforms: linux/amd64,linux/arm64
push: true
tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}
cache-to: type=inline
tags: ghcr.io/${{ needs.info.outputs.lowercase-github-actor }}/${{ env.IMAGE_NAME }}:${{ needs.info.outputs.tag }}
cache-from: |
type=registry,ref=ghcr.io/${{ needs.info.outputs.lowercase-github-repository_owner }}/${{ env.IMAGE_NAME }}:latest
type=gha
build-args: |
IMAGE_NAME=${{ env.IMAGE_NAME }}
FFMPEG_DATE=${{ needs.info.outputs.ffmpeg-date }}
FFMPEG_VERSION=${{ needs.info.outputs.ffmpeg-version }}
YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }}
Dockerfile
+6
View File
@@ -410,6 +410,12 @@ RUN --mount=type=tmpfs,target=/cache \
PIPENV_VERBOSITY=64 \
PYTHONPYCACHEPREFIX=/cache/pycache \
pipenv install --system --skip-lock && \
# remove the getpot_bgutil_script plugin
find /usr/local/lib \
-name 'getpot_bgutil_script.py' \
-path '*/yt_dlp_plugins/extractor/getpot_bgutil_script.py' \
-type f -print -delete \
&& \
# Clean up
apt-get -y autoremove --purge \
default-libmysqlclient-dev \
Pipfile
+3 -1
View File
@@ -7,7 +7,7 @@ verify_ssl = true
autopep8 = "*"
[packages]
django = "*"
django = "~=5.2.1"
django-sass-processor = {extras = ["management-command"], version = "*"}
pillow = "*"
whitenoise = "*"
@@ -24,3 +24,5 @@ yt-dlp = {extras = ["default", "curl-cffi"], version = "*"}
emoji = "*"
brotli = "*"
html5lib = "*"
yt-dlp-get-pot = "*"
bgutil-ytdlp-pot-provider = "*"
config/root/etc/nginx/nginx.conf
+5
View File
@@ -5,6 +5,8 @@ worker_processes auto;
worker_cpu_affinity auto;
pid /run/nginx.pid;
env YT_POT_BGUTIL_BASE_URL;
events {
worker_connections 1024;
}
@@ -135,4 +137,7 @@ http {
}
}
# Proof-of-Origin Token Server
include /etc/nginx/token_server.conf;
}
config/root/etc/nginx/token_server.conf
+29
View File
@@ -0,0 +1,29 @@
upstream token_server {
server 127.0.0.2:4416 down;
}
server {
# Ports
listen 4416;
listen [::]:4416;
# Server domain name
server_name _;
set_by_lua_block $pot_url {
local default = 'http://token_server'
local url = os.getenv('YT_POT_BGUTIL_BASE_URL')
if not url then
return default
end
if #url and url:find('://') then
return url
end
return default
}
location / {
proxy_pass $pot_url;
}
}
config/root/etc/s6-overlay/s6-rc.d/nginx/run
+11
View File
@@ -2,4 +2,15 @@
cd /
https="${TUBESYNC_POT_HTTPS:+https}"
ip_address="${TUBESYNC_POT_IPADDR:-${POTSERVER_PORT_4416_TCP_ADDR}}"
: "${TUBESYNC_POT_PORT:=${POTSERVER_PORT_4416_TCP_PORT}}"
port="${TUBESYNC_POT_PORT:+:}${TUBESYNC_POT_PORT}"
if [ -n "${ip_address}" ]
then
YT_POT_BGUTIL_BASE_URL="${https:-http}://${ip_address}${port}"
export YT_POT_BGUTIL_BASE_URL
fi
exec /usr/bin/openresty -c /etc/nginx/nginx.conf -e stderr
tubesync/common/json.py
+10
View File
@@ -1,4 +1,6 @@
from datetime import datetime
from django.core.serializers.json import DjangoJSONEncoder
from yt_dlp.utils import LazyList
class JSONEncoder(DjangoJSONEncoder):
@@ -14,3 +16,11 @@ class JSONEncoder(DjangoJSONEncoder):
return list(iterable)
return super().default(obj)
def json_serial(obj):
if isinstance(obj, datetime):
return obj.isoformat()
if isinstance(obj, LazyList):
return list(obj)
raise TypeError(f'Type {type(obj)} is not json_serial()-able')
tubesync/common/urls.py
-1
View File
@@ -1,7 +1,6 @@
from django.conf import settings
from django.urls import path
from django.views.generic.base import RedirectView
from django.views.generic import TemplateView
from django.http import HttpResponse
from .views import error403, error404, error500, HealthCheckView
tubesync/common/utils.py
+8 -13
View File
@@ -6,13 +6,11 @@ import os
import pstats
import string
import time
from datetime import datetime
from django.core.paginator import Paginator
from functools import partial
from operator import attrgetter, itemgetter
from pathlib import Path
from urllib.parse import urlunsplit, urlencode, urlparse
from yt_dlp.utils import LazyList
from .errors import DatabaseConnectionError
@@ -140,14 +138,11 @@ def parse_database_connection_string(database_connection_string):
f'invalid driver, must be one of {valid_drivers}')
django_driver = django_backends.get(driver)
host_parts = user_pass_host_port.split('@')
if len(host_parts) != 2:
raise DatabaseConnectionError(f'Database connection string netloc must be in '
f'the format of user:pass@host')
user_pass_parts = host_parts[0].split(':')
if len(host_parts) != 2 or len(user_pass_parts) != 2:
raise DatabaseConnectionError('Database connection string netloc must be in '
'the format of user:pass@host')
user_pass, host_port = host_parts
user_pass_parts = user_pass.split(':')
if len(user_pass_parts) != 2:
raise DatabaseConnectionError(f'Database connection string netloc must be in '
f'the format of user:pass@host')
username, password = user_pass_parts
host_port_parts = host_port.split(':')
if len(host_port_parts) == 1:
@@ -169,13 +164,13 @@ def parse_database_connection_string(database_connection_string):
f'65535, got {port}')
else:
# Malformed
raise DatabaseConnectionError(f'Database connection host must be a hostname or '
f'a hostname:port combination')
raise DatabaseConnectionError('Database connection host must be a hostname or '
'a hostname:port combination')
if database.startswith('/'):
database = database[1:]
if not database:
raise DatabaseConnectionError(f'Database connection string path must be a '
f'string in the format of /databasename')
raise DatabaseConnectionError('Database connection string path must be a '
'string in the format of /databasename')
if '/' in database:
raise DatabaseConnectionError(f'Database connection string path can only '
f'contain a single string name, got: {database}')
tubesync/sync/fields.py
+1 -1
View File
@@ -63,7 +63,7 @@ class CommaSepChoiceField(models.CharField):
def __init__(self, *args, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, **kwargs):
kwargs.setdefault('max_length', 128)
self.separator = str(separator)
self.possible_choices = possible_choices or choices
self.possible_choices = possible_choices or kwargs.get('choices')
self.selected_choices = list()
self.allow_all = allow_all
self.all_label = all_label
tubesync/sync/filtering.py
-1
View File
@@ -5,7 +5,6 @@
from common.logger import log
from .models import Media
from datetime import datetime
from django.utils import timezone
from .overrides.custom_filter import filter_custom
tubesync/sync/hooks.py
-2
View File
@@ -1,9 +1,7 @@
import os
import yt_dlp
from common.logger import log
from common.utils import remove_enclosed
from django.conf import settings
progress_hook = {
tubesync/sync/management/commands/delete-source.py
+3 -4
View File
@@ -1,16 +1,15 @@
import os
import uuid
from django.utils.translation import gettext_lazy as _
from django.core.management.base import BaseCommand, CommandError
from django.db.transaction import atomic
from django.utils.translation import gettext_lazy as _
from common.logger import log
from sync.models import Source, Media, MediaServer
from sync.models import Source
from sync.tasks import schedule_media_servers_update
class Command(BaseCommand):
help = _('Deletes a source by UUID')
help = 'Deletes a source by UUID'
def add_arguments(self, parser):
parser.add_argument('--source', action='store', required=True, help=_('Source UUID'))
tubesync/sync/management/commands/import-existing-media.py
+2 -2
View File
@@ -1,6 +1,6 @@
import os
from pathlib import Path
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand, CommandError # noqa
from common.logger import log
from common.timestamp import timestamp_to_datetime
from sync.choices import FileExtension
@@ -18,7 +18,7 @@ class Command(BaseCommand):
dirmap = {}
for s in Source.objects.all():
dirmap[str(s.directory_path)] = s
log.info(f'Scanning sources...')
log.info('Scanning sources...')
file_extensions = list(FileExtension.values) + self.extra_extensions
for sourceroot, source in dirmap.items():
media = list(Media.objects.filter(source=source, downloaded=False,
tubesync/sync/management/commands/list-sources.py
+2 -3
View File
@@ -1,7 +1,6 @@
import os
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand, CommandError # noqa
from common.logger import log
from sync.models import Source, Media, MediaServer
from sync.models import Source
class Command(BaseCommand):
tubesync/sync/management/commands/reset-tasks.py
+4 -6
View File
@@ -1,14 +1,12 @@
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand, CommandError # noqa
from django.db.transaction import atomic
from django.utils.translation import gettext_lazy as _
from background_task.models import Task
from common.logger import log
from sync.models import Source
from sync.tasks import index_source_task, check_source_directory_exists
from common.logger import log
class Command(BaseCommand):
help = 'Resets all tasks'
@@ -31,10 +29,10 @@ class Command(BaseCommand):
index_source_task(
str(source.pk),
repeat=source.index_schedule,
schedule=source.index_schedule,
verbose_name=verbose_name.format(source.name),
)
with atomic(durable=True):
for source in Source.objects.all():
# This also chains down to call each Media objects .save() as well
source.save()
log.info('Done')
tubesync/sync/management/commands/sync-missing-metadata.py
+1 -2
View File
@@ -1,6 +1,5 @@
import os
from shutil import copyfile
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand, CommandError # noqa
from django.db.models import Q
from common.logger import log
from sync.models import Source, Media
tubesync/sync/management/commands/youtube-dl-info.py
+3 -3
View File
@@ -1,7 +1,7 @@
import json
from django.core.management.base import BaseCommand, CommandError
from django.core.management.base import BaseCommand, CommandError # noqa
from sync.youtube import get_media_info
from common.utils import json_serial
from common.json import JSONEncoder
class Command(BaseCommand):
@@ -15,6 +15,6 @@ class Command(BaseCommand):
url = options['url']
self.stdout.write(f'Showing information for URL: {url}')
info = get_media_info(url)
d = json.dumps(info, indent=4, sort_keys=True, default=json_serial)
d = json.dumps(info, indent=4, sort_keys=True, cls=JSONEncoder)
self.stdout.write(d)
self.stdout.write('Done')
tubesync/sync/mediaservers.py
-3
View File
@@ -117,9 +117,6 @@ class PlexMediaServer(MediaServer):
raise ValidationError('Plex Media Server "port" must be between 1 '
'and 65535')
options = self.object.options
if 'token' not in options:
raise ValidationError('Plex Media Server requires a "token"')
token = options['token'].strip()
if 'token' not in options:
raise ValidationError('Plex Media Server requires a "token"')
if 'libraries' not in options:
tubesync/sync/migrations/0011_auto_20220201_1654.py
-2
View File
@@ -1,8 +1,6 @@
# Generated by Django 3.2.11 on 2022-02-01 16:54
import django.core.files.storage
from django.db import migrations, models
import sync.models
class Migration(migrations.Migration):
tubesync/sync/migrations/0013_fix_elative_media_file.py
+1 -1
View File
@@ -1,7 +1,7 @@
# Generated by Django 3.2.12 on 2022-04-06 06:19
from django.conf import settings
from django.db import migrations, models
from django.db import migrations
def fix_media_file(apps, schema_editor):
tubesync/sync/models/__init__.py
+6
View File
@@ -17,3 +17,9 @@ from .media import Media
from .metadata import Metadata
from .metadata_format import MetadataFormat
__all__ = [
'get_media_file_path', 'get_media_thumb_path',
'media_file_storage', 'MediaServer', 'Source',
'Media', 'Metadata', 'MetadataFormat',
]
tubesync/sync/models/_private.py
+1 -1
View File
@@ -1,4 +1,4 @@
from ..choices import Val, YouTube_SourceType
from ..choices import Val, YouTube_SourceType # noqa
_srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) ))
tubesync/sync/models/media.py
+13 -5
View File
@@ -15,9 +15,9 @@ from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from common.logger import log
from common.errors import NoFormatException
from common.json import JSONEncoder
from common.utils import (
clean_filename, clean_emoji,
django_queryset_generator as qs_gen,
)
from ..youtube import (
get_media_info as get_youtube_media_info,
@@ -40,6 +40,9 @@ from ._migrations import (
media_file_storage, get_media_thumb_path, get_media_file_path,
)
from ._private import _srctype_dict, _nfo_element
from .media__tasks import (
download_checklist, download_finished, wait_for_premiere,
)
from .source import Source
@@ -575,14 +578,13 @@ class Media(models.Model):
def metadata_dumps(self, arg_dict=dict()):
from common.utils import json_serial
fallback = dict()
try:
fallback.update(self.new_metadata.with_formats)
except ObjectDoesNotExist:
pass
data = arg_dict or fallback
return json.dumps(data, separators=(',', ':'), default=json_serial)
return json.dumps(data, separators=(',', ':'), cls=JSONEncoder)
def metadata_loads(self, arg_str='{}'):
@@ -685,7 +687,7 @@ class Media(models.Model):
pass
setattr(self, '_cached_metadata_dict', data)
return data
except Exception as e:
except Exception:
return {}
@@ -1216,6 +1218,12 @@ class Media(models.Model):
parent_dir.rmdir()
log.info(f'Removed empty directory: {parent_dir!s}')
parent_dir = parent_dir.parent
except OSError as e:
except OSError:
pass
# add imported functions
Media.download_checklist = download_checklist
Media.download_finished = download_finished
Media.wait_for_premiere = wait_for_premiere
tubesync/sync/models/media__tasks.py
+118
View File
@@ -0,0 +1,118 @@
import os
from pathlib import Path
from common.logger import log
from common.errors import (
NoMetadataException,
)
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from ..choices import Val, SourceResolution
def download_checklist(self, skip_checks=False):
media = self
if skip_checks:
return True
if not media.source.download_media:
log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
f'the source {media.source} has since been marked to not download, '
f'not downloading')
return False
if media.skip or media.manual_skip:
# Media was toggled to be skipped after the task was scheduled
log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
f'it is now marked to be skipped, not downloading')
return False
# metadata is required to generate the proper filepath
if not media.has_metadata:
raise NoMetadataException('Metadata is not yet available.')
downloaded_file_exists = (
media.downloaded and
media.has_metadata and
(
media.media_file_exists or
media.filepath.exists()
)
)
if downloaded_file_exists:
# Media has been marked as downloaded before the download_media task was fired,
# skip it
log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
f'it has already been marked as downloaded, not downloading again')
return False
max_cap_age = media.source.download_cap_date
published = media.published
if max_cap_age and published:
if published <= max_cap_age:
log.warn(f'Download task triggered media: {media} (UUID: {media.pk}) but '
f'the source has a download cap and the media is now too old, '
f'not downloading')
return False
return True
def download_finished(self, format_str, container, downloaded_filepath=None):
media = self
if downloaded_filepath is None:
downloaded_filepath = self.filepath
filepath = Path(downloaded_filepath)
# Media has been downloaded successfully
log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: '
f'"{filepath}"')
# Link the media file to the object and update info about the download
self.media_file.name = str(filepath.relative_to(self.media_file.storage.location))
media.downloaded = True
media.download_date = timezone.now()
media.downloaded_filesize = os.path.getsize(filepath)
media.downloaded_container = container
if '+' in format_str:
# Seperate audio and video streams
vformat_code, aformat_code = format_str.split('+')
aformat = media.get_format_by_code(aformat_code)
vformat = media.get_format_by_code(vformat_code)
media.downloaded_format = vformat['format']
media.downloaded_height = vformat['height']
media.downloaded_width = vformat['width']
media.downloaded_audio_codec = aformat['acodec']
media.downloaded_video_codec = vformat['vcodec']
media.downloaded_container = container
media.downloaded_fps = vformat['fps']
media.downloaded_hdr = vformat['is_hdr']
else:
# Combined stream or audio-only stream
cformat_code = format_str
cformat = media.get_format_by_code(cformat_code)
media.downloaded_audio_codec = cformat['acodec']
if cformat['vcodec']:
# Combined
media.downloaded_format = cformat['format']
media.downloaded_height = cformat['height']
media.downloaded_width = cformat['width']
media.downloaded_video_codec = cformat['vcodec']
media.downloaded_fps = cformat['fps']
media.downloaded_hdr = cformat['is_hdr']
else:
self.downloaded_format = Val(SourceResolution.AUDIO)
def wait_for_premiere(self):
hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60)))
in_hours = None
if self.has_metadata or not self.published:
return (False, in_hours,)
now = timezone.now()
if self.published < now:
in_hours = 0
self.manual_skip = False
self.skip = False
else:
in_hours = hours(self.published - now)
self.manual_skip = True
self.title = _(f'Premieres in {in_hours} hours')
return (True, in_hours,)
tubesync/sync/models/source.py
+18 -9
View File
@@ -1,6 +1,7 @@
import os
import re
import uuid
from collections import deque as queue
from pathlib import Path
from django import db
from django.conf import settings
@@ -510,7 +511,7 @@ class Source(db.models.Model):
def get_example_media_format(self):
try:
return self.media_format.format(**self.example_media_format_dict)
except Exception as e:
except Exception:
return ''
def is_regex_match(self, media_item_title):
@@ -527,23 +528,31 @@ class Source(db.models.Model):
days = timezone.timedelta(seconds=self.download_cap).days
response = indexer(self.get_index_url(type=type), days=days)
if not isinstance(response, dict):
return []
entries = response.get('entries', [])
return list()
entries = response.get('entries', list())
return entries
def index_media(self):
'''
Index the media source returning a list of media metadata as dicts.
Index the media source returning a queue of media metadata as dicts.
'''
entries = list()
entries = queue(list(), getattr(settings, 'MAX_ENTRIES_PROCESSING', 0) or None)
if self.index_videos:
entries += self.get_index('videos')
entries.extend(reversed(self.get_index('videos')))
# Playlists do something different that I have yet to figure out
if not self.is_playlist:
if self.index_streams:
entries += self.get_index('streams')
streams = self.get_index('streams')
if entries.maxlen is None or 0 == len(entries):
entries.extend(reversed(streams))
else:
# share the queue between streams and videos
allowed_streams = max(
entries.maxlen // 2,
entries.maxlen - len(entries),
)
entries.extend(reversed(streams[: allowed_streams]))
if settings.MAX_ENTRIES_PROCESSING:
entries = entries[:settings.MAX_ENTRIES_PROCESSING]
return entries
tubesync/sync/signals.py
+5 -5
View File
@@ -10,11 +10,11 @@ from django.utils.translation import gettext_lazy as _
from background_task.signals import task_failed
from background_task.models import Task
from common.logger import log
from .models import Source, Media, MediaServer, Metadata
from .models import Source, Media, Metadata
from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task,
download_media_thumbnail, download_media_metadata,
map_task_to_instance, check_source_directory_exists,
download_media, rescan_media_server, download_source_images,
download_media, download_source_images,
delete_all_media_for_source, save_all_media_for_source,
rename_media, get_media_metadata_task, get_media_download_task)
from .utils import delete_file, glob_quote, mkdir_p
@@ -270,7 +270,7 @@ def media_post_save(sender, instance, created, **kwargs):
if not (media_file_exists or existing_media_download_task):
# The file was deleted after it was downloaded, skip this media.
if instance.can_download and instance.downloaded:
skip_changed = True != instance.skip
skip_changed = True if not instance.skip else False
instance.skip = True
downloaded = False
if (instance.source.download_media and instance.can_download) and not (
@@ -374,13 +374,13 @@ def media_post_delete(sender, instance, **kwargs):
try:
p.rmdir()
log.info(f'Deleted directory for: {instance} path: {p!s}')
except OSError as e:
except OSError:
pass
# Delete the directory itself
try:
other_path.rmdir()
log.info(f'Deleted directory for: {instance} path: {other_path!s}')
except OSError as e:
except OSError:
pass
# Get all files that start with the bare file path
all_related_files = video_path.parent.glob(f'{glob_quote(video_path.with_suffix("").name)}*')
tubesync/sync/tasks.py
+60 -99
View File
@@ -6,7 +6,6 @@
import os
import json
import math
import random
import requests
import time
@@ -14,9 +13,8 @@ import uuid
from io import BytesIO
from hashlib import sha1
from pathlib import Path
from datetime import datetime, timedelta
from datetime import timedelta
from shutil import copyfile, rmtree
from PIL import Image
from django import db
from django.conf import settings
from django.core.files.base import ContentFile
@@ -30,13 +28,13 @@ from background_task.exceptions import InvalidTaskError
from background_task.models import Task, CompletedTask
from common.logger import log
from common.errors import ( NoFormatException, NoMediaException,
NoMetadataException, NoThumbnailException,
NoThumbnailException,
DownloadFailedException, )
from common.utils import ( django_queryset_generator as qs_gen,
remove_enclosed, )
from .choices import Val, TaskQueue
from .models import Source, Media, MediaServer
from .utils import ( get_remote_image, resize_image_to_height, delete_file,
from .utils import ( get_remote_image, resize_image_to_height,
write_text_file, filter_response, )
from .youtube import YouTubeError
@@ -122,7 +120,7 @@ def get_error_message(task):
def update_task_status(task, status):
if not task:
return False
if not task._verbose_name:
if not hasattr(task, '_verbose_name'):
task._verbose_name = remove_enclosed(
task.verbose_name, '[', ']', ' ',
)
@@ -226,7 +224,7 @@ def save_model(instance):
@atomic(durable=False)
def schedule_media_servers_update():
# Schedule a task to update media servers
log.info(f'Scheduling media server updates')
log.info('Scheduling media server updates')
verbose_name = _('Request media server rescan for "{}"')
for mediaserver in MediaServer.objects.all():
rescan_media_server(
@@ -235,6 +233,35 @@ def schedule_media_servers_update():
)
def wait_for_errors(model, /, *, task_name=None):
if task_name is None:
task_name=tuple((
'sync.tasks.download_media',
'sync.tasks.download_media_metadata',
))
elif isinstance(task_name, str):
task_name = tuple((task_name,))
tasks = list()
for tn in task_name:
ft = get_first_task(tn, instance=model)
if ft:
tasks.append(ft)
window = timezone.timedelta(hours=3) + timezone.now()
tqs = Task.objects.filter(
task_name__in=task_name,
attempts__gt=0,
locked_at__isnull=True,
run_at__lte=window,
last_error__contains='HTTPError 429: Too Many Requests',
)
for task in tasks:
update_task_status(task, 'paused (429)')
log.info(f'waiting for errors: 429 ({tqs.count()}): {model}')
time.sleep(10 * tqs.count())
for task in tasks:
update_task_status(task, None)
def cleanup_old_media():
with atomic():
for source in qs_gen(Source.objects.filter(delete_old_media=True, days_to_keep__gt=0)):
@@ -255,7 +282,7 @@ def cleanup_old_media():
schedule_media_servers_update()
def cleanup_removed_media(source, videos):
def cleanup_removed_media(source, video_keys):
if not source.delete_removed_media:
return
log.info(f'Cleaning up media no longer in source: {source}')
@@ -265,8 +292,7 @@ def cleanup_removed_media(source, videos):
source=source,
)
for media in qs_gen(mqs):
matching_source_item = [video['id'] for video in videos if video['id'] == media.key]
if not matching_source_item:
if media.key not in video_keys:
log.info(f'{media.name} is no longer in source, removing')
with atomic():
media.delete()
@@ -316,12 +342,17 @@ def index_source_task(source_id):
end=task.verbose_name.find('Index'),
)
tvn_format = '{:,}' + f'/{num_videos:,}'
for vn, video in enumerate(videos, start=1):
vn = 0
video_keys = set()
while len(videos) > 0:
vn += 1
video = videos.popleft()
# Create or update each video as a Media object
key = video.get(source.key_field, None)
if not key:
# Video has no unique key (ID), it can't be indexed
continue
video_keys.add(key)
update_task_status(task, tvn_format.format(vn))
# media, new_media = Media.objects.get_or_create(key=key, source=source)
try:
@@ -376,7 +407,7 @@ def index_source_task(source_id):
# Reset task.verbose_name to the saved value
update_task_status(task, None)
# Cleanup of media no longer available from the source
cleanup_removed_media(source, videos)
cleanup_removed_media(source, video_keys)
videos = video = None
@@ -416,7 +447,7 @@ def download_source_images(source_id):
log.info(f'Thumbnail URL for source with ID: {source_id} / {source} '
f'Avatar: {avatar} '
f'Banner: {banner}')
if banner != None:
if banner is not None:
url = banner
i = get_remote_image(url)
image_file = BytesIO()
@@ -432,7 +463,7 @@ def download_source_images(source_id):
f.write(django_file.read())
i = image_file = None
if avatar != None:
if avatar is not None:
url = avatar
i = get_remote_image(url)
image_file = BytesIO()
@@ -467,6 +498,7 @@ def download_media_metadata(media_id):
log.info(f'Task for ID: {media_id} / {media} skipped, due to task being manually skipped.')
return
source = media.source
wait_for_errors(media, task_name='sync.tasks.download_media_metadata')
try:
metadata = media.index_metadata()
except YouTubeError as e:
@@ -604,7 +636,7 @@ def download_media_thumbnail(media_id, url):
@background(schedule=dict(priority=30, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True)
def download_media(media_id):
def download_media(media_id, override=False):
'''
Downloads the media to disk and attaches it to the Media instance.
'''
@@ -613,41 +645,13 @@ def download_media(media_id):
except Media.DoesNotExist as e:
# Task triggered but the media no longer exists, do nothing
raise InvalidTaskError(_('no such media')) from e
if not media.source.download_media:
log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
f'the source {media.source} has since been marked to not download, '
f'not downloading')
return
if media.skip or media.manual_skip:
# Media was toggled to be skipped after the task was scheduled
log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
f'it is now marked to be skipped, not downloading')
return
# metadata is required to generate the proper filepath
if not media.has_metadata:
raise NoMetadataException('Metadata is not yet available.')
downloaded_file_exists = (
media.downloaded and
media.has_metadata and
(
media.media_file_exists or
media.filepath.exists()
)
)
if downloaded_file_exists:
# Media has been marked as downloaded before the download_media task was fired,
# skip it
log.warn(f'Download task triggered for media: {media} (UUID: {media.pk}) but '
f'it has already been marked as downloaded, not downloading again')
return
max_cap_age = media.source.download_cap_date
published = media.published
if max_cap_age and published:
if published <= max_cap_age:
log.warn(f'Download task triggered media: {media} (UUID: {media.pk}) but '
f'the source has a download cap and the media is now too old, '
f'not downloading')
else:
if not media.download_checklist(override):
# any condition that needs to reschedule the task
# should raise an exception to avoid this
return
wait_for_errors(media, task_name='sync.tasks.download_media')
filepath = media.filepath
container = format_str = None
log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"')
@@ -680,42 +684,7 @@ def download_media(media_id):
raise DownloadFailedException(err)
# Media has been downloaded successfully
log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: '
f'"{filepath}"')
# Link the media file to the object and update info about the download
media.media_file.name = str(media.source.type_directory_path / media.filename)
media.downloaded = True
media.download_date = timezone.now()
media.downloaded_filesize = os.path.getsize(filepath)
media.downloaded_container = container
if '+' in format_str:
# Seperate audio and video streams
vformat_code, aformat_code = format_str.split('+')
aformat = media.get_format_by_code(aformat_code)
vformat = media.get_format_by_code(vformat_code)
media.downloaded_format = vformat['format']
media.downloaded_height = vformat['height']
media.downloaded_width = vformat['width']
media.downloaded_audio_codec = aformat['acodec']
media.downloaded_video_codec = vformat['vcodec']
media.downloaded_container = container
media.downloaded_fps = vformat['fps']
media.downloaded_hdr = vformat['is_hdr']
else:
# Combined stream or audio-only stream
cformat_code = format_str
cformat = media.get_format_by_code(cformat_code)
media.downloaded_audio_codec = cformat['acodec']
if cformat['vcodec']:
# Combined
media.downloaded_format = cformat['format']
media.downloaded_height = cformat['height']
media.downloaded_width = cformat['width']
media.downloaded_video_codec = cformat['vcodec']
media.downloaded_fps = cformat['fps']
media.downloaded_hdr = cformat['is_hdr']
else:
media.downloaded_format = 'audio'
media.download_finished(format_str, container, filepath)
save_model(media)
# If selected, copy the thumbnail over as well
if media.source.copy_thumbnails:
@@ -887,26 +856,18 @@ def rename_all_media_for_source(source_id):
@background(schedule=dict(priority=0, run_at=60), queue=Val(TaskQueue.DB), remove_existing_tasks=True)
def wait_for_media_premiere(media_id):
hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60)))
try:
media = Media.objects.get(pk=media_id)
except Media.DoesNotExist as e:
raise InvalidTaskError(_('no such media')) from e
else:
if media.has_metadata:
valid, hours = media.wait_for_premiere()
if not valid:
return
now = timezone.now()
if media.published < now:
media.manual_skip = False
media.skip = False
# the download tasks start after the media is saved
else:
media.manual_skip = True
media.title = _(f'Premieres in {hours(media.published - now)} hours')
if hours:
task = get_media_premiere_task(media_id)
if task:
update_task_status(task, f'available in {hours(media.published - now)} hours')
update_task_status(task, f'available in {hours} hours')
save_model(media)
@@ -918,7 +879,7 @@ def delete_all_media_for_source(source_id, source_name, source_directory):
assert source_directory
try:
source = Source.objects.get(pk=source_id)
except Source.DoesNotExist as e:
except Source.DoesNotExist:
# Task triggered but the source no longer exists, do nothing
log.warn(f'Task delete_all_media_for_source(pk={source_id}) called but no '
f'source exists with ID: {source_id}')
tubesync/sync/tests.py
+5 -6
View File
@@ -6,7 +6,6 @@
import logging
import os
from datetime import datetime, timedelta
from pathlib import Path
from urllib.parse import urlsplit
@@ -1822,13 +1821,13 @@ class TasksTestCase(TestCase):
now = timezone.now()
m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5))
m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25))
m13 = Media.objects.create(source=src1, downloaded=False, key='a13')
m11 = Media.objects.create(source=src1, downloaded=True, key='a11', download_date=now - timedelta(days=5)) # noqa
m12 = Media.objects.create(source=src1, downloaded=True, key='a12', download_date=now - timedelta(days=25)) # noqa
m13 = Media.objects.create(source=src1, downloaded=False, key='a13') # noqa
m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5))
m21 = Media.objects.create(source=src2, downloaded=True, key='a21', download_date=now - timedelta(days=5)) # noqa
m22 = Media.objects.create(source=src2, downloaded=True, key='a22', download_date=now - timedelta(days=25))
m23 = Media.objects.create(source=src2, downloaded=False, key='a23')
m23 = Media.objects.create(source=src2, downloaded=False, key='a23') # noqa
self.assertEqual(src1.media_source.all().count(), 3)
self.assertEqual(src2.media_source.all().count(), 3)
tubesync/sync/views.py
+14 -7
View File
@@ -35,7 +35,7 @@ from .tasks import (map_task_to_instance, get_error_message,
from .choices import (Val, MediaServerType, SourceResolution, IndexSchedule,
YouTube_SourceType, youtube_long_source_types,
youtube_help, youtube_validation_urls)
from . import signals
from . import signals # noqa
from . import youtube
@@ -258,7 +258,7 @@ class ValidateSourceView(FormView):
self.key = youtube.get_channel_id(
index_url.replace('/channel/', '/')
)
except youtube.YouTubeError as e:
except youtube.YouTubeError:
# It did not work, revert to previous behavior
self.key = old_key
self.source_type = old_source_type
@@ -296,10 +296,13 @@ class EditSourceMixin:
def form_valid(self, form: Form):
# Perform extra validation to make sure the media_format is valid
obj = form.save(commit=False)
source_type = form.cleaned_data['media_format']
# temporarily use media_format from the form
saved_media_format = obj.media_format
obj.media_format = form.cleaned_data['media_format']
example_media_file = obj.get_example_media_format()
obj.media_format = saved_media_format
if example_media_file == '':
if '' == example_media_file:
form.add_error(
'media_format',
ValidationError(self.errors['invalid_media_format'])
@@ -307,12 +310,16 @@ class EditSourceMixin:
# Check for suspicious file path(s)
try:
targetCheck = form.cleaned_data['directory']+"/.virt"
newdir = safe_join(settings.DOWNLOAD_ROOT,targetCheck)
targetCheck = form.cleaned_data['directory'] + '/.virt'
safe_join(settings.DOWNLOAD_ROOT, targetCheck)
except SuspiciousFileOperation:
form.add_error(
'directory',
ValidationError(self.errors['dir_outside_dlroot'].replace("%BASEDIR%",str(settings.DOWNLOAD_ROOT)))
ValidationError(
self.errors['dir_outside_dlroot'].replace(
"%BASEDIR%", str(settings.DOWNLOAD_ROOT)
)
),
)
if form.errors:
tubesync/sync/youtube.py
+5 -9
View File
@@ -6,7 +6,6 @@
import os
from collections import namedtuple
from common.logger import log
from copy import deepcopy
from pathlib import Path
@@ -102,7 +101,7 @@ def get_channel_image_info(url):
avatar_url = thumbnail['url']
if thumbnail['id'] == 'banner_uncropped':
banner_url = thumbnail['url']
if banner_url != None and avatar_url != None:
if banner_url is not None and avatar_url is not None:
break
return avatar_url, banner_url
@@ -143,7 +142,7 @@ def get_media_info(url, /, *, days=None, info_json=None):
if days is not None:
try:
days = int(str(days), 10)
except Exception as e:
except (TypeError, ValueError):
days = None
start = (
f'yesterday-{days!s}days' if days else None
@@ -326,13 +325,10 @@ def download_media(
final_path = Path(output_file).resolve(strict=False)
expected_file = shell_quote(str(final_path))
cmds = pp_opts.exec_cmd.get('after_move', list())
# It is important that we use a tuple for strings.
# Otherwise, list adds each character instead.
# That last comma is really necessary!
cmds += (
cmds.append(
f'test -f {expected_file} || '
'mv -T -u -- %(filepath,_filename|)q '
f'{expected_file}',
f'{expected_file}'
)
# assignment is the quickest way to cover both 'get' cases
pp_opts.exec_cmd['after_move'] = cmds
@@ -387,7 +383,7 @@ def download_media(
youtube_ea_dict = ytopts['extractor_args'].get('youtube', dict())
formats_list = youtube_ea_dict.get('formats', list())
if 'missing_pot' not in formats_list:
formats_list += ('missing_pot',)
formats_list.append('missing_pot')
youtube_ea_dict.update({
'formats': formats_list,
})
tubesync/tubesync/settings.py
+3 -3
View File
@@ -8,7 +8,7 @@ CONFIG_BASE_DIR = BASE_DIR
DOWNLOADS_BASE_DIR = BASE_DIR
VERSION = '0.15.2'
VERSION = '0.15.3'
SECRET_KEY = ''
DEBUG = False
ALLOWED_HOSTS = []
@@ -198,7 +198,7 @@ RENAME_SOURCES = None
# You have been warned!
try:
from .local_settings import *
from .local_settings import * # noqa
except ImportError as e:
import sys
sys.stderr.write(f'Unable to import local_settings: {e}\n')
@@ -222,5 +222,5 @@ if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS:
BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS
from .dbutils import patch_ensure_connection
from .dbutils import patch_ensure_connection # noqa
patch_ensure_connection()