Merge branch 'main' into patch-2

README.md

@@ -63,7 +63,7 @@ directory will be a `video` and `audio` subdirectories. All media which only has
 audio stream (such as music) will download to the `audio` directory. All media with a
 video stream will be downloaded to the `video` directory. All administration of
 TubeSync is performed via a web interface. You can optionally add a media server,
-currently just Plex, to complete the PVR experience.
+currently only Jellyfin or Plex, to complete the PVR experience.
 
 
 # Installation
@@ -221,7 +221,7 @@ As media is indexed and downloaded it will appear in the "media" tab.
 
 ### 3. Media Server updating
 
-Currently TubeSync supports Plex as a media server. You can add your local Plex server
+Currently TubeSync supports Plex and Jellyfin as media servers. You can add your local Jellyfin or Plex server
 under the "media servers" tab.
 
 
@@ -234,6 +234,13 @@ view these with:
 $ docker logs --follow tubesync
 ```
 
+To include logs with an issue report, please exteact a file and attach it to the issue.
+The command below creates the `TubeSync.logs.txt` file with the logs from the `tubesync` container:
+
+```bash
+docker logs -t tubesync > TubeSync.logs.txt 2>&1
+```
+
 
 # Advanced usage guides
 
@@ -371,22 +378,26 @@ There are a number of other environment variables you can set. These are, mostly
 **NOT** required to be set in the default container installation, they are really only
 useful if you are manually installing TubeSync in some other environment. These are:
 
-| Name                         | What                                                          | Example                              |
-| ---------------------------- | ------------------------------------------------------------- |--------------------------------------|
-| DJANGO_SECRET_KEY            | Django's SECRET_KEY                                           | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l     |
-| DJANGO_URL_PREFIX            | Run TubeSync in a sub-URL on the web server                   | /somepath/                           |
-| TUBESYNC_DEBUG               | Enable debugging                                              | True                                 |
-| TUBESYNC_WORKERS             | Number of background workers, default is 2, max allowed is 8  | 2                                    |
-| TUBESYNC_HOSTS               | Django's ALLOWED_HOSTS, defaults to `*`                       | tubesync.example.com,otherhost.com   |
-| TUBESYNC_RESET_DOWNLOAD_DIR  | Toggle resetting `/downloads` permissions, defaults to True   | True                                 |
-| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download         | 240                                  |
-| TUBESYNC_DIRECTORY_PREFIX    | Enable `video` and `audio` directory prefixes in `/downloads` | True                                 |
-| GUNICORN_WORKERS             | Number of gunicorn workers to spawn                           | 3                                    |
-| LISTEN_HOST                  | IP address for gunicorn to listen on                          | 127.0.0.1                            |
-| LISTEN_PORT                  | Port number for gunicorn to listen on                         | 8080                                 |
-| HTTP_USER                    | Sets the username for HTTP basic authentication               | some-username                        |
-| HTTP_PASS                    | Sets the password for HTTP basic authentication               | some-secure-password                 |
-| DATABASE_CONNECTION          | Optional external database connection details                 | mysql://user:pass@host:port/database |
+| Name                         | What                                                          | Example                                                                       |
+| ---------------------------- | ------------------------------------------------------------- |-------------------------------------------------------------------------------|
+| DJANGO_SECRET_KEY            | Django's SECRET_KEY                                           | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l                   |
+| DJANGO_URL_PREFIX            | Run TubeSync in a sub-URL on the web server                   | /somepath/                                                                    |
+| TUBESYNC_DEBUG               | Enable debugging                                              | True                                                                          |
+| TUBESYNC_HOSTS               | Django's ALLOWED_HOSTS, defaults to `*`                       | tubesync.example.com,otherhost.com                 |
+| TUBESYNC_RESET_DOWNLOAD_DIR  | Toggle resetting `/downloads` permissions, defaults to True   | True                                                                          |
+| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download         | 240                                                                           |
+| TUBESYNC_RENAME_SOURCES      | Rename media files from selected sources                      | Source1_directory,Source2_directory                |
+| TUBESYNC_RENAME_ALL_SOURCES  | Rename media files from all sources                           | True                                                                          |
+| TUBESYNC_DIRECTORY_PREFIX    | Enable `video` and `audio` directory prefixes in `/downloads` | True                                                                          |
+| TUBESYNC_SHRINK_NEW          | Filter unneeded information from newly retrieved metadata     | True                                                                          |
+| TUBESYNC_SHRINK_OLD          | Filter unneeded information from metadata loaded from the database | True                                                                     |
+| TUBESYNC_WORKERS             | Number of background threads per (task runner) process. Default is 1. Max allowed is 8. | 2                                                  |
+| GUNICORN_WORKERS             | Number of `gunicorn` (web request) workers to spawn           | 3                                                                             |
+| LISTEN_HOST                  | IP address for `gunicorn` to listen on                        | 127.0.0.1                                                                     |
+| LISTEN_PORT                  | Port number for `gunicorn` to listen on                       | 8080                                                                          |
+| HTTP_USER                    | Sets the username for HTTP basic authentication               | some-username                                                                 |
+| HTTP_PASS                    | Sets the password for HTTP basic authentication               | some-secure-password                                                          |
+| DATABASE_CONNECTION          | Optional external database connection details                 | postgresql://user:pass@host:port/database          |
 
 
 # Manual, non-containerised, installation
@@ -396,7 +407,7 @@ following this rough guide, you are on your own and should be knowledgeable abou
 installing and running WSGI-based Python web applications before attempting this.
 
 1. Clone or download this repo
-2. Make sure you're running a modern version of Python (>=3.6) and have Pipenv
+2. Make sure you're running a modern version of Python (>=3.9) and have Pipenv
    installed
 3. Set up the environment with `pipenv install`
 4. Copy `tubesync/tubesync/local_settings.py.example` to

tubesync/restart_services.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env sh
+
+dir='/run/service'
+svc_path() (
+    cd "${dir}"
+    realpath -e -s "$@"
+)
+
+if [ 0 -eq $# ]
+then
+    set -- \
+        $( cd "${dir}" && svc_path tubesync*-worker ) \
+        "$( svc_path gunicorn )" \
+        "$( svc_path nginx )"
+fi
+
+for service in $( svc_path "$@" )
+do
+    printf -- 'Restarting %-28s' "${service#${dir}/}..."
+    _began="$( date '+%s' )"
+    /command/s6-svc -wr -r "${service}"
+    _ended="$( date '+%s' )"
+    printf -- '\tcompleted (in %2.1d seconds).\n' \
+        "$( expr "${_ended}" - "${_began}" )"
+done
+unset -v _began _ended service

tubesync/sync/management/commands/delete-source.py

@@ -2,7 +2,7 @@ import os
 import uuid
 from django.utils.translation import gettext_lazy as _
 from django.core.management.base import BaseCommand, CommandError
-from django.db.models import signals
+from django.db.transaction import atomic
 from common.logger import log
 from sync.models import Source, Media, MediaServer
 from sync.tasks import schedule_media_servers_update
@@ -10,10 +10,10 @@ from sync.tasks import schedule_media_servers_update
 
 class Command(BaseCommand):
 
-    help = ('Deletes a source by UUID')
+    help = _('Deletes a source by UUID')
 
     def add_arguments(self, parser):
-        parser.add_argument('--source', action='store', required=True, help='Source UUID')
+        parser.add_argument('--source', action='store', required=True, help=_('Source UUID'))
 
     def handle(self, *args, **options):
         source_uuid_str = options.get('source', '')
@@ -29,13 +29,15 @@ class Command(BaseCommand):
             raise CommandError(f'Source does not exist with '
                                f'UUID: {source_uuid}')
         # Reconfigure the source to not update the disk or media servers
-        source.deactivate()
+        with atomic(durable=True):
+            source.deactivate()
         # Delete the source, triggering pre-delete signals for each media item
         log.info(f'Found source with UUID "{source.uuid}" with name '
                  f'"{source.name}" and deleting it, this may take some time!')
         log.info(f'Source directory: {source.directory_path}')
-        source.delete()
-        # Update any media servers
-        schedule_media_servers_update()
+        with atomic(durable=True):
+            source.delete()
+            # Update any media servers
+            schedule_media_servers_update()
         # All done
         log.info('Done')

tubesync/sync/mediaservers.py

@@ -5,6 +5,7 @@ from django.forms import ValidationError
 from urllib.parse import urlsplit, urlunsplit, urlencode
 from django.utils.translation import gettext_lazy as _
 from common.logger import log
+from django.conf import settings
 
 
 class MediaServerError(Exception):
@@ -18,14 +19,52 @@ class MediaServer:
 
     TIMEOUT = 0
     HELP = ''
+    default_headers = {'User-Agent': 'TubeSync'}
 
     def __init__(self, mediaserver_instance):
         self.object = mediaserver_instance
+        self.headers = dict(**self.default_headers)
+        self.token = None
+
+    def make_request_args(self, uri='/', token_header=None, headers={}, token_param=None, params={}):
+        base_parts = urlsplit(self.object.url)
+        if self.token is None:
+            self.token = self.object.loaded_options['token'] or None
+        if token_header and self.token:
+            headers.update({token_header: self.token})
+        self.headers.update(headers)
+        if token_param and self.token:
+            params.update({token_param: self.token})
+        qs = urlencode(params)
+        enable_verify = (
+            base_parts.scheme.endswith('s') and
+            self.object.verify_https
+        )
+        url = urlunsplit((base_parts.scheme, base_parts.netloc, uri, qs, ''))
+        return (url, dict(
+                headers=self.headers,
+                verify=enable_verify,
+                timeout=self.TIMEOUT,
+            ))
+
+    def make_request(self, uri='/', /, *, headers={}, params={}):
+        '''
+            A very simple implementation is:
+                url, kwargs = self.make_request_args(uri=uri, headers=headers, params=params)
+                return requests.get(url, **kwargs)
+        '''
+        raise NotImplementedError('MediaServer.make_request() must be implemented')
 
     def validate(self):
+        '''
+            Called to check that the configured media server values are correct.
+        '''
         raise NotImplementedError('MediaServer.validate() must be implemented')
 
     def update(self):
+        '''
+            Called after the `Media` instance has saved a downloaded file.
+        '''
         raise NotImplementedError('MediaServer.update() must be implemented')
 
 
@@ -48,30 +87,22 @@ class PlexMediaServer(MediaServer):
              '<a href="https://www.plexopedia.com/plex-media-server/api/server/libraries/" '
              'target="_blank">here</a></p>.')
 
-    def make_request(self, uri='/', params={}):
-        headers = {'User-Agent': 'TubeSync'}
-        token = self.object.loaded_options['token']
-        params['X-Plex-Token'] = token
-        base_parts = urlsplit(self.object.url)
-        qs = urlencode(params)
-        url = urlunsplit((base_parts.scheme, base_parts.netloc, uri, qs, ''))
-        if self.object.verify_https:
-            log.debug(f'[plex media server] Making HTTP GET request to: {url}')
-            return requests.get(url, headers=headers, verify=True,
-                                timeout=self.TIMEOUT)
-        else:
+    def make_request(self, uri='/', /, *, headers={}, params={}):
+        url, kwargs = self.make_request_args(uri=uri, headers=headers, token_param='X-Plex-Token', params=params)
+        log.debug(f'[plex media server] Making HTTP GET request to: {url}')
+        if self.object.use_https and not kwargs['verify']:
             # If not validating SSL, given this is likely going to be for an internal
             # or private network, that Plex issues certs *.hash.plex.direct and that
             # the warning won't ever been sensibly seen in the HTTPS logs, hide it
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore")
-                return requests.get(url, headers=headers, verify=False,
-                                    timeout=self.TIMEOUT)
+                return requests.get(url, **kwargs)
+        return requests.get(url, **kwargs)
 
     def validate(self):
         '''
             A Plex server requires a host, port, access token and a comma-separated
-            list if library IDs.
+            list of library IDs.
         '''
         # Check all the required values are present
         if not self.object.host:
@@ -172,27 +203,46 @@ class JellyfinMediaServer(MediaServer):
     HELP = _('<p>To connect your TubeSync server to your Jellyfin Media Server, please enter the details below.</p>'
              '<p>The <strong>host</strong> can be either an IP address or a valid hostname.</p>'
              '<p>The <strong>port</strong> should be between 1 and 65536.</p>'
-             '<p>The <strong>token</strong> is required for API access. You can generate a token in your Jellyfin user profile settings.</p>'
-             '<p>The <strong>libraries</strong> is a comma-separated list of library IDs in Jellyfin.</p>')
-
-    def make_request(self, uri='/', params={}, *, data={}, json=None, method='GET'):
-        headers = {
-            'User-Agent': 'TubeSync',
-            'X-Emby-Token': self.object.loaded_options['token']  # Jellyfin uses the same `X-Emby-Token` header as Emby
-        }
+             '<p>The "API Key" <strong>token</strong> is required for API access. Your Jellyfin administrator can generate an "API Key" token for use with TubeSync for you.</p>'
+             '<p>The <strong>libraries</strong> is a comma-separated list of library IDs in Jellyfin. Leave this blank to see a list.</p>')
 
+    def make_request(self, uri='/', /, *, headers={}, params={}, data={}, json=None, method='GET'):
         assert method in {'GET', 'POST'}, f'Unimplemented method: {method}'
-        url = f'{self.object.url}{uri}'
-        log.debug(f'[jellyfin media server] Making HTTP {method} request to: {url}')
+        
+        headers.update({'Content-Type': 'application/json'})
+        url, kwargs = self.make_request_args(uri=uri, token_header='X-Emby-Token', headers=headers, params=params)
+        # From the Emby source code;
+        #   this is the order in which the headers are tried:
+        # X-Emby-Authorization: ('MediaBrowser'|'Emby') 'Token'=<token_value>, 'Client'=<client_value>, 'Version'=<version_value>
+        # X-Emby-Token: <token_value>
+        # X-MediaBrowser-Token: <token_value>
+        # Jellyfin uses 'Authorization' first,
+        # then optionally falls back to the 'X-Emby-Authorization' header.
+        # Jellyfin uses (") around values, but not keys in that header.
+        token = kwargs['headers'].get('X-Emby-Token', None)
+        if token:
+            kwargs['headers'].update({
+                'X-MediaBrowser-Token': token,
+                'X-Emby-Authorization': f'Emby Token={token}, Client=TubeSync, Version={settings.VERSION}',
+                'Authorization': f'MediaBrowser Token="{token}", Client="TubeSync", Version="{settings.VERSION}"',
+            })
 
+        log.debug(f'[jellyfin media server] Making HTTP {method} request to: {url}')
+        if self.object.use_https and not kwargs['verify']:
+            # not verifying certificates
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                return requests.request(
+                    method, url,
+                    data=data,
+                    json=json,
+                    **kwargs,
+                )
         return requests.request(
             method, url,
-            headers=headers,
-            params=params,
             data=data,
             json=json,
-            verify=self.object.verify_https,
-            timeout=self.TIMEOUT,
+            **kwargs,
         )
 
     def validate(self):

tubesync/sync/migrations/0031_metadata_metadataformat.py

@@ -0,0 +1,52 @@
+# Generated by Django 5.1.8 on 2025-04-11 07:36
+
+import django.db.models.deletion
+import sync.models
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0030_alter_source_source_vcodec'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Metadata',
+            fields=[
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the metadata', primary_key=True, serialize=False, verbose_name='uuid')),
+                ('site', models.CharField(blank=True, default='Youtube', help_text='Site from which the metadata was retrieved', max_length=256, verbose_name='site')),
+                ('key', models.CharField(blank=True, default='', help_text='Media identifier at the site from which the metadata was retrieved', max_length=256, verbose_name='key')),
+                ('created', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was created', verbose_name='created')),
+                ('retrieved', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Date and time the metadata was retrieved', verbose_name='retrieved')),
+                ('uploaded', models.DateTimeField(help_text='Date and time the media was uploaded', null=True, verbose_name='uploaded')),
+                ('published', models.DateTimeField(help_text='Date and time the media was published', null=True, verbose_name='published')),
+                ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata object', verbose_name='value')),
+                ('media', models.ForeignKey(help_text='Media the metadata belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadata_media', to='sync.media')),
+            ],
+            options={
+                'verbose_name': 'Metadata about a Media item',
+                'verbose_name_plural': 'Metadata about a Media item',
+                'unique_together': {('media', 'site', 'key')},
+            },
+        ),
+        migrations.CreateModel(
+            name='MetadataFormat',
+            fields=[
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the format', primary_key=True, serialize=False, verbose_name='uuid')),
+                ('site', models.CharField(blank=True, default='Youtube', help_text='Site from which the format is available', max_length=256, verbose_name='site')),
+                ('key', models.CharField(blank=True, default='', help_text='Media identifier at the site for which this format is available', max_length=256, verbose_name='key')),
+                ('number', models.PositiveIntegerField(help_text='Ordering number for this format', verbose_name='number')),
+                ('code', models.CharField(blank=True, default='', help_text='Format identification code', max_length=64, verbose_name='code')),
+                ('value', models.JSONField(default=dict, encoder=sync.models.JSONEncoder, help_text='JSON metadata format object', verbose_name='value')),
+                ('metadata', models.ForeignKey(help_text='Metadata the format belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='metadataformat_metadata', to='sync.metadata')),
+            ],
+            options={
+                'verbose_name': 'Format from the Metadata about a Media item',
+                'verbose_name_plural': 'Formats from the Metadata about a Media item',
+                'unique_together': {('metadata', 'site', 'key', 'code'), ('metadata', 'site', 'key', 'number')},
+            },
+        ),
+    ]

tubesync/sync/models.py

@@ -11,7 +11,9 @@ from django.conf import settings
 from django.db import models
 from django.core.exceptions import SuspiciousOperation
 from django.core.files.storage import FileSystemStorage
+from django.core.serializers.json import DjangoJSONEncoder
 from django.core.validators import RegexValidator
+from django.db.transaction import atomic
 from django.utils.text import slugify
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
@@ -35,6 +37,20 @@ from .choices import (Val, CapChoices, Fallback, FileExtension,
 media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/')
 _srctype_dict = lambda n: dict(zip( YouTube_SourceType.values, (n,) * len(YouTube_SourceType.values) ))
 
+class JSONEncoder(DjangoJSONEncoder):
+    item_separator = ','
+    key_separator = ':'
+
+    def default(self, obj):
+        try:
+            iterable = iter(obj)
+        except TypeError:
+            pass
+        else:
+            return list(iterable)
+        return super().default(obj)
+
+
 class Source(models.Model):
     '''
         A Source is a source of media. Currently, this is either a YouTube channel
@@ -833,11 +849,14 @@ class Media(models.Model):
         fields = self.METADATA_FIELDS.get(field, {})
         return fields.get(self.source.source_type, field)
 
-    def get_metadata_first_value(self, iterable, default=None, /):
+    def get_metadata_first_value(self, iterable, default=None, /, *, arg_dict=None):
         '''
             fetch the first key with a value from metadata
         '''
         
+        if arg_dict is None:
+            arg_dict = self.loaded_metadata
+        assert isinstance(arg_dict, dict), type(arg_dict)
         # str is an iterable of characters
         # we do not want to look for each character!
         if isinstance(iterable, str):
@@ -845,7 +864,7 @@ class Media(models.Model):
         for key in tuple(iterable):
             # reminder: unmapped fields return the key itself
             field = self.get_metadata_field(key)
-            value = self.loaded_metadata.get(field)
+            value = arg_dict.get(field)
             # value can be None because:
             #   - None was stored at the key
             #   - the key was not in the dictionary
@@ -1079,6 +1098,24 @@ class Media(models.Model):
         return self.metadata is not None
 
 
+    @atomic(durable=False)
+    def metadata_load(self, arg_str='{}'):
+        data = json.loads(arg_str) or self.loaded_metadata
+        site = self.get_metadata_first_value('extractor_key', arg_dict=data)
+        epoch = self.get_metadata_first_value('epoch', arg_dict=data)
+        epoch_dt = self.metadata_published( epoch )
+        release = self.get_metadata_first_value(('release_timestamp', 'timestamp',), arg_dict=data)
+        release_dt = self.metadata_published( release )
+        md = self.metadata_media.get_or_create(site=site, key=self.key)[0]
+        md.value = data
+        formats = md.value.pop(self.get_metadata_field('formats'), list())
+        md.retrieved = epoch_dt
+        md.uploaded = self.published
+        md.published = release_dt or self.published
+        md.save()
+        md.ingest_formats(formats)
+
+
     def save_to_metadata(self, key, value, /):
         data = self.loaded_metadata
         data[key] = value
@@ -1681,6 +1718,152 @@ class Media(models.Model):
                         pass
 
 
+class Metadata(models.Model):
+    '''
+        Metadata for an indexed `Media` item.
+    '''
+    class Meta:
+        verbose_name = _('Metadata about a Media item')
+        verbose_name_plural = _('Metadata about a Media item')
+        unique_together = (
+            ('media', 'site', 'key'),
+        )
+
+    uuid = models.UUIDField(
+        _('uuid'),
+        primary_key=True,
+        editable=False,
+        default=uuid.uuid4,
+        help_text=_('UUID of the metadata'),
+    )
+    media = models.ForeignKey(
+        Media,
+        # on_delete=models.DO_NOTHING,
+        on_delete=models.CASCADE,
+        related_name='metadata_media',
+        help_text=_('Media the metadata belongs to'),
+        null=False,
+    )
+    site = models.CharField(
+        _('site'),
+        max_length=256,
+        blank=True,
+        null=False,
+        default='Youtube',
+        help_text=_('Site from which the metadata was retrieved'),
+    )
+    key = models.CharField(
+        _('key'),
+        max_length=256,
+        blank=True,
+        null=False,
+        default='',
+        help_text=_('Media identifier at the site from which the metadata was retrieved'),
+    )
+    created = models.DateTimeField(
+        _('created'),
+        auto_now_add=True,
+        db_index=True,
+        help_text=_('Date and time the metadata was created'),
+    )
+    retrieved = models.DateTimeField(
+        _('retrieved'),
+        auto_now_add=True,
+        db_index=True,
+        help_text=_('Date and time the metadata was retrieved'),
+    )
+    uploaded = models.DateTimeField(
+        _('uploaded'),
+        null=True,
+        help_text=_('Date and time the media was uploaded'),
+    )
+    published = models.DateTimeField(
+        _('published'),
+        null=True,
+        help_text=_('Date and time the media was published'),
+    )
+    value = models.JSONField(
+        _('value'),
+        encoder=JSONEncoder,
+        null=False,
+        default=dict,
+        help_text=_('JSON metadata object'),
+    )
+
+    @atomic(durable=False)
+    def ingest_formats(self, formats=list(), /):
+        for number, format in enumerate(formats, start=1):
+            mdf = self.metadataformat_metadata.get_or_create(site=self.site, key=self.key, code=format.get('format_id'), number=number)[0]
+            mdf.value = format
+            mdf.save()
+
+
+class MetadataFormat(models.Model):
+    '''
+        A format from the Metadata for an indexed `Media` item.
+    '''
+    class Meta:
+        verbose_name = _('Format from the Metadata about a Media item')
+        verbose_name_plural = _('Formats from the Metadata about a Media item')
+        unique_together = (
+            ('metadata', 'site', 'key', 'number'),
+            ('metadata', 'site', 'key', 'code'),
+        )
+
+    uuid = models.UUIDField(
+        _('uuid'),
+        primary_key=True,
+        editable=False,
+        default=uuid.uuid4,
+        help_text=_('UUID of the format'),
+    )
+    metadata = models.ForeignKey(
+        Metadata,
+        # on_delete=models.DO_NOTHING,
+        on_delete=models.CASCADE,
+        related_name='metadataformat_metadata',
+        help_text=_('Metadata the format belongs to'),
+        null=False,
+    )
+    site = models.CharField(
+        _('site'),
+        max_length=256,
+        blank=True,
+        null=False,
+        default='Youtube',
+        help_text=_('Site from which the format is available'),
+    )
+    key = models.CharField(
+        _('key'),
+        max_length=256,
+        blank=True,
+        null=False,
+        default='',
+        help_text=_('Media identifier at the site for which this format is available'),
+    )
+    number = models.PositiveIntegerField(
+        _('number'),
+        blank=False,
+        null=False,
+        help_text=_('Ordering number for this format')
+    )
+    code = models.CharField(
+        _('code'),
+        max_length=64,
+        blank=True,
+        null=False,
+        default='',
+        help_text=_('Format identification code'),
+    )
+    value = models.JSONField(
+        _('value'),
+        encoder=JSONEncoder,
+        null=False,
+        default=dict,
+        help_text=_('JSON metadata format object'),
+    )
+
+
 class MediaServer(models.Model):
     '''
         A remote media server, such as a Plex server.

tubesync/sync/signals.py

@@ -1,8 +1,9 @@
+from functools import partial
 from pathlib import Path
-from shutil import rmtree
 from tempfile import TemporaryDirectory
 from django.conf import settings
 from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
+from django.db.transaction import on_commit
 from django.dispatch import receiver
 from django.utils.translation import gettext_lazy as _
 from background_task.signals import task_failed
@@ -20,6 +21,20 @@ from .filtering import filter_media
 from .choices import Val, YouTube_SourceType
 
 
+def is_relative_to(self, *other):
+        """Return True if the path is relative to another path or False.
+        """
+        try:
+            self.relative_to(*other)
+            return True
+        except ValueError:
+            return False
+
+# patch Path for Python 3.8
+if not hasattr(Path, 'is_relative_to'):
+    Path.is_relative_to = is_relative_to
+
+
 @receiver(pre_save, sender=Source)
 def source_pre_save(sender, instance, **kwargs):
     # Triggered before a source is saved, if the schedule has been updated recreate
@@ -134,6 +149,7 @@ def source_post_save(sender, instance, created, **kwargs):
 def source_pre_delete(sender, instance, **kwargs):
     # Triggered before a source is deleted, delete all media objects to trigger
     # the Media models post_delete signal
+    source = instance
     log.info(f'Deactivating source: {instance.name}')
     instance.deactivate()
     log.info(f'Deleting tasks for source: {instance.name}')
@@ -141,20 +157,22 @@ def source_pre_delete(sender, instance, **kwargs):
     delete_task_by_source('sync.tasks.check_source_directory_exists', instance.pk)
     delete_task_by_source('sync.tasks.rename_all_media_for_source', instance.pk)
     delete_task_by_source('sync.tasks.save_all_media_for_source', instance.pk)
-    # Schedule deletion of media
-    delete_task_by_source('sync.tasks.delete_all_media_for_source', instance.pk)
-    verbose_name = _('Deleting all media for source "{}"')
-    delete_all_media_for_source(
-        str(instance.pk),
-        str(instance.name),
-        verbose_name=verbose_name.format(instance.name),
-    )
-    # Try to do it all immediately
-    # If this is killed, the scheduled task should do the work instead.
-    delete_all_media_for_source.now(
-        str(instance.pk),
-        str(instance.name),
-    )
+
+    # Fetch the media source
+    sqs = Source.objects.filter(filter_text=str(source.pk))
+    if sqs.count():
+        media_source = sqs[0]
+        # Schedule deletion of media
+        delete_task_by_source('sync.tasks.delete_all_media_for_source', media_source.pk)
+        verbose_name = _('Deleting all media for source "{}"')
+        on_commit(partial(
+            delete_all_media_for_source,
+            str(media_source.pk),
+            str(media_source.name),
+            str(media_source.directory_path),
+            priority=1,
+            verbose_name=verbose_name.format(media_source.name),
+        ))
 
 
 @receiver(post_delete, sender=Source)
@@ -164,14 +182,8 @@ def source_post_delete(sender, instance, **kwargs):
     log.info(f'Deleting tasks for removed source: {source.name}')
     delete_task_by_source('sync.tasks.index_source_task', instance.pk)
     delete_task_by_source('sync.tasks.check_source_directory_exists', instance.pk)
-    delete_task_by_source('sync.tasks.delete_all_media_for_source', instance.pk)
     delete_task_by_source('sync.tasks.rename_all_media_for_source', instance.pk)
     delete_task_by_source('sync.tasks.save_all_media_for_source', instance.pk)
-    # Remove the directory, if the user requested that
-    directory_path = Path(source.directory_path)
-    if (directory_path / '.to_be_removed').is_file():
-        log.info(f'Deleting directory for: {source.name}: {directory_path}')
-        rmtree(directory_path, True)
 
 
 @receiver(task_failed, sender=Task)
@@ -212,10 +224,10 @@ def media_post_save(sender, instance, created, **kwargs):
                     if not instance.can_download:
                         instance.can_download = True
                         can_download_changed = True
-                    else:
-                        if instance.can_download:
-                            instance.can_download = False
-                            can_download_changed = True
+                else:
+                    if instance.can_download:
+                        instance.can_download = False
+                        can_download_changed = True
             # Recalculate the "skip_changed" flag
             skip_changed = filter_media(instance)
     else:
@@ -250,8 +262,10 @@ def media_post_save(sender, instance, created, **kwargs):
     if not instance.thumb and not instance.skip:
         thumbnail_url = instance.thumbnail
         if thumbnail_url:
-            log.info(f'Scheduling task to download thumbnail for: {instance.name} '
-                     f'from: {thumbnail_url}')
+            log.info(
+                'Scheduling task to download thumbnail'
+                f' for: {instance.name} from: {thumbnail_url}'
+            )
             verbose_name = _('Downloading thumbnail for "{}"')
             download_media_thumbnail(
                 str(instance.pk),
@@ -289,8 +303,10 @@ def media_pre_delete(sender, instance, **kwargs):
     delete_task_by_media('sync.tasks.wait_for_media_premiere', (str(instance.pk),))
     thumbnail_url = instance.thumbnail
     if thumbnail_url:
-        delete_task_by_media('sync.tasks.download_media_thumbnail',
-                             (str(instance.pk), thumbnail_url))
+        delete_task_by_media(
+            'sync.tasks.download_media_thumbnail',
+            (str(instance.pk), thumbnail_url,),
+        )
     # Remove thumbnail file for deleted media
     if instance.thumb:
         instance.thumb.delete(save=False)

tubesync/sync/tasks.py

@@ -10,8 +10,9 @@ import math
 import uuid
 from io import BytesIO
 from hashlib import sha1
+from pathlib import Path
 from datetime import datetime, timedelta
-from shutil import copyfile
+from shutil import copyfile, rmtree
 from PIL import Image
 from django.conf import settings
 from django.core.files.base import ContentFile
@@ -24,12 +25,13 @@ from background_task import background
 from background_task.exceptions import InvalidTaskError
 from background_task.models import Task, CompletedTask
 from common.logger import log
-from common.errors import NoMediaException, NoMetadataException, DownloadFailedException
+from common.errors import ( NoFormatException, NoMediaException,
+                            NoMetadataException, DownloadFailedException, )
 from common.utils import json_serial, remove_enclosed
 from .choices import Val, TaskQueue
 from .models import Source, Media, MediaServer
-from .utils import (get_remote_image, resize_image_to_height, delete_file,
-                    write_text_file, filter_response)
+from .utils import ( get_remote_image, resize_image_to_height, delete_file,
+                    write_text_file, filter_response, )
 from .youtube import YouTubeError
 
 
@@ -573,9 +575,36 @@ def download_media(media_id):
                      f'not downloading')
             return
     filepath = media.filepath
+    container = format_str = None
     log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"')
-    format_str, container = media.download_media()
-    if os.path.exists(filepath):
+    try:
+        format_str, container = media.download_media()
+    except NoFormatException as e:
+        # Try refreshing formats
+        if media.has_metadata:
+            log.debug(f'Scheduling a task to refresh metadata for: {media.key}: "{media.name}"')
+            refresh_formats(
+                str(media.pk),
+                verbose_name=f'Refreshing metadata formats for: {media.key}: "{media.name}"',
+            )
+        log.exception(str(e))
+        raise
+    else:
+        if not os.path.exists(filepath):
+            # Try refreshing formats
+            if media.has_metadata:
+                log.debug(f'Scheduling a task to refresh metadata for: {media.key}: "{media.name}"')
+                refresh_formats(
+                    str(media.pk),
+                    verbose_name=f'Refreshing metadata formats for: {media.key}: "{media.name}"',
+                )
+            # Expected file doesn't exist on disk
+            err = (f'Failed to download media: {media} (UUID: {media.pk}) to disk, '
+                   f'expected outfile does not exist: {filepath}')
+            log.error(err)
+            # Raising an error here triggers the task to be re-attempted (or fail)
+            raise DownloadFailedException(err)
+
         # Media has been downloaded successfully
         log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: '
                  f'"{filepath}"')
@@ -637,19 +666,6 @@ def download_media(media_id):
                 pass
         # Schedule a task to update media servers
         schedule_media_servers_update()
-    else:
-        # Expected file doesn't exist on disk
-        err = (f'Failed to download media: {media} (UUID: {media.pk}) to disk, '
-               f'expected outfile does not exist: {filepath}')
-        log.error(err)
-        # Try refreshing formats
-        if media.has_metadata:
-            refresh_formats(
-                str(media.pk),
-                verbose_name=f'Refreshing metadata formats for: {media.key}: "{media.name}"',
-            )
-        # Raising an error here triggers the task to be re-attempted (or fail)
-        raise DownloadFailedException(err)
 
 
 @background(schedule=dict(priority=0, run_at=30), queue=Val(TaskQueue.NET), remove_existing_tasks=True)
@@ -819,8 +835,9 @@ def wait_for_media_premiere(media_id):
         if task:
             update_task_status(task, f'available in {hours(media.published - now)} hours')
 
-@background(schedule=dict(priority=1, run_at=300), queue=Val(TaskQueue.FS), remove_existing_tasks=False)
-def delete_all_media_for_source(source_id, source_name):
+
+@background(schedule=dict(priority=1, run_at=90), queue=Val(TaskQueue.FS), remove_existing_tasks=False)
+def delete_all_media_for_source(source_id, source_name, source_directory):
     source = None
     try:
         source = Source.objects.get(pk=source_id)
@@ -834,8 +851,21 @@ def delete_all_media_for_source(source_id, source_name):
     ).filter(
         source=source or source_id,
     )
-    for media in mqs:
-        log.info(f'Deleting media for source: {source_name} item: {media.name}')
-        with atomic():
-            media.delete()
+    with atomic(durable=True):
+        for media in mqs:
+            log.info(f'Deleting media for source: {source_name} item: {media.name}')
+            with atomic():
+                media.delete()
+    # Remove the directory, if the user requested that
+    directory_path = Path(source_directory)
+    remove = (
+        (source and source.delete_removed_media) or
+        (directory_path / '.to_be_removed').is_file()
+    )
+    if source:
+        with atomic(durable=True):
+            source.delete()
+    if remove:
+        log.info(f'Deleting directory for: {source_name}: {directory_path}')
+        rmtree(directory_path, True)
 

tubesync/sync/views.py

@@ -26,12 +26,12 @@ from .models import Source, Media, MediaServer
 from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm,
                     SkipMediaForm, EnableMediaForm, ResetTasksForm,
                     ConfirmDeleteMediaServerForm)
-from .utils import validate_url, delete_file, multi_key_sort
+from .utils import validate_url, delete_file, multi_key_sort, mkdir_p
 from .tasks import (map_task_to_instance, get_error_message,
                     get_source_completed_tasks, get_media_download_task,
                     delete_task_by_media, index_source_task,
                     check_source_directory_exists, migrate_queues)
-from .choices import (Val, MediaServerType, SourceResolution,
+from .choices import (Val, MediaServerType, SourceResolution, IndexSchedule,
                         YouTube_SourceType, youtube_long_source_types,
                         youtube_help, youtube_validation_urls)
 from . import signals
@@ -411,11 +411,39 @@ class DeleteSourceView(DeleteView, FormMixin):
     context_object_name = 'source'
 
     def post(self, request, *args, **kwargs):
+        source = self.get_object()
+        media_source = dict(
+            uuid=None,
+            index_schedule=IndexSchedule.NEVER,
+            download_media=False,
+            index_videos=False,
+            index_streams=False,
+            filter_text=str(source.pk),
+        )
+        copy_fields = set(map(lambda f: f.name, source._meta.fields)) - set(media_source.keys())
+        for k, v in source.__dict__.items():
+            if k in copy_fields:
+                media_source[k] = v
+        media_source = Source(**media_source)
         delete_media_val = request.POST.get('delete_media', False)
         delete_media = True if delete_media_val is not False else False
+        # overload this boolean for our own use
+        media_source.delete_removed_media = delete_media
+        # adjust the directory and key on the source to be deleted
+        source.directory = source.directory + '/deleted'
+        source.key = source.key + '/deleted'
+        source.name = f'[Deleting] {source.name}'
+        source.save(update_fields={'directory', 'key', 'name'})
+        source.refresh_from_db()
+        # save the new media source now that it is not a duplicate
+        media_source.uuid = None
+        media_source.save()
+        media_source.refresh_from_db()
+        # switch the media to the new source instance
+        Media.objects.filter(source=source).update(source=media_source)
         if delete_media:
-            source = self.get_object()
-            directory_path = pathlib.Path(source.directory_path)
+            directory_path = pathlib.Path(media_source.directory_path)
+            mkdir_p(directory_path)
             (directory_path / '.to_be_removed').touch(exist_ok=True)
         return super().post(request, *args, **kwargs)