diff --git a/README.md b/README.md index 8ca7e21..0052985 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,10 @@ services: - REMOVE_METADATA_MISSING=True - REMOVE_ORPHANS=True - REMOVE_UNMONITORED=True + - REMOVE_SLOW=True + - MIN_DOWNLOAD_SPEED=100 - PERMITTED_ATTEMPTS=3 - - NO_STALLED_REMOVAL_QBIT_TAG=Don't Kill If Stalled + - NO_STALLED_REMOVAL_QBIT_TAG=Don't Kill # Radarr - RADARR_URL=http://radarr:7878 - RADARR_KEY=$RADARR_API_KEY @@ -135,18 +137,34 @@ Note: The `config.conf` is disregarded when running via docker-compose.yml - Permissible Values: True, False - Is Mandatory: No (Defaults to False) +**REMOVE_SLOW** +- Steers whether slow downloads are removed from the queue +- Slow downloads are added to the blocklist, so that they are not re-requested in the future +- A new download from another source is automatically added by sonarr/radarr (if available) +- Type: Boolean +- Permissible Values: True, False +- Is Mandatory: No (Defaults to False) + +**MIN_DOWNLOAD_SPEED** +- Sets the minimum download speed for active downloads +- If the increase in the downloaded file size of a download is less than this value between two consecutive checks, the download is considered slow and is removed if happening more ofthen than the permitted attempts. +- Type: Integer +- Unit: KBytes per second +- Is Mandatory: No (Defaults to 100, but is only enforced when "REMOVE_SLOW" is true) + **PERMITTED_ATTEMPTS** -- Defines how many times a download has to be caught as stalled or stuck downloading metadata before it is removed +- Defines how many times a download has to be caught as stalled, slow or stuck downloading metadata before it is removed - Type: Integer - Unit: Number of scans - Is Mandatory: No (Defaults to 3) **NO_STALLED_REMOVAL_QBIT_TAG** - Downloads in qBittorrent tagged with this tag will not be killed even if they are stalled +- Also protects slow downloads or those stuck downloading meta data - Tag is automatically created in qBittorrent (required qBittorrent is reachable on `QBITTORRENT_URL`) - Also protects unmonitored downloads from being removed (relevant for multi-season packs) - Type: String -- Is Mandatory: No (Defaults to `Don't Kill If Stalled`) +- Is Mandatory: No (Defaults to `Don't Kill`) --- diff --git a/config/config.conf-Example b/config/config.conf-Example index 0942e11..b49e744 100644 --- a/config/config.conf-Example +++ b/config/config.conf-Example @@ -9,8 +9,10 @@ REMOVE_STALLED = True REMOVE_METADATA_MISSING = True REMOVE_ORPHANS = True REMOVE_UNMONITORED = True +REMOVE_SLOW = True +MIN_DOWNLOAD_SPEED = 100 PERMITTED_ATTEMPTS = 3 -NO_STALLED_REMOVAL_QBIT_TAG = Don't Kill If Stalled +NO_STALLED_REMOVAL_QBIT_TAG = Don't Kill [radarr] RADARR_URL = http://radarr:7878 diff --git a/config/config.py b/config/config.py index 01feeba..26c0daa 100644 --- a/config/config.py +++ b/config/config.py @@ -86,8 +86,10 @@ REMOVE_STALLED = get_config_value('REMOVE_STALLED', REMOVE_METADATA_MISSING = get_config_value('REMOVE_METADATA_MISSING', 'features', False, bool, False) REMOVE_ORPHANS = get_config_value('REMOVE_ORPHANS' , 'features', False, bool, False) REMOVE_UNMONITORED = get_config_value('REMOVE_UNMONITORED' , 'features', False, bool, False) +REMOVE_SLOW = get_config_value('REMOVE_SLOW' , 'features', False, bool, False) +MIN_DOWNLOAD_SPEED = get_config_value('MIN_DOWNLOAD_SPEED', 'features', False, int, 0) PERMITTED_ATTEMPTS = get_config_value('PERMITTED_ATTEMPTS', 'features', False, int, 3) -NO_STALLED_REMOVAL_QBIT_TAG = get_config_value('NO_STALLED_REMOVAL_QBIT_TAG', 'features', False, str, 'Don\'t Kill If Stalled') +NO_STALLED_REMOVAL_QBIT_TAG = get_config_value('NO_STALLED_REMOVAL_QBIT_TAG', 'features', False, str, 'Don\'t Kill') # Radarr RADARR_URL = get_config_value('RADARR_URL', 'radarr', False, str) diff --git a/main.py b/main.py index a2cb61d..416f0f2 100644 --- a/main.py +++ b/main.py @@ -22,6 +22,10 @@ class Defective_Tracker: # Keeps track of which downloads were already caught as stalled previously def __init__(self, dict): self.dict = dict +class Download_Sizes: + # Keeps track of the file sizes of the downloads + def __init__(self, dict): + self.dict = dict # Main function async def main(): @@ -52,13 +56,15 @@ async def main(): logger.info('*** Current Settings ***') logger.info('%s | Removing failed downloads', str(settings_dict['REMOVE_FAILED'])) logger.info('%s | Removing stalled downloads', str(settings_dict['REMOVE_STALLED'])) + logger.info('%s | Removing slow downloads', str(settings_dict['REMOVE_SLOW'])) logger.info('%s | Removing downloads missing metadata', str(settings_dict['REMOVE_METADATA_MISSING'])) logger.info('%s | Removing orphan downloads', str(settings_dict['REMOVE_ORPHANS'])) logger.info('%s | Removing downloads belonging to unmonitored TV shows/movies', str(settings_dict['REMOVE_UNMONITORED'])) logger.info('') - logger.info('Running every: %s', fmt.format(rd(minutes=settings_dict['REMOVE_TIMER']))) - logger.info('Permitted number of times before stalled/missing metadata downloads are removed: %s', str(settings_dict['PERMITTED_ATTEMPTS'])) + logger.info('Running every: %s', fmt.format(rd(minutes=settings_dict['REMOVE_TIMER']))) + if settings_dict['REMOVE_SLOW']: logger.info('%s | Minimum speed enforced: ', str(settings_dict['MIN_DOWNLOAD_SPEED']) + 'KB/s') + logger.info('Permitted number of times before stalled/missing metadata/slow downloads are removed: %s', str(settings_dict['PERMITTED_ATTEMPTS'])) if settings_dict['QBITTORRENT_URL']: logger.info('Downloads with this tag will be skipped: \"%s\"', settings_dict['NO_STALLED_REMOVAL_QBIT_TAG']) logger.info('') @@ -140,9 +146,9 @@ async def main(): # Start application while True: logger.verbose('-' * 50) - if settings_dict['RADARR_URL']: await queue_cleaner(settings_dict, 'radarr', defective_tracker) - if settings_dict['SONARR_URL']: await queue_cleaner(settings_dict, 'sonarr', defective_tracker) - if settings_dict['LIDARR_URL']: await queue_cleaner(settings_dict, 'lidarr', defective_tracker) + if settings_dict['RADARR_URL']: await queue_cleaner(settings_dict, 'radarr', defective_tracker, download_sizes) + if settings_dict['SONARR_URL']: await queue_cleaner(settings_dict, 'sonarr', defective_tracker, download_sizes) + if settings_dict['LIDARR_URL']: await queue_cleaner(settings_dict, 'lidarr', defective_tracker, download_sizes) logger.verbose('') logger.verbose('Queue clean-up complete!') await asyncio.sleep(settings_dict['REMOVE_TIMER']*60) @@ -153,5 +159,6 @@ if __name__ == '__main__': {settings_dict['SONARR_URL']: {}} if settings_dict['SONARR_URL'] else {} + \ {settings_dict['LIDARR_URL']: {}} if settings_dict['LIDARR_URL'] else {} defective_tracker = Defective_Tracker(instances) + download_sizes = Download_Sizes() asyncio.run(main()) diff --git a/src/queue_cleaner.py b/src/queue_cleaner.py index 4d1c9e5..41e0258 100644 --- a/src/queue_cleaner.py +++ b/src/queue_cleaner.py @@ -6,6 +6,8 @@ import json from src.utils.nest_functions import (add_keys_nested_dict, nested_get) import sys, os import traceback + + class Deleted_Downloads: # Keeps track of which downloads have already been deleted (to not double-delete) def __init__(self, dict): @@ -141,6 +143,41 @@ async def remove_unmonitored(settings_dict, BASE_URL, API_KEY, deleted_downloads logger.debug('remove_unmonitored/queue OUT: %s', str(await get_queue(BASE_URL, API_KEY) )) return len(unmonitoredItems) +async def remove_slow(settings_dict, BASE_URL, API_KEY, deleted_downloads, defective_tracker, download_sizes): + # Detects slow downloads and triggers delete. Adds to blocklist + queue = await get_queue(BASE_URL, API_KEY) + if not queue: return 0 + logger.debug('remove_slow/queue: %s', str(queue)) + if settings_dict['QBITTORRENT_URL']: + protected_dowloadItems = await rest_get(settings_dict['QBITTORRENT_URL']+'/torrents/info',params={'tag': settings_dict['NO_STALLED_REMOVAL_QBIT_TAG']}, cookies=settings_dict['QBIT_COOKIE'] ) + protected_downloadIDs = [str.upper(item['hash']) for item in protected_dowloadItems] + else: + protected_downloadIDs = [] + slowItems = [] + already_detected = [] + + for queueItem in queue['records']: + if 'downloadId' in queueItem and 'size' in queueItem and 'sizeleft' in queueItem and 'status' in queueItem: + downloaded_size = int((queueItem['size'] - queueItem['sizeleft']) / 1000) + speed = (downloaded_size - download_sizes.dict.get(queueItem['downloadId'], 0)) / (settings_dict['REMOVE_TIMER'] * 60) + if queueItem['status'] == 'downloading' and \ + queueItem['downloadId'] in download_sizes.dict and \ + speed < settings_dict['MIN_DOWNLOAD_SPEED']: + if queueItem['downloadId'] in protected_downloadIDs: + if queueItem['downloadId'] not in already_detected: + already_detected.append(queueItem['downloadId']) + logger.verbose('>>> Detected slow download, tagged not to be killed: %s (%dKB/s)',queueItem['title'], speed) + else: + slowItems.append(queueItem) + try: + logger.verbose(f'{(downloaded_size - download_sizes.dict[queueItem["downloadId"]]) * settings_dict["REMOVE_TIMER"] / 60}, : {queueItem["title"]}') + except: pass + download_sizes.dict[queueItem['downloadId']] = downloaded_size + await check_permitted_attempts(settings_dict, slowItems, 'slow', True, deleted_downloads, BASE_URL, API_KEY, defective_tracker) + queue = await get_queue(BASE_URL, API_KEY) + logger.debug('remove_slow/queue OUT: %s', str(queue)) + return len(slowItems) + async def check_permitted_attempts(settings_dict, current_defective_items, failType, blocklist, deleted_downloads, BASE_URL, API_KEY, defective_tracker): # Checks if downloads are repeatedly found as stalled / stuck in metadata and if yes, deletes them # 1. Create list of currently defective @@ -219,7 +256,7 @@ async def queue_cleaner(settings_dict, arr_type, defective_tracker): items_detected = 0 #items_detected += await test_remove_ALL( settings_dict, BASE_URL, API_KEY, deleted_downloads, defective_tracker) - + if settings_dict['REMOVE_FAILED']: items_detected += await remove_failed( settings_dict, BASE_URL, API_KEY, deleted_downloads) @@ -235,6 +272,9 @@ async def queue_cleaner(settings_dict, arr_type, defective_tracker): if settings_dict['REMOVE_UNMONITORED']: items_detected += await remove_unmonitored( settings_dict, BASE_URL, API_KEY, deleted_downloads, arr_type) + if settings_dict['REMOVE_SLOW']: + items_detected += await remove_slow( settings_dict, BASE_URL, API_KEY, deleted_downloads, defective_tracker, download_sizes) + if items_detected == 0: logger.verbose('>>> Queue is clean.') except Exception as error: