diff --git a/README.md b/README.md index 6c058fb..ae88b55 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Feature overview: - Automatically delete stalled downloads, after they have been found to be stalled multiple times in a row (& trigger download from another source) - Automatically delete slow downloads, after they have been found to be slow multiple times in a row (& trigger download from another source) - Automatically delete downloads belonging to radarr/sonarr/etc. items that are unmonitored +- Automatically delete downloads that failed importing since they are not a format upgrade (i.e. a better version is already present) You may run this locally by launching main.py, or by pulling the docker image. You can find a sample docker-compose.yml in the docker folder. @@ -63,6 +64,7 @@ services: - REMOVE_FAILED=True - REMOVE_METADATA_MISSING=True - REMOVE_MISSING_FILES=True + - REMOVE_NO_FORMAT_UPGRADE=True - REMOVE_ORPHANS=True - REMOVE_SLOW=True - REMOVE_STALLED=True @@ -146,13 +148,6 @@ Steers which type of cleaning is applied to the downloads queue - Permissible Values: True, False - Is Mandatory: No (Defaults to False) -**REMOVE_STALLED** -- Steers whether stalled downloads with no connections are removed from the queue -- These downloads are added to the blocklist, so that they are not re-requested in the future -- Type: Boolean -- Permissible Values: True, False -- Is Mandatory: No (Defaults to False) - **REMOVE_METADATA_MISSING** - Steers whether downloads stuck obtaining metadata are removed from the queue - These downloads are added to the blocklist, so that they are not re-requested @@ -161,6 +156,21 @@ Steers which type of cleaning is applied to the downloads queue - Permissible Values: True, False - Is Mandatory: No (Defaults to False) +**REMOVE_MISSING_FILES** +- Steers whether downloads that have the warning "Files Missing" are removed from the queue +- These downloads are not added to the blocklist +- Type: Boolean +- Permissible Values: True, False +- Is Mandatory: No (Defaults to False) + +**REMOVE_NO_FORMAT_UPGRADE** +- Steers whether downloads that failed importing since they are not a format upgrade are removed from the queue +- This occurs when a better version is already present +- These downloads are added to the blocklist +- Type: Boolean +- Permissible Values: True, False +- Is Mandatory: No (Defaults to False) + **REMOVE_ORPHANS** - Steers whether orphan downloads are removed from the queue - Orphan downloads are those that do not belong to any requested media anymore (Since the media was removed from radarr/sonarr/lidarr/readarr after the download started) @@ -169,6 +179,20 @@ Steers which type of cleaning is applied to the downloads queue - Permissible Values: True, False - Is Mandatory: No (Defaults to False) +**REMOVE_SLOW** +- Steers whether slow downloads are removed from the queue +- Slow downloads are added to the blocklist, so that they are not re-requested in the future +- Type: Boolean +- Permissible Values: True, False +- Is Mandatory: No (Defaults to False) + +**REMOVE_STALLED** +- Steers whether stalled downloads with no connections are removed from the queue +- These downloads are added to the blocklist, so that they are not re-requested in the future +- Type: Boolean +- Permissible Values: True, False +- Is Mandatory: No (Defaults to False) + **REMOVE_UNMONITORED** - Steers whether downloads belonging to unmonitored media are removed from the queue - Note: Will only remove from queue if all TV shows depending on the same download are unmonitored @@ -178,20 +202,6 @@ Steers which type of cleaning is applied to the downloads queue - Permissible Values: True, False - Is Mandatory: No (Defaults to False) -**REMOVE_MISSING_FILES** -- Steers whether downloads that have the warning "Files Missing" are removed from the queue -- These downloads are not added to the blocklist -- Type: Boolean -- Permissible Values: True, False -- Is Mandatory: No (Defaults to False) - -**REMOVE_SLOW** -- Steers whether slow downloads are removed from the queue -- Slow downloads are added to the blocklist, so that they are not re-requested in the future -- Type: Boolean -- Permissible Values: True, False -- Is Mandatory: No (Defaults to False) - **MIN_DOWNLOAD_SPEED** - Sets the minimum download speed for active downloads - If the increase in the downloaded file size of a download is less than this value between two consecutive checks, the download is considered slow and is removed if happening more ofthen than the permitted attempts diff --git a/config/config.conf-Example b/config/config.conf-Example index 7b7fca8..84ef35d 100644 --- a/config/config.conf-Example +++ b/config/config.conf-Example @@ -7,6 +7,7 @@ REMOVE_TIMER = 10 REMOVE_FAILED = True REMOVE_METADATA_MISSING = True REMOVE_MISSING_FILES = True +REMOVE_NO_FORMAT_UPGRADE = True REMOVE_ORPHANS = True REMOVE_SLOW = True REMOVE_STALLED = True diff --git a/config/config.py b/config/config.py index 80f36f1..550322e 100644 --- a/config/config.py +++ b/config/config.py @@ -85,12 +85,13 @@ SSL_VERIFICATION = get_config_value('SSL_VERIFICATION', # Features REMOVE_TIMER = get_config_value('REMOVE_TIMER', 'features', False, float, 10) REMOVE_FAILED = get_config_value('REMOVE_FAILED', 'features', False, bool, False) -REMOVE_STALLED = get_config_value('REMOVE_STALLED', 'features', False, bool, False) REMOVE_METADATA_MISSING = get_config_value('REMOVE_METADATA_MISSING', 'features', False, bool, False) -REMOVE_ORPHANS = get_config_value('REMOVE_ORPHANS' , 'features', False, bool, False) -REMOVE_UNMONITORED = get_config_value('REMOVE_UNMONITORED' , 'features', False, bool, False) REMOVE_MISSING_FILES = get_config_value('REMOVE_MISSING_FILES' , 'features', False, bool, False) +REMOVE_NO_FORMAT_UPGRADE = get_config_value('REMOVE_NO_FORMAT_UPGRADE' , 'features', False, bool, False) +REMOVE_ORPHANS = get_config_value('REMOVE_ORPHANS' , 'features', False, bool, False) REMOVE_SLOW = get_config_value('REMOVE_SLOW' , 'features', False, bool, False) +REMOVE_STALLED = get_config_value('REMOVE_STALLED', 'features', False, bool, False) +REMOVE_UNMONITORED = get_config_value('REMOVE_UNMONITORED' , 'features', False, bool, False) MIN_DOWNLOAD_SPEED = get_config_value('MIN_DOWNLOAD_SPEED', 'features', False, int, 0) PERMITTED_ATTEMPTS = get_config_value('PERMITTED_ATTEMPTS', 'features', False, int, 3) NO_STALLED_REMOVAL_QBIT_TAG = get_config_value('NO_STALLED_REMOVAL_QBIT_TAG', 'features', False, str, 'Don\'t Kill') diff --git a/main.py b/main.py index e8fcc1f..af9f849 100644 --- a/main.py +++ b/main.py @@ -29,6 +29,20 @@ class Download_Sizes_Tracker: # Main function async def main(settingsDict): + # Adds to settings Dict the instances that are actually configures + arrApplications = ['RADARR', 'SONARR', 'LIDARR', 'READARR'] + settingsDict['INSTANCES'] = [] + for arrApplication in arrApplications: + if settingsDict[arrApplication + '_URL']: + settingsDict['INSTANCES'].append(arrApplication) + + # Pre-populates the dictionaries (in classes) that track the items that were already caught as having problems or removed + defectiveTrackingInstances = {} + for instance in settingsDict['INSTANCES']: + defectiveTrackingInstances[instance] = {} + defective_tracker = Defective_Tracker(defectiveTrackingInstances) + download_sizes_tracker = Download_Sizes_Tracker({}) + # Get name of arr-instances for instance in settingsDict['INSTANCES']: settingsDict = await getArrInstanceName(settingsDict, instance) @@ -75,16 +89,6 @@ async def main(settingsDict): return if __name__ == '__main__': - arrApplications = ['RADARR', 'SONARR', 'LIDARR', 'READARR'] - defectiveTrackingInstances = {} - settingsDict['INSTANCES'] = [] - for arrApplication in arrApplications: - if settingsDict[arrApplication + '_URL']: - settingsDict['INSTANCES'].append(arrApplication) - defectiveTrackingInstances[arrApplication] = {} - - defective_tracker = Defective_Tracker(defectiveTrackingInstances) - download_sizes_tracker = Download_Sizes_Tracker({}) asyncio.run(main(settingsDict)) diff --git a/src/decluttarr.py b/src/decluttarr.py index 434fb06..771d1a6 100644 --- a/src/decluttarr.py +++ b/src/decluttarr.py @@ -5,6 +5,7 @@ from src.utils.shared import (errorDetails, get_queue) from src.jobs.remove_failed import remove_failed from src.jobs.remove_metadata_missing import remove_metadata_missing from src.jobs.remove_missing_files import remove_missing_files +from src.jobs.remove_no_format_upgrade import remove_no_format_upgrade from src.jobs.remove_orphans import remove_orphans from src.jobs.remove_slow import remove_slow from src.jobs.remove_stalled import remove_stalled @@ -50,31 +51,37 @@ async def queueCleaner(settingsDict, arr_type, defective_tracker, download_sizes if not full_queue: logger.verbose('>>> Queue is empty.') return + else: + logger.debug('queueCleaner/full_queue at start:') + logger.debug(full_queue) deleted_downloads = Deleted_Downloads([]) items_detected = 0 try: if settingsDict['REMOVE_FAILED']: items_detected += await remove_failed( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs) - - if settingsDict['REMOVE_STALLED']: - items_detected += await remove_stalled( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs) if settingsDict['REMOVE_METADATA_MISSING']: items_detected += await remove_metadata_missing( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs) - if settingsDict['REMOVE_ORPHANS']: - items_detected += await remove_orphans( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, full_queue_param) - - if settingsDict['REMOVE_UNMONITORED']: - items_detected += await remove_unmonitored( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, arr_type) - if settingsDict['REMOVE_MISSING_FILES']: items_detected += await remove_missing_files( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs) + if settingsDict['REMOVE_NO_FORMAT_UPGRADE']: + items_detected += await remove_no_format_upgrade( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs) + + if settingsDict['REMOVE_ORPHANS']: + items_detected += await remove_orphans( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, full_queue_param) + if settingsDict['REMOVE_SLOW']: items_detected += await remove_slow( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, download_sizes_tracker) + if settingsDict['REMOVE_STALLED']: + items_detected += await remove_stalled( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs) + + if settingsDict['REMOVE_UNMONITORED']: + items_detected += await remove_unmonitored( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, arr_type) + if items_detected == 0: logger.verbose('>>> Queue is clean.') except Exception as error: diff --git a/src/jobs/remove_no_format_upgrade.py b/src/jobs/remove_no_format_upgrade.py new file mode 100644 index 0000000..cbf831f --- /dev/null +++ b/src/jobs/remove_no_format_upgrade.py @@ -0,0 +1,38 @@ +from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download) +import sys, os, traceback +import logging, verboselogs +logger = verboselogs.VerboseLogger(__name__) + +async def remove_no_format_upgrade(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs): + # Detects downloads stuck downloading meta data and triggers repeat check and subsequent delete. Adds to blocklist + try: + failType = 'no format upgrade' + queue = await get_queue(BASE_URL, API_KEY) + logger.debug('remove_no_format_upgrade/queue IN: %s', formattedQueueInfo(queue)) + if not queue: return 0 + # Find items affected + affectedItems = [] + for queueItem in queue['records']: + if 'status' in queueItem \ + and 'trackedDownloadStatus' in queueItem \ + and 'trackedDownloadState' in queueItem \ + and 'statusMessages' in queueItem: + + if queueItem['status'] == 'completed' \ + and queueItem['trackedDownloadStatus'] == 'warning' \ + and queueItem['trackedDownloadState'] == 'importPending': + + for status_message in queueItem['statusMessages']: + if any(message.startswith("Not a Custom Format upgrade for existing") for message in status_message.get('messages', [])): + affectedItems.append(queueItem) + break + + affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, + addToBlocklist = True, + doPrivateTrackerCheck = True, + doProtectedDownloadCheck = True, + doPermittedAttemptsCheck = True) + return len(affectedItems) + except Exception as error: + errorDetails(NAME, error) + return 0 diff --git a/src/utils/loadScripts.py b/src/utils/loadScripts.py index 164cc8b..18de686 100644 --- a/src/utils/loadScripts.py +++ b/src/utils/loadScripts.py @@ -41,6 +41,7 @@ def showSettings(settingsDict): logger.info('%s | Removing failed downloads', str(settingsDict['REMOVE_FAILED'])) logger.info('%s | Removing downloads missing metadata', str(settingsDict['REMOVE_METADATA_MISSING'])) logger.info('%s | Removing downloads missing files', str(settingsDict['REMOVE_MISSING_FILES'])) + logger.info('%s | Removing downloads that fail on import (no format upgrade)', str(settingsDict['REMOVE_NO_FORMAT_UPGRADE'])) logger.info('%s | Removing orphan downloads', str(settingsDict['REMOVE_ORPHANS'])) logger.info('%s | Removing slow downloads', str(settingsDict['REMOVE_SLOW'])) logger.info('%s | Removing stalled downloads', str(settingsDict['REMOVE_STALLED']))