starred/decluttarr
1
0
Fork 0
mirror of https://github.com/ManiMatter/decluttarr.git synced 2026-04-26 02:35:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #70 from ManiMatter/dev

Browse Source 
Fixing problems with dict + dict issue // adding compatibility for delay profiles // code cleanup
This commit is contained in:
ManiMatter
2024-03-30 10:17:01 +01:00
committed by GitHub
parent 14ad2d8ead 3490d4daa4
commit 9d5deef2f3
14 changed files with 315 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/dev.yml vendored
View File

@@ -73,6 +73,7 @@ jobs:
--build-arg SHORT_COMMIT_ID=$SHORT_COMMIT_ID \ --build-arg SHORT_COMMIT_ID=$SHORT_COMMIT_ID \
--push \ --push \
# "annotations": { "org.opencontainers.image.description": "DESCRIPTION" }
# - name: "Delete untagged versions" # - name: "Delete untagged versions"
# uses: actions/delete-package-versions@v4 # uses: actions/delete-package-versions@v4

4
config/config.py
View File

@@ -135,8 +135,8 @@ if READARR_URL: READARR_URL += '/api/v1'
if QBITTORRENT_URL: QBITTORRENT_URL += '/api/v2' if QBITTORRENT_URL: QBITTORRENT_URL += '/api/v2'
########### Add Variables to Dictionary ########### Add Variables to Dictionary
settings_dict = {} settingsDict = {}
for var_name in dir(): for var_name in dir():
if var_name.isupper(): if var_name.isupper():
settings_dict[var_name] = locals()[var_name] settingsDict[var_name] = locals()[var_name]

231
main.py
View File

@@ -1,30 +1,22 @@
########### Import Libraries ########### Import Libraries
import asyncio import asyncio
import logging, verboselogs import logging, verboselogs
from src.utils.rest import rest_get, rest_post logger = verboselogs.VerboseLogger(__name__)
from requests.exceptions import RequestException
import json ########### Import Functions
from dateutil.relativedelta import relativedelta as rd from config.config import settingsDict
from config.config import settings_dict from src.utils.loadScripts import *
from src.decluttarr import queueCleaner from src.decluttarr import queueCleaner
#print(json.dumps(settings_dict,indent=4)) from src.utils.rest import rest_get, rest_post
import requests
import platform
from packaging import version
# Hide SSL Verification Warnings # Hide SSL Verification Warnings
if settings_dict['SSL_VERIFICATION']==False: if settingsDict['SSL_VERIFICATION']==False:
import warnings import warnings
warnings.filterwarnings("ignore", message="Unverified HTTPS request") warnings.filterwarnings("ignore", message="Unverified HTTPS request")
########### Enabling Logging ########### Enabling Logging
# Set up logging # Set up logging
log_level_num=logging.getLevelName(settings_dict['LOG_LEVEL']) setLoggingFormat(settingsDict)
logger = verboselogs.VerboseLogger(__name__)
logging.basicConfig(
format=('' if settings_dict['IS_IN_DOCKER'] else '%(asctime)s ') + ('[%(levelname)-7s]' if settings_dict['LOG_LEVEL']=='VERBOSE' else '[%(levelname)s]') + ': %(message)s',
level=log_level_num
)
class Defective_Tracker: class Defective_Tracker:
# Keeps track of which downloads were already caught as stalled previously # Keeps track of which downloads were already caught as stalled previously
@@ -36,194 +28,63 @@ class Download_Sizes_Tracker:
self.dict = dict self.dict = dict
# Main function # Main function
async def main(): async def main(settingsDict):
# Get name of Radarr / Sonarr instances # Get name of arr-instances
try: for instance in settingsDict['INSTANCES']:
if settings_dict['RADARR_URL']: settingsDict = await getArrInstanceName(settingsDict, instance)
settings_dict['RADARR_NAME'] = (await rest_get(settings_dict['RADARR_URL']+'/system/status', settings_dict['RADARR_KEY']))['instanceName']
except:
settings_dict['RADARR_NAME'] = 'Radarr'
try:
if settings_dict['SONARR_URL']:
settings_dict['SONARR_NAME'] = (await rest_get(settings_dict['SONARR_URL']+'/system/status', settings_dict['SONARR_KEY']))['instanceName']
except:
settings_dict['SONARR_NAME'] = 'Sonarr'
try: # Display current settings when loading script
if settings_dict['LIDARR_URL']: showSettings(settingsDict)
settings_dict['LIDARR_NAME'] = (await rest_get(settings_dict['LIDARR_URL']+'/system/status', settings_dict['LIDARR_KEY']))['instanceName']
except:
settings_dict['LIDARR_NAME'] = 'Lidarr'
try: # Check Minimum Version and if instances are reachable and retrieve qbit cookie
if settings_dict['READARR_URL']: settingsDict['RADARR_MIN_VERSION'] = '5.3.6.8608'
settings_dict['READARR_NAME'] = (await rest_get(settings_dict['READARR_URL']+'/system/status', settings_dict['READARR_KEY']))['instanceName'] settingsDict['SONARR_MIN_VERSION'] = '4.0.1.1131'
except: settingsDict['LIDARR_MIN_VERSION'] = None
settings_dict['READARR_NAME'] = 'Readarr' settingsDict['READARR_MIN_VERSION'] = None
settingsDict['QBITTORRENT_MIN_VERSION'] = '4.3.0'
settingsDict = await instanceChecks(settingsDict)
# Print Settings # Create qBit protection tag if not existing
fmt = '{0.days} days {0.hours} hours {0.minutes} minutes' await createQbitProtectionTag(settingsDict)
logger.info('#' * 50)
logger.info('Decluttarr - Application Started!')
logger.info('')
logger.info('*** Current Settings ***')
logger.info('Version: %s', settings_dict['IMAGE_TAG'])
logger.info('Commit: %s', settings_dict['SHORT_COMMIT_ID'])
logger.info('')
logger.info('%s | Removing failed downloads', str(settings_dict['REMOVE_FAILED']))
logger.info('%s | Removing downloads missing metadata', str(settings_dict['REMOVE_METADATA_MISSING']))
logger.info('%s | Removing downloads missing files', str(settings_dict['REMOVE_MISSING_FILES']))
logger.info('%s | Removing orphan downloads', str(settings_dict['REMOVE_ORPHANS']))
logger.info('%s | Removing slow downloads', str(settings_dict['REMOVE_SLOW']))
logger.info('%s | Removing stalled downloads', str(settings_dict['REMOVE_STALLED']))
logger.info('%s | Removing downloads belonging to unmonitored TV shows/movies', str(settings_dict['REMOVE_UNMONITORED']))
logger.info('')
logger.info('Running every: %s', fmt.format(rd(minutes=settings_dict['REMOVE_TIMER'])))
if settings_dict['REMOVE_SLOW']:
logger.info('Minimum speed enforced: %s KB/s', str(settings_dict['MIN_DOWNLOAD_SPEED']))
logger.info('Permitted number of times before stalled/missing metadata/slow downloads are removed: %s', str(settings_dict['PERMITTED_ATTEMPTS']))
if settings_dict['QBITTORRENT_URL']:
logger.info('Downloads with this tag will be skipped: \"%s\"', settings_dict['NO_STALLED_REMOVAL_QBIT_TAG'])
logger.info('Private Trackers will be skipped: %s', settings_dict['IGNORE_PRIVATE_TRACKERS'])
logger.info('') # Show Logger settings
logger.info('*** Configured Instances ***') showLoggerSettings(settingsDict)
if settings_dict['RADARR_URL']: logger.info('%s: %s', settings_dict['RADARR_NAME'], settings_dict['RADARR_URL'])
if settings_dict['SONARR_URL']: logger.info('%s: %s', settings_dict['SONARR_NAME'], settings_dict['SONARR_URL'])
if settings_dict['LIDARR_URL']: logger.info('%s: %s', settings_dict['LIDARR_NAME'], settings_dict['LIDARR_URL'])
if settings_dict['READARR_URL']: logger.info('%s: %s', settings_dict['READARR_NAME'], settings_dict['READARR_URL'])
if settings_dict['QBITTORRENT_URL']: logger.info('qBittorrent: %s', settings_dict['QBITTORRENT_URL'])
logger.info('') # Start Cleaning
logger.info('*** Check Instances ***')
if settings_dict['RADARR_URL']:
error_occured = False
try:
await asyncio.get_event_loop().run_in_executor(None, lambda: requests.get(settings_dict['RADARR_URL']+'/system/status', params=None, headers={'X-Api-Key': settings_dict['RADARR_KEY']}, verify=settings_dict['SSL_VERIFICATION']))
except Exception as error:
error_occured = True
logger.error('-- | %s *** Error: %s ***', settings_dict['RADARR_NAME'], error)
if not error_occured:
radarr_version = (await rest_get(settings_dict['RADARR_URL']+'/system/status', settings_dict['RADARR_KEY']))['version']
if version.parse(radarr_version) < version.parse('5.3.6.8608'):
error_occured = True
logger.error('-- | %s *** Error: %s ***', settings_dict['RADARR_NAME'], 'Please update Radarr to at least version 5.3.6.8608. Current version: ' + radarr_version)
if not error_occured:
logger.info('OK | %s', settings_dict['RADARR_NAME'])
if settings_dict['SONARR_URL']:
try:
await asyncio.get_event_loop().run_in_executor(None, lambda: requests.get(settings_dict['SONARR_URL']+'/system/status', params=None, headers={'X-Api-Key': settings_dict['SONARR_KEY']}, verify=settings_dict['SSL_VERIFICATION']))
except Exception as error:
error_occured = True
logger.error('-- | %s *** Error: %s ***', settings_dict['SONARR_NAME'], error)
if not error_occured:
sonarr_version = (await rest_get(settings_dict['SONARR_URL']+'/system/status', settings_dict['SONARR_KEY']))['version']
if version.parse(sonarr_version) < version.parse('4.0.1.1131'):
error_occured = True
logger.error('-- | %s *** Error: %s ***', settings_dict['SONARR_NAME'], 'Please update Sonarr to at least version 4.0.1.1131. Current version: ' + sonarr_version)
if not error_occured:
logger.info('OK | %s', settings_dict['SONARR_NAME'])
if settings_dict['LIDARR_URL']:
try:
await asyncio.get_event_loop().run_in_executor(None, lambda: requests.get(settings_dict['LIDARR_URL']+'/system/status', params=None, headers={'X-Api-Key': settings_dict['LIDARR_KEY']}, verify=settings_dict['SSL_VERIFICATION']))
logger.info('OK | %s', settings_dict['LIDARR_NAME'])
except Exception as error:
error_occured = True
logger.error('-- | %s *** Error: %s ***', settings_dict['LIDARR_NAME'], error)
if settings_dict['READARR_URL']:
try:
await asyncio.get_event_loop().run_in_executor(None, lambda: requests.get(settings_dict['READARR_URL']+'/system/status', params=None, headers={'X-Api-Key': settings_dict['READARR_KEY']}, verify=settings_dict['SSL_VERIFICATION']))
logger.info('OK | %s', settings_dict['READARR_NAME'])
except Exception as error:
error_occured = True
logger.error('-- | %s *** Error: %s ***', settings_dict['READARR_NAME'], error)
if settings_dict['QBITTORRENT_URL']:
# Checking if qbit can be reached, and checking if version is OK
try:
response = await asyncio.get_event_loop().run_in_executor(None, lambda: requests.post(settings_dict['QBITTORRENT_URL']+'/auth/login', data={'username': settings_dict['QBITTORRENT_USERNAME'], 'password': settings_dict['QBITTORRENT_PASSWORD']}, headers={'content-type': 'application/x-www-form-urlencoded'}, verify=settings_dict['SSL_VERIFICATION']))
if response.text == 'Fails.':
raise ConnectionError('Login failed.')
response.raise_for_status()
settings_dict['QBIT_COOKIE'] = {'SID': response.cookies['SID']}
except Exception as error:
error_occured = True
logger.error('-- | %s *** Error: %s / Reponse: %s ***', 'qBittorrent', error, response.text)
if not error_occured:
qbit_version = await rest_get(settings_dict['QBITTORRENT_URL']+'/app/version',cookies=settings_dict['QBIT_COOKIE'])
qbit_version = qbit_version[1:] # version without _v
if version.parse(qbit_version) < version.parse('4.3.0'):
error_occured = True
logger.error('-- | %s *** Error: %s ***', 'qBittorrent', 'Please update qBittorrent to at least version 4.3.0. Current version: ' + qbit_version)
if not error_occured:
logger.info('OK | %s', 'qBittorrent')
if error_occured:
logger.warning('At least one instance was not reachable. Waiting for 60 seconds, then exiting Decluttarr.')
await asyncio.sleep(60)
exit()
logger.info('')
logger.info('#' * 50)
if settings_dict['LOG_LEVEL'] == 'INFO':
logger.info('LOG_LEVEL = INFO: Only logging changes (switch to VERBOSE for more info)')
else:
logger.info(f'')
if settings_dict['TEST_RUN']:
logger.info(f'*'* 50)
logger.info(f'*'* 50)
logger.info(f'')
logger.info(f'!! TEST_RUN FLAG IS SET !!')
logger.info(f'NO UPDATES/DELETES WILL BE PERFORMED')
logger.info(f'')
logger.info(f'*'* 50)
logger.info(f'*'* 50)
# Check if Qbit Tag exists:
if settings_dict['QBITTORRENT_URL']:
current_tags = await rest_get(settings_dict['QBITTORRENT_URL']+'/torrents/tags',cookies=settings_dict['QBIT_COOKIE'])
if not settings_dict['NO_STALLED_REMOVAL_QBIT_TAG'] in current_tags:
if settings_dict['QBITTORRENT_URL']:
logger.info('Creating tag in qBittorrent: %s', settings_dict['NO_STALLED_REMOVAL_QBIT_TAG'])
if not settings_dict['TEST_RUN']:
await rest_post(url=settings_dict['QBITTORRENT_URL']+'/torrents/createTags', data={'tags': settings_dict['NO_STALLED_REMOVAL_QBIT_TAG']}, headers={'content-type': 'application/x-www-form-urlencoded'}, cookies=settings_dict['QBIT_COOKIE'])
# Start application
while True: while True:
logger.verbose('-' * 50) logger.verbose('-' * 50)
# Cache protected (via Tag) and private torrents # Cache protected (via Tag) and private torrents
protectedDownloadIDs = [] protectedDownloadIDs = []
privateDowloadIDs = [] privateDowloadIDs = []
if settings_dict['QBITTORRENT_URL']: if settingsDict['QBITTORRENT_URL']:
protectedDowloadItems = await rest_get(settings_dict['QBITTORRENT_URL']+'/torrents/info',params={'tag': settings_dict['NO_STALLED_REMOVAL_QBIT_TAG']}, cookies=settings_dict['QBIT_COOKIE'] ) protectedDowloadItems = await rest_get(settingsDict['QBITTORRENT_URL']+'/torrents/info',params={'tag': settingsDict['NO_STALLED_REMOVAL_QBIT_TAG']}, cookies=settingsDict['QBIT_COOKIE'] )
logger.debug('main/protectedDowloadItems: %s', str(protectedDowloadItems)) logger.debug('main/protectedDowloadItems: %s', str(protectedDowloadItems))
protectedDownloadIDs = [str.upper(item['hash']) for item in protectedDowloadItems] protectedDownloadIDs = [str.upper(item['hash']) for item in protectedDowloadItems]
if settings_dict['IGNORE_PRIVATE_TRACKERS']: if settingsDict['IGNORE_PRIVATE_TRACKERS']:
privateDowloadItems = await rest_get(settings_dict['QBITTORRENT_URL']+'/torrents/info',params={}, cookies=settings_dict['QBIT_COOKIE'] ) privateDowloadItems = await rest_get(settingsDict['QBITTORRENT_URL']+'/torrents/info',params={}, cookies=settingsDict['QBIT_COOKIE'] )
privateDowloadIDs = [str.upper(item['hash']) for item in privateDowloadItems if item.get('is_private', False)] privateDowloadIDs = [str.upper(item['hash']) for item in privateDowloadItems if item.get('is_private', False)]
if settings_dict['RADARR_URL']: await queueCleaner(settings_dict, 'radarr', defective_tracker, download_sizes_tracker, protectedDownloadIDs, privateDowloadIDs) # Run script for each instance
if settings_dict['SONARR_URL']: await queueCleaner(settings_dict, 'sonarr', defective_tracker, download_sizes_tracker, protectedDownloadIDs, privateDowloadIDs) for instance in settingsDict['INSTANCES']:
if settings_dict['LIDARR_URL']: await queueCleaner(settings_dict, 'lidarr', defective_tracker, download_sizes_tracker, protectedDownloadIDs, privateDowloadIDs) await queueCleaner(settingsDict, instance, defective_tracker, download_sizes_tracker, protectedDownloadIDs, privateDowloadIDs)
if settings_dict['READARR_URL']: await queueCleaner(settings_dict, 'readarr', defective_tracker, download_sizes_tracker, protectedDownloadIDs, privateDowloadIDs)
logger.verbose('') logger.verbose('')
logger.verbose('Queue clean-up complete!') logger.verbose('Queue clean-up complete!')
await asyncio.sleep(settings_dict['REMOVE_TIMER']*60)
# Wait for the next run
await asyncio.sleep(settingsDict['REMOVE_TIMER']*60)
return return
if __name__ == '__main__': if __name__ == '__main__':
instances = {settings_dict['RADARR_URL']: {}} if settings_dict['RADARR_URL'] else {} + \ arrApplications = ['RADARR', 'SONARR', 'LIDARR', 'READARR']
{settings_dict['SONARR_URL']: {}} if settings_dict['SONARR_URL'] else {} + \ defectiveTrackingInstances = {}
{settings_dict['LIDARR_URL']: {}} if settings_dict['LIDARR_URL'] else {} + \ settingsDict['INSTANCES'] = []
{settings_dict['READARR_URL']: {}} if settings_dict['READARR_URL'] else {} for arrApplication in arrApplications:
defective_tracker = Defective_Tracker(instances) if settingsDict[arrApplication + '_URL']:
settingsDict['INSTANCES'].append(arrApplication)
defectiveTrackingInstances[arrApplication] = {}
defective_tracker = Defective_Tracker(defectiveTrackingInstances)
download_sizes_tracker = Download_Sizes_Tracker({}) download_sizes_tracker = Download_Sizes_Tracker({})
asyncio.run(main()) asyncio.run(main(settingsDict))

76
src/decluttarr.py
View File

@@ -2,13 +2,13 @@
import logging, verboselogs import logging, verboselogs
logger = verboselogs.VerboseLogger(__name__) logger = verboselogs.VerboseLogger(__name__)
from src.utils.shared import (errorDetails, get_queue) from src.utils.shared import (errorDetails, get_queue)
from src.remove_failed import remove_failed from src.jobs.remove_failed import remove_failed
from src.remove_metadata_missing import remove_metadata_missing from src.jobs.remove_metadata_missing import remove_metadata_missing
from src.remove_missing_files import remove_missing_files from src.jobs.remove_missing_files import remove_missing_files
from src.remove_orphans import remove_orphans from src.jobs.remove_orphans import remove_orphans
from src.remove_slow import remove_slow from src.jobs.remove_slow import remove_slow
from src.remove_stalled import remove_stalled from src.jobs.remove_stalled import remove_stalled
from src.remove_unmonitored import remove_unmonitored from src.jobs.remove_unmonitored import remove_unmonitored
class Deleted_Downloads: class Deleted_Downloads:
# Keeps track of which downloads have already been deleted (to not double-delete) # Keeps track of which downloads have already been deleted (to not double-delete)
@@ -16,28 +16,28 @@ class Deleted_Downloads:
self.dict = dict self.dict = dict
async def queueCleaner(settings_dict, arr_type, defective_tracker, download_sizes_tracker, protectedDownloadIDs, privateDowloadIDs): async def queueCleaner(settingsDict, arr_type, defective_tracker, download_sizes_tracker, protectedDownloadIDs, privateDowloadIDs):
# Read out correct instance depending on radarr/sonarr flag # Read out correct instance depending on radarr/sonarr flag
run_dict = {} run_dict = {}
if arr_type == 'radarr': if arr_type == 'RADARR':
BASE_URL = settings_dict['RADARR_URL'] BASE_URL = settingsDict['RADARR_URL']
API_KEY = settings_dict['RADARR_KEY'] API_KEY = settingsDict['RADARR_KEY']
NAME = settings_dict['RADARR_NAME'] NAME = settingsDict['RADARR_NAME']
full_queue_param = 'includeUnknownMovieItems' full_queue_param = 'includeUnknownMovieItems'
elif arr_type == 'sonarr': elif arr_type == 'SONARR':
BASE_URL = settings_dict['SONARR_URL'] BASE_URL = settingsDict['SONARR_URL']
API_KEY = settings_dict['SONARR_KEY'] API_KEY = settingsDict['SONARR_KEY']
NAME = settings_dict['SONARR_NAME'] NAME = settingsDict['SONARR_NAME']
full_queue_param = 'includeUnknownSeriesItems' full_queue_param = 'includeUnknownSeriesItems'
elif arr_type == 'lidarr': elif arr_type == 'LIDARR':
BASE_URL = settings_dict['LIDARR_URL'] BASE_URL = settingsDict['LIDARR_URL']
API_KEY = settings_dict['LIDARR_KEY'] API_KEY = settingsDict['LIDARR_KEY']
NAME = settings_dict['LIDARR_NAME'] NAME = settingsDict['LIDARR_NAME']
full_queue_param = 'includeUnknownArtistItems' full_queue_param = 'includeUnknownArtistItems'
elif arr_type == 'readarr': elif arr_type == 'READARR':
BASE_URL = settings_dict['READARR_URL'] BASE_URL = settingsDict['READARR_URL']
API_KEY = settings_dict['READARR_KEY'] API_KEY = settingsDict['READARR_KEY']
NAME = settings_dict['READARR_NAME'] NAME = settingsDict['READARR_NAME']
full_queue_param = 'includeUnknownAuthorItems' full_queue_param = 'includeUnknownAuthorItems'
else: else:
logger.error('Unknown arr_type specified, exiting: %s', str(arr_type)) logger.error('Unknown arr_type specified, exiting: %s', str(arr_type))
@@ -54,26 +54,26 @@ async def queueCleaner(settings_dict, arr_type, defective_tracker, download_size
deleted_downloads = Deleted_Downloads([]) deleted_downloads = Deleted_Downloads([])
items_detected = 0 items_detected = 0
try: try:
if settings_dict['REMOVE_FAILED']: if settingsDict['REMOVE_FAILED']:
items_detected += await remove_failed( settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs) items_detected += await remove_failed( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs)
if settings_dict['REMOVE_STALLED']: if settingsDict['REMOVE_STALLED']:
items_detected += await remove_stalled( settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs) items_detected += await remove_stalled( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs)
if settings_dict['REMOVE_METADATA_MISSING']: if settingsDict['REMOVE_METADATA_MISSING']:
items_detected += await remove_metadata_missing( settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs) items_detected += await remove_metadata_missing( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs)
if settings_dict['REMOVE_ORPHANS']: if settingsDict['REMOVE_ORPHANS']:
items_detected += await remove_orphans( settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, full_queue_param) items_detected += await remove_orphans( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, full_queue_param)
if settings_dict['REMOVE_UNMONITORED']: if settingsDict['REMOVE_UNMONITORED']:
items_detected += await remove_unmonitored( settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, arr_type) items_detected += await remove_unmonitored( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, arr_type)
if settings_dict['REMOVE_MISSING_FILES']: if settingsDict['REMOVE_MISSING_FILES']:
items_detected += await remove_missing_files( settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs) items_detected += await remove_missing_files( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs)
if settings_dict['REMOVE_SLOW']: if settingsDict['REMOVE_SLOW']:
items_detected += await remove_slow( settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, download_sizes_tracker) items_detected += await remove_slow( settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, download_sizes_tracker)
if items_detected == 0: if items_detected == 0:
logger.verbose('>>> Queue is clean.') logger.verbose('>>> Queue is clean.')

4
src/remove_failed.py → src/jobs/remove_failed.py
View File

@@ -3,7 +3,7 @@ import sys, os, traceback
import logging, verboselogs import logging, verboselogs
logger = verboselogs.VerboseLogger(__name__) logger = verboselogs.VerboseLogger(__name__)
async def remove_failed(settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs): async def remove_failed(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs):
# Detects failed and triggers delete. Does not add to blocklist # Detects failed and triggers delete. Does not add to blocklist
try: try:
failType = 'failed' failType = 'failed'
@@ -16,7 +16,7 @@ async def remove_failed(settings_dict, BASE_URL, API_KEY, NAME, deleted_download
if 'errorMessage' in queueItem and 'status' in queueItem: if 'errorMessage' in queueItem and 'status' in queueItem:
if queueItem['status'] == 'failed': if queueItem['status'] == 'failed':
affectedItems.append(queueItem) affectedItems.append(queueItem)
affectedItems = await execute_checks(settings_dict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs,
addToBlocklist = False, addToBlocklist = False,
doPrivateTrackerCheck = True, doPrivateTrackerCheck = True,
doProtectedDownloadCheck = True, doProtectedDownloadCheck = True,

4
src/remove_metadata_missing.py → src/jobs/remove_metadata_missing.py
View File

@@ -3,7 +3,7 @@ import sys, os, traceback
import logging, verboselogs import logging, verboselogs
logger = verboselogs.VerboseLogger(__name__) logger = verboselogs.VerboseLogger(__name__)
async def remove_metadata_missing(settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs): async def remove_metadata_missing(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs):
# Detects downloads stuck downloading meta data and triggers repeat check and subsequent delete. Adds to blocklist # Detects downloads stuck downloading meta data and triggers repeat check and subsequent delete. Adds to blocklist
try: try:
failType = 'missing metadata' failType = 'missing metadata'
@@ -16,7 +16,7 @@ async def remove_metadata_missing(settings_dict, BASE_URL, API_KEY, NAME, delete
if 'errorMessage' in queueItem and 'status' in queueItem: if 'errorMessage' in queueItem and 'status' in queueItem:
if queueItem['status'] == 'queued' and queueItem['errorMessage'] == 'qBittorrent is downloading metadata': if queueItem['status'] == 'queued' and queueItem['errorMessage'] == 'qBittorrent is downloading metadata':
affectedItems.append(queueItem) affectedItems.append(queueItem)
affectedItems = await execute_checks(settings_dict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs,
addToBlocklist = True, addToBlocklist = True,
doPrivateTrackerCheck = True, doPrivateTrackerCheck = True,
doProtectedDownloadCheck = True, doProtectedDownloadCheck = True,

4
src/remove_missing_files.py → src/jobs/remove_missing_files.py
View File

@@ -3,7 +3,7 @@ import sys, os, traceback
import logging, verboselogs import logging, verboselogs
logger = verboselogs.VerboseLogger(__name__) logger = verboselogs.VerboseLogger(__name__)
async def remove_missing_files(settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs): async def remove_missing_files(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs):
# Detects downloads broken because of missing files. Does not add to blocklist # Detects downloads broken because of missing files. Does not add to blocklist
try: try:
failType = 'missing files' failType = 'missing files'
@@ -18,7 +18,7 @@ async def remove_missing_files(settings_dict, BASE_URL, API_KEY, NAME, deleted_d
(queueItem['errorMessage'] == 'DownloadClientQbittorrentTorrentStateMissingFiles' or (queueItem['errorMessage'] == 'DownloadClientQbittorrentTorrentStateMissingFiles' or
queueItem['errorMessage'] == 'The download is missing files')): queueItem['errorMessage'] == 'The download is missing files')):
affectedItems.append(queueItem) affectedItems.append(queueItem)
affectedItems = await execute_checks(settings_dict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs,
addToBlocklist = False, addToBlocklist = False,
doPrivateTrackerCheck = True, doPrivateTrackerCheck = True,
doProtectedDownloadCheck = True, doProtectedDownloadCheck = True,

4
src/remove_orphans.py → src/jobs/remove_orphans.py
View File

@@ -3,7 +3,7 @@ import sys, os, traceback
import logging, verboselogs import logging, verboselogs
logger = verboselogs.VerboseLogger(__name__) logger = verboselogs.VerboseLogger(__name__)
async def remove_orphans(settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, full_queue_param): async def remove_orphans(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, full_queue_param):
# Removes downloads belonging to movies/tv shows that have been deleted in the meantime. Does not add to blocklist # Removes downloads belonging to movies/tv shows that have been deleted in the meantime. Does not add to blocklist
try: try:
failType = 'orphan' failType = 'orphan'
@@ -22,7 +22,7 @@ async def remove_orphans(settings_dict, BASE_URL, API_KEY, NAME, deleted_downloa
if queueItem['id'] not in queueIDs: if queueItem['id'] not in queueIDs:
affectedItems.append(queueItem) affectedItems.append(queueItem)
affectedItems = await execute_checks(settings_dict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs,
addToBlocklist = False, addToBlocklist = False,
doPrivateTrackerCheck = True, doPrivateTrackerCheck = True,
doProtectedDownloadCheck = True, doProtectedDownloadCheck = True,

18
src/remove_slow.py → src/jobs/remove_slow.py
View File

@@ -3,7 +3,7 @@ import sys, os, traceback
import logging, verboselogs import logging, verboselogs
logger = verboselogs.VerboseLogger(__name__) logger = verboselogs.VerboseLogger(__name__)
async def remove_slow(settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, download_sizes_tracker): async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, download_sizes_tracker):
# Detects slow downloads and triggers delete. Adds to blocklist # Detects slow downloads and triggers delete. Adds to blocklist
try: try:
failType = 'slow' failType = 'slow'
@@ -18,17 +18,17 @@ async def remove_slow(settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads,
if queueItem['downloadId'] not in alreadyCheckedDownloadIDs: if queueItem['downloadId'] not in alreadyCheckedDownloadIDs:
alreadyCheckedDownloadIDs.append(queueItem['downloadId']) # One downloadId may occur in multiple queueItems - only check once for all of them per iteration alreadyCheckedDownloadIDs.append(queueItem['downloadId']) # One downloadId may occur in multiple queueItems - only check once for all of them per iteration
# determine if the downloaded bit on average between this and the last iteration is greater than the min threshold # determine if the downloaded bit on average between this and the last iteration is greater than the min threshold
downloadedSize, previousSize, increment, speed = await getDownloadedSize(settings_dict, queueItem, download_sizes_tracker, NAME) downloadedSize, previousSize, increment, speed = await getDownloadedSize(settingsDict, queueItem, download_sizes_tracker, NAME)
if queueItem['status'] == 'downloading' and \ if queueItem['status'] == 'downloading' and \
queueItem['downloadId'] in download_sizes_tracker.dict and \ queueItem['downloadId'] in download_sizes_tracker.dict and \
speed is not None: speed is not None:
if speed < settings_dict['MIN_DOWNLOAD_SPEED']: if speed < settingsDict['MIN_DOWNLOAD_SPEED']:
affectedItems.append(queueItem) affectedItems.append(queueItem)
logger.debug('remove_slow/slow speed detected: %s (Speed: %d KB/s, KB now: %s, KB previous: %s, Diff: %s, In Minutes: %s', \ logger.debug('remove_slow/slow speed detected: %s (Speed: %d KB/s, KB now: %s, KB previous: %s, Diff: %s, In Minutes: %s', \
queueItem['title'], speed, downloadedSize, previousSize, increment, settings_dict['REMOVE_TIMER']) queueItem['title'], speed, downloadedSize, previousSize, increment, settingsDict['REMOVE_TIMER'])
affectedItems = await execute_checks(settings_dict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs,
addToBlocklist = True, addToBlocklist = True,
doPrivateTrackerCheck = True, doPrivateTrackerCheck = True,
doProtectedDownloadCheck = True, doProtectedDownloadCheck = True,
@@ -39,12 +39,12 @@ async def remove_slow(settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads,
return 0 return 0
from src.utils.rest import (rest_get) from src.utils.rest import (rest_get)
async def getDownloadedSize(settings_dict, queueItem, download_sizes_tracker, NAME): async def getDownloadedSize(settingsDict, queueItem, download_sizes_tracker, NAME):
try: try:
# Determines the speed of download # Determines the speed of download
# Since Sonarr/Radarr do not update the downlodedSize on realtime, if possible, fetch it directly from qBit # Since Sonarr/Radarr do not update the downlodedSize on realtime, if possible, fetch it directly from qBit
if settings_dict['QBITTORRENT_URL'] and queueItem['downloadClient'] == 'qBittorrent': if settingsDict['QBITTORRENT_URL'] and queueItem['downloadClient'] == 'qBittorrent':
qbitInfo = await rest_get(settings_dict['QBITTORRENT_URL']+'/torrents/info',params={'hashes': queueItem['downloadId']}, cookies=settings_dict['QBIT_COOKIE'] ) qbitInfo = await rest_get(settingsDict['QBITTORRENT_URL']+'/torrents/info',params={'hashes': queueItem['downloadId']}, cookies=settingsDict['QBIT_COOKIE'] )
downloadedSize = qbitInfo[0]['completed'] downloadedSize = qbitInfo[0]['completed']
else: else:
logger.debug('getDownloadedSize/WARN: Using imprecise method to determine download increments because no direct qBIT query is possible') logger.debug('getDownloadedSize/WARN: Using imprecise method to determine download increments because no direct qBIT query is possible')
@@ -52,7 +52,7 @@ async def getDownloadedSize(settings_dict, queueItem, download_sizes_tracker, NA
if queueItem['downloadId'] in download_sizes_tracker.dict: if queueItem['downloadId'] in download_sizes_tracker.dict:
previousSize = download_sizes_tracker.dict.get(queueItem['downloadId']) previousSize = download_sizes_tracker.dict.get(queueItem['downloadId'])
increment = downloadedSize - previousSize increment = downloadedSize - previousSize
speed = round(increment / 1000 / (settings_dict['REMOVE_TIMER'] * 60),1) speed = round(increment / 1000 / (settingsDict['REMOVE_TIMER'] * 60),1)
else: else:
previousSize = None previousSize = None
increment = None increment = None

4
src/remove_stalled.py → src/jobs/remove_stalled.py
View File

@@ -3,7 +3,7 @@ import sys, os, traceback
import logging, verboselogs import logging, verboselogs
logger = verboselogs.VerboseLogger(__name__) logger = verboselogs.VerboseLogger(__name__)
async def remove_stalled(settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs): async def remove_stalled(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs):
# Detects stalled and triggers repeat check and subsequent delete. Adds to blocklist # Detects stalled and triggers repeat check and subsequent delete. Adds to blocklist
try: try:
failType = 'stalled' failType = 'stalled'
@@ -16,7 +16,7 @@ async def remove_stalled(settings_dict, BASE_URL, API_KEY, NAME, deleted_downloa
if 'errorMessage' in queueItem and 'status' in queueItem: if 'errorMessage' in queueItem and 'status' in queueItem:
if queueItem['status'] == 'warning' and queueItem['errorMessage'] == 'The download is stalled with no connections': if queueItem['status'] == 'warning' and queueItem['errorMessage'] == 'The download is stalled with no connections':
affectedItems.append(queueItem) affectedItems.append(queueItem)
affectedItems = await execute_checks(settings_dict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs,
addToBlocklist = True, addToBlocklist = True,
doPrivateTrackerCheck = True, doPrivateTrackerCheck = True,
doProtectedDownloadCheck = True, doProtectedDownloadCheck = True,

12
src/remove_unmonitored.py → src/jobs/remove_unmonitored.py
View File

@@ -4,7 +4,7 @@ import logging, verboselogs
logger = verboselogs.VerboseLogger(__name__) logger = verboselogs.VerboseLogger(__name__)
from src.utils.rest import rest_get from src.utils.rest import rest_get
async def remove_unmonitored(settings_dict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, arr_type): async def remove_unmonitored(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, arr_type):
# Removes downloads belonging to movies/tv shows that are not monitored. Does not add to blocklist # Removes downloads belonging to movies/tv shows that are not monitored. Does not add to blocklist
try: try:
failType = 'unmonitored' failType = 'unmonitored'
@@ -14,13 +14,13 @@ async def remove_unmonitored(settings_dict, BASE_URL, API_KEY, NAME, deleted_dow
# Find items affected # Find items affected
monitoredDownloadIDs = [] monitoredDownloadIDs = []
for queueItem in queue['records']: for queueItem in queue['records']:
if arr_type == 'sonarr': if arr_type == 'SONARR':
isMonitored = (await rest_get(f'{BASE_URL}/episode/{str(queueItem["episodeId"])}', API_KEY))['monitored'] isMonitored = (await rest_get(f'{BASE_URL}/episode/{str(queueItem["episodeId"])}', API_KEY))['monitored']
elif arr_type == 'radarr': elif arr_type == 'RADARR':
isMonitored = (await rest_get(f'{BASE_URL}/movie/{str(queueItem["movieId"])}', API_KEY))['monitored'] isMonitored = (await rest_get(f'{BASE_URL}/movie/{str(queueItem["movieId"])}', API_KEY))['monitored']
elif arr_type == 'lidarr': elif arr_type == 'LIDARR':
isMonitored = (await rest_get(f'{BASE_URL}/album/{str(queueItem["albumId"])}', API_KEY))['monitored'] isMonitored = (await rest_get(f'{BASE_URL}/album/{str(queueItem["albumId"])}', API_KEY))['monitored']
elif arr_type == 'readarr': elif arr_type == 'READARR':
isMonitored = (await rest_get(f'{BASE_URL}/book/{str(queueItem["bookId"])}', API_KEY))['monitored'] isMonitored = (await rest_get(f'{BASE_URL}/book/{str(queueItem["bookId"])}', API_KEY))['monitored']
if isMonitored: if isMonitored:
monitoredDownloadIDs.append(queueItem['downloadId']) monitoredDownloadIDs.append(queueItem['downloadId'])
@@ -30,7 +30,7 @@ async def remove_unmonitored(settings_dict, BASE_URL, API_KEY, NAME, deleted_dow
if queueItem['downloadId'] not in monitoredDownloadIDs: if queueItem['downloadId'] not in monitoredDownloadIDs:
affectedItems.append(queueItem) # One downloadID may be shared by multiple queueItems. Only removes it if ALL queueitems are unmonitored affectedItems.append(queueItem) # One downloadID may be shared by multiple queueItems. Only removes it if ALL queueitems are unmonitored
affectedItems = await execute_checks(settings_dict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs,
addToBlocklist = False, addToBlocklist = False,
doPrivateTrackerCheck = True, doPrivateTrackerCheck = True,
doProtectedDownloadCheck = True, doProtectedDownloadCheck = True,

159
src/utils/loadScripts.py Normal file
View File

@@ -0,0 +1,159 @@
########### Import Libraries
import logging, verboselogs
logger = verboselogs.VerboseLogger(__name__)
from dateutil.relativedelta import relativedelta as rd
import requests
from src.utils.rest import rest_get, rest_post #
import asyncio
from packaging import version
def setLoggingFormat(settingsDict):
# Sets logger output to specific format
log_level_num=logging.getLevelName(settingsDict['LOG_LEVEL'])
logging.basicConfig(
format=('' if settingsDict['IS_IN_DOCKER'] else '%(asctime)s ') + ('[%(levelname)-7s]' if settingsDict['LOG_LEVEL']=='VERBOSE' else '[%(levelname)s]') + ': %(message)s',
level=log_level_num
)
return
async def getArrInstanceName(settingsDict, arrApp):
# Retrieves the names of the arr instances, and if not defined, sets a default
try:
if settingsDict[arrApp + '_URL']:
settingsDict[arrApp + '_NAME'] = (await rest_get(settingsDict[arrApp + '_URL']+'/system/status', settingsDict[arrApp + '_KEY']))['instanceName']
except:
settingsDict[arrApp + '_NAME'] = arrApp.capitalize()
return settingsDict
def showSettings(settingsDict):
# Prints out the settings
fmt = '{0.days} days {0.hours} hours {0.minutes} minutes'
logger.info('#' * 50)
logger.info('Decluttarr - Application Started!')
logger.info('')
logger.info('*** Current Settings ***')
logger.info('Version: %s', settingsDict['IMAGE_TAG'])
logger.info('Commit: %s', settingsDict['SHORT_COMMIT_ID'])
logger.info('')
logger.info('%s | Removing failed downloads', str(settingsDict['REMOVE_FAILED']))
logger.info('%s | Removing downloads missing metadata', str(settingsDict['REMOVE_METADATA_MISSING']))
logger.info('%s | Removing downloads missing files', str(settingsDict['REMOVE_MISSING_FILES']))
logger.info('%s | Removing orphan downloads', str(settingsDict['REMOVE_ORPHANS']))
logger.info('%s | Removing slow downloads', str(settingsDict['REMOVE_SLOW']))
logger.info('%s | Removing stalled downloads', str(settingsDict['REMOVE_STALLED']))
logger.info('%s | Removing downloads belonging to unmonitored items', str(settingsDict['REMOVE_UNMONITORED']))
logger.info('')
logger.info('Running every: %s', fmt.format(rd(minutes=settingsDict['REMOVE_TIMER'])))
if settingsDict['REMOVE_SLOW']:
logger.info('Minimum speed enforced: %s KB/s', str(settingsDict['MIN_DOWNLOAD_SPEED']))
logger.info('Permitted number of times before stalled/missing metadata/slow downloads are removed: %s', str(settingsDict['PERMITTED_ATTEMPTS']))
if settingsDict['QBITTORRENT_URL']:
logger.info('Downloads with this tag will be skipped: \"%s\"', settingsDict['NO_STALLED_REMOVAL_QBIT_TAG'])
logger.info('Private Trackers will be skipped: %s', settingsDict['IGNORE_PRIVATE_TRACKERS'])
logger.info('')
logger.info('*** Configured Instances ***')
for instance in settingsDict['INSTANCES']:
if settingsDict[instance + '_URL']:
logger.info('%s: %s', settingsDict[instance + '_NAME'], settingsDict[instance + '_URL'])
if settingsDict['QBITTORRENT_URL']: logger.info('qBittorrent: %s', settingsDict['QBITTORRENT_URL'])
logger.info('')
return
async def instanceChecks(settingsDict):
# Checks if the arr and qbit instances are reachable, and returns the settings dictionary with the qbit cookie
logger.info('*** Check Instances ***')
error_occured = False
# Check ARR-apps
for instance in settingsDict['INSTANCES']:
if settingsDict[instance + '_URL']:
try:
await asyncio.get_event_loop().run_in_executor(None, lambda: requests.get(settingsDict[instance + '_URL']+'/system/status', params=None, headers={'X-Api-Key': settingsDict[instance + '_KEY']}, verify=settingsDict['SSL_VERIFICATION']))
except Exception as error:
error_occured = True
logger.error('!! %s Error: !!', settingsDict[instance + '_NAME'])
logger.error(error)
if not error_occured:
current_version = (await rest_get(settingsDict[instance + '_URL']+'/system/status', settingsDict[instance + '_KEY']))['version']
if settingsDict[instance + '_MIN_VERSION']:
if version.parse(current_version) < version.parse(settingsDict[instance + '_MIN_VERSION']):
error_occured = True
logger.error('!! %s Error: !!', settingsDict[instance + '_NAME'])
logger.error('Please update %s to at least version %s. Current version: %s', settingsDict[instance + 'MIN_VERSION'],current_version)
if not error_occured:
logger.info('OK | %s', settingsDict[instance + '_NAME'])
logger.debug('Current version of %s: %s', instance, current_version)
# Check Bittorrent
if settingsDict['QBITTORRENT_URL']:
# Checking if qbit can be reached, and checking if version is OK
try:
response = await asyncio.get_event_loop().run_in_executor(None, lambda: requests.post(settingsDict['QBITTORRENT_URL']+'/auth/login', data={'username': settingsDict['QBITTORRENT_USERNAME'], 'password': settingsDict['QBITTORRENT_PASSWORD']}, headers={'content-type': 'application/x-www-form-urlencoded'}, verify=settingsDict['SSL_VERIFICATION']))
if response.text == 'Fails.':
raise ConnectionError('Login failed.')
response.raise_for_status()
settingsDict['QBIT_COOKIE'] = {'SID': response.cookies['SID']}
except Exception as error:
error_occured = True
logger.error('!! %s Error: !!', 'qBittorrent')
logger.error(error)
logger.error('Details:')
logger.error(response.text)
if not error_occured:
qbit_version = await rest_get(settingsDict['QBITTORRENT_URL']+'/app/version',cookies=settingsDict['QBIT_COOKIE'])
qbit_version = qbit_version[1:] # version without _v
if version.parse(qbit_version) < version.parse(settingsDict['QBITTORRENT_MIN_VERSION']):
error_occured = True
logger.error('-- | %s *** Error: %s ***', 'qBittorrent', 'Please update qBittorrent to at least version %s Current version: %s',settingsDict['QBITTORRENT_MIN_VERSION'], qbit_version)
if not error_occured:
logger.info('OK | %s', 'qBittorrent')
logger.debug('Current version of %s: %s', 'qBittorrent', qbit_version)
if error_occured:
logger.warning('At least one instance was not reachable. Waiting for 60 seconds, then exiting Decluttarr.')
await asyncio.sleep(60)
exit()
logger.info('')
return settingsDict
async def createQbitProtectionTag(settingsDict):
# Creates the qBit Protection tag if not already present
if settingsDict['QBITTORRENT_URL']:
current_tags = await rest_get(settingsDict['QBITTORRENT_URL']+'/torrents/tags',cookies=settingsDict['QBIT_COOKIE'])
if not settingsDict['NO_STALLED_REMOVAL_QBIT_TAG'] in current_tags:
if settingsDict['QBITTORRENT_URL']:
logger.info('Creating tag in qBittorrent: %s', settingsDict['NO_STALLED_REMOVAL_QBIT_TAG'])
if not settingsDict['TEST_RUN']:
await rest_post(url=settingsDict['QBITTORRENT_URL']+'/torrents/createTags', data={'tags': settingsDict['NO_STALLED_REMOVAL_QBIT_TAG']}, headers={'content-type': 'application/x-www-form-urlencoded'}, cookies=settingsDict['QBIT_COOKIE'])
def showLoggerSettings(settingsDict):
logger.info('#' * 50)
if settingsDict['LOG_LEVEL'] == 'INFO':
logger.info('LOG_LEVEL = INFO: Only logging changes (switch to VERBOSE for more info)')
else:
logger.info(f'')
if settingsDict['TEST_RUN']:
logger.info(f'*'* 50)
logger.info(f'*'* 50)
logger.info(f'')
logger.info(f'!! TEST_RUN FLAG IS SET !!')
logger.info(f'NO UPDATES/DELETES WILL BE PERFORMED')
logger.info(f'')
logger.info(f'*'* 50)
logger.info(f'*'* 50)

16
src/utils/rest.py
View File

@@ -4,13 +4,13 @@ import asyncio
import requests import requests
from requests.exceptions import RequestException from requests.exceptions import RequestException
import json import json
from config.config import settings_dict from config.config import settingsDict
# GET # GET
async def rest_get(url, api_key=None, params=None, cookies=None): async def rest_get(url, api_key=None, params=None, cookies=None):
try: try:
headers = {'X-Api-Key': api_key} if api_key else None headers = {'X-Api-Key': api_key} if api_key else None
response = await asyncio.get_event_loop().run_in_executor(None, lambda: requests.get(url, params=params, headers=headers, cookies=cookies, verify=settings_dict['SSL_VERIFICATION'])) response = await asyncio.get_event_loop().run_in_executor(None, lambda: requests.get(url, params=params, headers=headers, cookies=cookies, verify=settingsDict['SSL_VERIFICATION']))
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()
except requests.exceptions.HTTPError as e: except requests.exceptions.HTTPError as e:
@@ -23,10 +23,10 @@ async def rest_get(url, api_key=None, params=None, cookies=None):
# DELETE # DELETE
async def rest_delete(url, api_key, params=None): async def rest_delete(url, api_key, params=None):
if settings_dict['TEST_RUN']: return if settingsDict['TEST_RUN']: return
try: try:
headers = {'X-Api-Key': api_key} headers = {'X-Api-Key': api_key}
response = await asyncio.get_event_loop().run_in_executor(None, lambda: requests.delete(url, params=params, headers=headers, verify=settings_dict['SSL_VERIFICATION'])) response = await asyncio.get_event_loop().run_in_executor(None, lambda: requests.delete(url, params=params, headers=headers, verify=settingsDict['SSL_VERIFICATION']))
response.raise_for_status() response.raise_for_status()
if response.status_code in [200, 204]: if response.status_code in [200, 204]:
return None return None
@@ -40,9 +40,9 @@ async def rest_delete(url, api_key, params=None):
# POST # POST
async def rest_post(url, data=None, json=None, headers=None, cookies=None): async def rest_post(url, data=None, json=None, headers=None, cookies=None):
if settings_dict['TEST_RUN']: return if settingsDict['TEST_RUN']: return
try: try:
response = await asyncio.get_event_loop().run_in_executor(None, lambda: requests.post(url, data=data, json=json, headers=headers, cookies=cookies, verify=settings_dict['SSL_VERIFICATION'])) response = await asyncio.get_event_loop().run_in_executor(None, lambda: requests.post(url, data=data, json=json, headers=headers, cookies=cookies, verify=settingsDict['SSL_VERIFICATION']))
response.raise_for_status() response.raise_for_status()
if response.status_code in (200,201): if response.status_code in (200,201):
return None return None
@@ -57,10 +57,10 @@ async def rest_post(url, data=None, json=None, headers=None, cookies=None):
# PUT # PUT
async def rest_put(url, api_key, data): async def rest_put(url, api_key, data):
if settings_dict['TEST_RUN']: return if settingsDict['TEST_RUN']: return
try: try:
headers = {'X-Api-Key': api_key} | {"content-type": "application/json"} headers = {'X-Api-Key': api_key} | {"content-type": "application/json"}
response = await asyncio.get_event_loop().run_in_executor(None, lambda: requests.put(url, data=data, headers=headers, verify=settings_dict['SSL_VERIFICATION'])) response = await asyncio.get_event_loop().run_in_executor(None, lambda: requests.put(url, data=data, headers=headers, verify=settingsDict['SSL_VERIFICATION']))
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()
except RequestException as e: except RequestException as e:

50
src/utils/shared.py
View File

@@ -12,16 +12,36 @@ async def get_queue(BASE_URL, API_KEY, params = {}):
if totalRecords == 0: if totalRecords == 0:
return None return None
queue = await rest_get(f'{BASE_URL}/queue', API_KEY, {'page': '1', 'pageSize': totalRecords}|params) queue = await rest_get(f'{BASE_URL}/queue', API_KEY, {'page': '1', 'pageSize': totalRecords}|params)
queue = filterOutDelayedQueueItems(queue)
return queue return queue
def privateTrackerCheck(settings_dict, affectedItems, failType, privateDowloadIDs): def filterOutDelayedQueueItems(queue):
# Ignores delayed queue items
if queue is None:
return None
seen_combinations = set()
filtered_records = []
for record in queue['records']:
combination = (record['title'], record['indexer'])
if record['status'] == 'delay':
if combination not in seen_combinations:
seen_combinations.add(combination)
logger.debug('>>> Delayed queue item ignored: %s (Indexer: %s)', record['title'], record['indexer'])
else:
filtered_records.append(record)
if not filtered_records:
return None
queue['records'] = filtered_records
return queue
def privateTrackerCheck(settingsDict, affectedItems, failType, privateDowloadIDs):
# Ignores private tracker items (if setting is turned on) # Ignores private tracker items (if setting is turned on)
for affectedItem in reversed(affectedItems): for affectedItem in reversed(affectedItems):
if settings_dict['IGNORE_PRIVATE_TRACKERS'] and affectedItem['downloadId'] in privateDowloadIDs: if settingsDict['IGNORE_PRIVATE_TRACKERS'] and affectedItem['downloadId'] in privateDowloadIDs:
affectedItems.remove(affectedItem) affectedItems.remove(affectedItem)
return affectedItems return affectedItems
def protectedDownloadCheck(settings_dict, affectedItems, failType, protectedDownloadIDs): def protectedDownloadCheck(settingsDict, affectedItems, failType, protectedDownloadIDs):
# Checks if torrent is protected and skips # Checks if torrent is protected and skips
logger.debug('protectedDownloadCheck/protectedDownloadIDs (failType: %s): %s', failType, str(protectedDownloadIDs)) logger.debug('protectedDownloadCheck/protectedDownloadIDs (failType: %s): %s', failType, str(protectedDownloadIDs))
for affectedItem in reversed(affectedItems): for affectedItem in reversed(affectedItems):
@@ -32,7 +52,7 @@ def protectedDownloadCheck(settings_dict, affectedItems, failType, protectedDown
return affectedItems return affectedItems
async def execute_checks(settings_dict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, addToBlocklist, doPrivateTrackerCheck, doProtectedDownloadCheck, doPermittedAttemptsCheck): async def execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, addToBlocklist, doPrivateTrackerCheck, doProtectedDownloadCheck, doPermittedAttemptsCheck):
# Goes over the affected items and performs the checks that are parametrized # Goes over the affected items and performs the checks that are parametrized
try: try:
# De-duplicates the affected items (one downloadid may be shared by multiple affected items) # De-duplicates the affected items (one downloadid may be shared by multiple affected items)
@@ -44,17 +64,17 @@ async def execute_checks(settings_dict, affectedItems, failType, BASE_URL, API_K
affectedItems.remove(affectedItem) affectedItems.remove(affectedItem)
# Skips protected items # Skips protected items
if doPrivateTrackerCheck: if doPrivateTrackerCheck:
affectedItems = privateTrackerCheck(settings_dict, affectedItems, failType, privateDowloadIDs) affectedItems = privateTrackerCheck(settingsDict, affectedItems, failType, privateDowloadIDs)
if doProtectedDownloadCheck: if doProtectedDownloadCheck:
affectedItems = protectedDownloadCheck(settings_dict, affectedItems, failType, protectedDownloadIDs) affectedItems = protectedDownloadCheck(settingsDict, affectedItems, failType, protectedDownloadIDs)
# Checks if failing more often than permitted # Checks if failing more often than permitted
if doPermittedAttemptsCheck: if doPermittedAttemptsCheck:
affectedItems = permittedAttemptsCheck(settings_dict, affectedItems, failType, BASE_URL, defective_tracker) affectedItems = permittedAttemptsCheck(settingsDict, affectedItems, failType, BASE_URL, defective_tracker)
# Deletes all downloads that have not survived the checks # Deletes all downloads that have not survived the checks
for affectedItem in affectedItems: for affectedItem in affectedItems:
await remove_download(settings_dict, BASE_URL, API_KEY, affectedItem, failType, addToBlocklist, deleted_downloads) await remove_download(settingsDict, BASE_URL, API_KEY, affectedItem, failType, addToBlocklist, deleted_downloads)
# Exit Logs # Exit Logs
if settings_dict['LOG_LEVEL'] == 'DEBUG': if settingsDict['LOG_LEVEL'] == 'DEBUG':
queue = await get_queue(BASE_URL, API_KEY) queue = await get_queue(BASE_URL, API_KEY)
logger.debug('execute_checks/queue OUT (failType: %s): %s', failType, formattedQueueInfo(queue)) logger.debug('execute_checks/queue OUT (failType: %s): %s', failType, formattedQueueInfo(queue))
# Return removed items # Return removed items
@@ -63,7 +83,7 @@ async def execute_checks(settings_dict, affectedItems, failType, BASE_URL, API_K
errorDetails(NAME, error) errorDetails(NAME, error)
return [] return []
def permittedAttemptsCheck(settings_dict, affectedItems, failType, BASE_URL, defective_tracker): def permittedAttemptsCheck(settingsDict, affectedItems, failType, BASE_URL, defective_tracker):
# Checks if downloads are repeatedly found as stalled / stuck in metadata. Removes the items that are not exeeding permitted attempts # Checks if downloads are repeatedly found as stalled / stuck in metadata. Removes the items that are not exeeding permitted attempts
# Shows all affected items (for debugging) # Shows all affected items (for debugging)
logger.debug('permittedAttemptsCheck/affectedItems: %s', ', '.join(f"{affectedItem['id']}:{affectedItem['title']}:{affectedItem['downloadId']}" for affectedItem in affectedItems)) logger.debug('permittedAttemptsCheck/affectedItems: %s', ', '.join(f"{affectedItem['id']}:{affectedItem['title']}:{affectedItem['downloadId']}" for affectedItem in affectedItems))
@@ -86,25 +106,25 @@ def permittedAttemptsCheck(settings_dict, affectedItems, failType, BASE_URL, def
defective_tracker.dict[BASE_URL][failType][affectedItem['downloadId']]['Attempts'] += 1 defective_tracker.dict[BASE_URL][failType][affectedItem['downloadId']]['Attempts'] += 1
except KeyError: except KeyError:
add_keys_nested_dict(defective_tracker.dict,[BASE_URL, failType, affectedItem['downloadId']], {'title': affectedItem['title'], 'Attempts': 1}) add_keys_nested_dict(defective_tracker.dict,[BASE_URL, failType, affectedItem['downloadId']], {'title': affectedItem['title'], 'Attempts': 1})
attempts_left = settings_dict['PERMITTED_ATTEMPTS'] - defective_tracker.dict[BASE_URL][failType][affectedItem['downloadId']]['Attempts'] attempts_left = settingsDict['PERMITTED_ATTEMPTS'] - defective_tracker.dict[BASE_URL][failType][affectedItem['downloadId']]['Attempts']
# If not exceeding the number of permitted times, remove from being affected # If not exceeding the number of permitted times, remove from being affected
if attempts_left >= 0: # Still got attempts left if attempts_left >= 0: # Still got attempts left
logger.info('>>> Detected %s download (%s out of %s permitted times): %s', failType, str(defective_tracker.dict[BASE_URL][failType][affectedItem['downloadId']]['Attempts']), str(settings_dict['PERMITTED_ATTEMPTS']), affectedItem['title']) logger.info('>>> Detected %s download (%s out of %s permitted times): %s', failType, str(defective_tracker.dict[BASE_URL][failType][affectedItem['downloadId']]['Attempts']), str(settingsDict['PERMITTED_ATTEMPTS']), affectedItem['title'])
affectedItems.remove(affectedItem) affectedItems.remove(affectedItem)
if attempts_left <= -1: # Too many attempts if attempts_left <= -1: # Too many attempts
logger.info('>>> Detected %s download too many times (%s out of %s permitted times): %s', failType, str(defective_tracker.dict[BASE_URL][failType][affectedItem['downloadId']]['Attempts']), str(settings_dict['PERMITTED_ATTEMPTS']), affectedItem['title']) logger.info('>>> Detected %s download too many times (%s out of %s permitted times): %s', failType, str(defective_tracker.dict[BASE_URL][failType][affectedItem['downloadId']]['Attempts']), str(settingsDict['PERMITTED_ATTEMPTS']), affectedItem['title'])
if attempts_left <= -2: # Too many attempts and should already have been removed if attempts_left <= -2: # Too many attempts and should already have been removed
# If supposedly deleted item keeps coming back, print out guidance for "Reject Blocklisted Torrent Hashes While Grabbing" # If supposedly deleted item keeps coming back, print out guidance for "Reject Blocklisted Torrent Hashes While Grabbing"
logger.verbose('>>> [Tip!] Since this download should already have been removed in a previous iteration but keeps coming back, this indicates the blocking of the torrent does not work correctly. Consider turning on the option "Reject Blocklisted Torrent Hashes While Grabbing" on the indexer in the *arr app: %s', affectedItem['title']) logger.verbose('>>> [Tip!] Since this download should already have been removed in a previous iteration but keeps coming back, this indicates the blocking of the torrent does not work correctly. Consider turning on the option "Reject Blocklisted Torrent Hashes While Grabbing" on the indexer in the *arr app: %s', affectedItem['title'])
logger.debug('permittedAttemptsCheck/defective_tracker.dict OUT: %s', str(defective_tracker.dict)) logger.debug('permittedAttemptsCheck/defective_tracker.dict OUT: %s', str(defective_tracker.dict))
return affectedItems return affectedItems
async def remove_download(settings_dict, BASE_URL, API_KEY, affectedItem, failType, addToBlocklist, deleted_downloads): async def remove_download(settingsDict, BASE_URL, API_KEY, affectedItem, failType, addToBlocklist, deleted_downloads):
# Removes downloads and creates log entry # Removes downloads and creates log entry
logger.debug('remove_download/deleted_downloads.dict IN: %s', str(deleted_downloads.dict)) logger.debug('remove_download/deleted_downloads.dict IN: %s', str(deleted_downloads.dict))
if affectedItem['downloadId'] not in deleted_downloads.dict: if affectedItem['downloadId'] not in deleted_downloads.dict:
logger.info('>>> Removing %s download: %s', failType, affectedItem['title']) logger.info('>>> Removing %s download: %s', failType, affectedItem['title'])
if not settings_dict['TEST_RUN']: if not settingsDict['TEST_RUN']:
await rest_delete(f'{BASE_URL}/queue/{affectedItem["id"]}', API_KEY, {'removeFromClient': True, 'blocklist': addToBlocklist}) await rest_delete(f'{BASE_URL}/queue/{affectedItem["id"]}', API_KEY, {'removeFromClient': True, 'blocklist': addToBlocklist})
deleted_downloads.dict.append(affectedItem['downloadId']) deleted_downloads.dict.append(affectedItem['downloadId'])