From 10ce4890d21b8a4c0c22541e89826cd16fd9c2e0 Mon Sep 17 00:00:00 2001 From: Jack <5182053+phyzical@users.noreply.github.com> Date: Sat, 11 May 2024 12:47:38 +0800 Subject: [PATCH 01/15] Preload requirements file Due tot he fact version pinning is in use, there's no point in reinstalling deps each build. Instead copy everything after dep install to increase caching --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 72d41d3..0fffa8b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,8 +18,8 @@ ENV IS_IN_DOCKER 1 WORKDIR /app -COPY . . +COPY ./docker/requirements.txt ./docker/requirements.txt RUN pip install --no-cache-dir -r docker/requirements.txt - +COPY . . CMD ["python", "main.py"] From 06874200bd04bceb01ef32a1642984c2c21bb193 Mon Sep 17 00:00:00 2001 From: Jack <5182053+phyzical@users.noreply.github.com> Date: Sat, 11 May 2024 13:18:11 +0800 Subject: [PATCH 02/15] Update remove_missing_files.py Support "no files found for import" auto removal --- src/jobs/remove_missing_files.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/jobs/remove_missing_files.py b/src/jobs/remove_missing_files.py index 2444848..68efb11 100644 --- a/src/jobs/remove_missing_files.py +++ b/src/jobs/remove_missing_files.py @@ -13,11 +13,19 @@ async def remove_missing_files(settingsDict, BASE_URL, API_KEY, NAME, deleted_do # Find items affected affectedItems = [] for queueItem in queue['records']: - if 'errorMessage' in queueItem and 'status' in queueItem: - if (queueItem['status'] == 'warning' and - (queueItem['errorMessage'] == 'DownloadClientQbittorrentTorrentStateMissingFiles' or + if 'status' in queueItem: + # case to check for failed torrents + if (queueItem['status'] == 'warning' and 'errorMessage' in queueItem and + (queueItem['errorMessage'] == 'DownloadClientQbittorrentTorrentStateMissingFiles' or queueItem['errorMessage'] == 'The download is missing files')): affectedItems.append(queueItem) + # case to check for failed nzb's/bad files/empty directory + if queueItem['status'] == 'completed' and 'statusMessages' in queueItem: + for statusMessage in queueItem['statusMessages']: + if 'messages' in statusMessage: + for message in statusMessage['messages']: + if message.startswith("No files found are eligible for import in"): + affectedItems.append(queueItem) affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, addToBlocklist = False, doPrivateTrackerCheck = True, @@ -26,4 +34,4 @@ async def remove_missing_files(settingsDict, BASE_URL, API_KEY, NAME, deleted_do return len(affectedItems) except Exception as error: errorDetails(NAME, error) - return 0 \ No newline at end of file + return 0 From ffbfea5f2cdc3dad5035f856e8051f2fb62f0d3d Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Sun, 12 May 2024 15:38:40 +0200 Subject: [PATCH 03/15] Checking if qBit is connected prior to slowness check --- src/jobs/remove_slow.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/jobs/remove_slow.py b/src/jobs/remove_slow.py index 9f26a03..f66b103 100644 --- a/src/jobs/remove_slow.py +++ b/src/jobs/remove_slow.py @@ -13,6 +13,13 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, # Find items affected affectedItems = [] alreadyCheckedDownloadIDs = [] + + if settingsDict['QBITTORRENT_URL']: + qBitConnectionStatus = await rest_get(settingsDict['QBITTORRENT_URL']+'/sync/maindata', cookies=settingsDict['QBIT_COOKIE'])['server_state']['connection_status'] + if qBitConnectionStatus == 'disconnected': + logger.warning('>>> qBittorrent is disconnected. Skipping %s queue cleaning failed on %s.',failType) + return 0 + for queueItem in queue['records']: if 'downloadId' in queueItem and 'size' in queueItem and 'sizeleft' in queueItem and 'status' in queueItem: if queueItem['downloadId'] not in alreadyCheckedDownloadIDs: From 3349acf39eab74d082f20551835c749f9dc4c0c4 Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Sun, 12 May 2024 15:39:29 +0200 Subject: [PATCH 04/15] Checking if qBit is connected prior to checking for slowness --- src/utils/loadScripts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/loadScripts.py b/src/utils/loadScripts.py index 18de686..a349d29 100644 --- a/src/utils/loadScripts.py +++ b/src/utils/loadScripts.py @@ -89,7 +89,7 @@ async def instanceChecks(settingsDict): logger.error('!! %s Error: !!', settingsDict[instance + '_NAME']) logger.error('Please update %s to at least version %s. Current version: %s', settingsDict[instance + 'MIN_VERSION'],current_version) if not error_occured: - logger.info('OK | %s', settingsDict[instance + '_NAME']) + logger.info('OK | %s', settingsDict[instance + '_NAME']) logger.debug('Current version of %s: %s', instance, current_version) # Check Bittorrent From 573d87c1dc282112f538c77037578a176676262c Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Sun, 12 May 2024 15:47:06 +0200 Subject: [PATCH 05/15] Small fix --- src/jobs/remove_slow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobs/remove_slow.py b/src/jobs/remove_slow.py index f66b103..1d51dfc 100644 --- a/src/jobs/remove_slow.py +++ b/src/jobs/remove_slow.py @@ -17,7 +17,7 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, if settingsDict['QBITTORRENT_URL']: qBitConnectionStatus = await rest_get(settingsDict['QBITTORRENT_URL']+'/sync/maindata', cookies=settingsDict['QBIT_COOKIE'])['server_state']['connection_status'] if qBitConnectionStatus == 'disconnected': - logger.warning('>>> qBittorrent is disconnected. Skipping %s queue cleaning failed on %s.',failType) + logger.warning('>>> qBittorrent is disconnected. Skipping %s queue cleaning failed on %s.',failType, NAME) return 0 for queueItem in queue['records']: From fcd6ea513b9e54229162b0af514f3dd123368f8d Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Mon, 13 May 2024 11:09:43 +0200 Subject: [PATCH 06/15] Skipping slow check for items that have nothing left to download, eventhough they are marked as "downloading" still May be the case for items that are being moved post download from one file system to another, before going to "seeding" stage --- src/jobs/remove_slow.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/jobs/remove_slow.py b/src/jobs/remove_slow.py index 9f26a03..c35dd90 100644 --- a/src/jobs/remove_slow.py +++ b/src/jobs/remove_slow.py @@ -17,6 +17,10 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, if 'downloadId' in queueItem and 'size' in queueItem and 'sizeleft' in queueItem and 'status' in queueItem: if queueItem['downloadId'] not in alreadyCheckedDownloadIDs: alreadyCheckedDownloadIDs.append(queueItem['downloadId']) # One downloadId may occur in multiple queueItems - only check once for all of them per iteration + if queueItem['sizeleft'] == 0: # Skip items that are finished downloading but are still marked as downloading. May be the case when files are moving + logger.debug('remove_slow/skipping completed item marked as downloading: %s (Speed: %d KB/s, KB now: %s, KB previous: %s, Diff: %s, In Minutes: %s', \ + queueItem['title'], speed, downloadedSize, previousSize, increment, settingsDict['REMOVE_TIMER']) + continue # determine if the downloaded bit on average between this and the last iteration is greater than the min threshold downloadedSize, previousSize, increment, speed = await getDownloadedSize(settingsDict, queueItem, download_sizes_tracker, NAME) if queueItem['status'] == 'downloading' and \ From 08327b76c6870f7ca75d92068154d41bebfe1d2f Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Mon, 13 May 2024 12:51:27 +0200 Subject: [PATCH 07/15] Excluding usenet downloads from slowness check --- README.md | 1 + src/jobs/remove_slow.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index b05daeb..c7c8095 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,7 @@ Steers which type of cleaning is applied to the downloads queue **REMOVE_SLOW** - Steers whether slow downloads are removed from the queue - Slow downloads are added to the blocklist, so that they are not re-requested in the future +- Note: Does not apply to usenet downloads (since there users pay for certain speed, slowness should not occurr) - Type: Boolean - Permissible Values: True, False - Is Mandatory: No (Defaults to False) diff --git a/src/jobs/remove_slow.py b/src/jobs/remove_slow.py index dedaf27..fda583a 100644 --- a/src/jobs/remove_slow.py +++ b/src/jobs/remove_slow.py @@ -24,6 +24,8 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, if 'downloadId' in queueItem and 'size' in queueItem and 'sizeleft' in queueItem and 'status' in queueItem: if queueItem['downloadId'] not in alreadyCheckedDownloadIDs: alreadyCheckedDownloadIDs.append(queueItem['downloadId']) # One downloadId may occur in multiple queueItems - only check once for all of them per iteration + if queueItem['usenet']: # No need to check for speed for usenet, since there users pay for speed + continue if queueItem['sizeleft'] == 0: # Skip items that are finished downloading but are still marked as downloading. May be the case when files are moving logger.debug('remove_slow/skipping completed item marked as downloading: %s (Speed: %d KB/s, KB now: %s, KB previous: %s, Diff: %s, In Minutes: %s', \ queueItem['title'], speed, downloadedSize, previousSize, increment, settingsDict['REMOVE_TIMER']) From 8841369fe9e538b03ef568d2ae749ba75e800af2 Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Mon, 13 May 2024 13:02:00 +0200 Subject: [PATCH 08/15] bugfix --- src/jobs/remove_slow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobs/remove_slow.py b/src/jobs/remove_slow.py index fda583a..43b3438 100644 --- a/src/jobs/remove_slow.py +++ b/src/jobs/remove_slow.py @@ -24,7 +24,7 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, if 'downloadId' in queueItem and 'size' in queueItem and 'sizeleft' in queueItem and 'status' in queueItem: if queueItem['downloadId'] not in alreadyCheckedDownloadIDs: alreadyCheckedDownloadIDs.append(queueItem['downloadId']) # One downloadId may occur in multiple queueItems - only check once for all of them per iteration - if queueItem['usenet']: # No need to check for speed for usenet, since there users pay for speed + if queueItem['protocol'] == 'usenet': # No need to check for speed for usenet, since there users pay for speed continue if queueItem['sizeleft'] == 0: # Skip items that are finished downloading but are still marked as downloading. May be the case when files are moving logger.debug('remove_slow/skipping completed item marked as downloading: %s (Speed: %d KB/s, KB now: %s, KB previous: %s, Diff: %s, In Minutes: %s', \ From cb9a386b9eddab625096dcdd70c5235808f1d808 Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Mon, 13 May 2024 16:55:22 +0200 Subject: [PATCH 09/15] Bug fix --- src/jobs/remove_slow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobs/remove_slow.py b/src/jobs/remove_slow.py index 1d51dfc..368af77 100644 --- a/src/jobs/remove_slow.py +++ b/src/jobs/remove_slow.py @@ -1,6 +1,7 @@ from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download) import sys, os, traceback import logging, verboselogs +from src.utils.rest import (rest_get) logger = verboselogs.VerboseLogger(__name__) async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, download_sizes_tracker): @@ -45,7 +46,6 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, errorDetails(NAME, error) return 0 -from src.utils.rest import (rest_get) async def getDownloadedSize(settingsDict, queueItem, download_sizes_tracker, NAME): try: # Determines the speed of download From 2f694722d1cfe98758a3f4b7802c415e0ad0f649 Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Mon, 13 May 2024 17:26:09 +0200 Subject: [PATCH 10/15] Bug fix --- src/jobs/remove_slow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobs/remove_slow.py b/src/jobs/remove_slow.py index 368af77..ae9b749 100644 --- a/src/jobs/remove_slow.py +++ b/src/jobs/remove_slow.py @@ -16,7 +16,7 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, alreadyCheckedDownloadIDs = [] if settingsDict['QBITTORRENT_URL']: - qBitConnectionStatus = await rest_get(settingsDict['QBITTORRENT_URL']+'/sync/maindata', cookies=settingsDict['QBIT_COOKIE'])['server_state']['connection_status'] + qBitConnectionStatus = (await rest_get(settingsDict['QBITTORRENT_URL']+'/sync/maindata', cookies=settingsDict['QBIT_COOKIE']))['server_state']['connection_status'] if qBitConnectionStatus == 'disconnected': logger.warning('>>> qBittorrent is disconnected. Skipping %s queue cleaning failed on %s.',failType, NAME) return 0 From 4e537429970ac7c8ec8c7f21e144c23a5ed43813 Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Mon, 13 May 2024 17:41:22 +0200 Subject: [PATCH 11/15] Give it a star on github.. --- src/utils/loadScripts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/utils/loadScripts.py b/src/utils/loadScripts.py index a349d29..bc802d5 100644 --- a/src/utils/loadScripts.py +++ b/src/utils/loadScripts.py @@ -34,6 +34,9 @@ def showSettings(settingsDict): logger.info('#' * 50) logger.info('Decluttarr - Application Started!') logger.info('') + logger.info('Like this app? Thanks for giving it a ⭐️ on GitHub!') + logger.info('https://github.com/ManiMatter/decluttarr/') + logger.info('') logger.info('*** Current Settings ***') logger.info('Version: %s', settingsDict['IMAGE_TAG']) logger.info('Commit: %s', settingsDict['SHORT_COMMIT_ID']) From 9f8d1960bcc5edc509395a04b380ec12ff892a39 Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Mon, 13 May 2024 17:55:25 +0200 Subject: [PATCH 12/15] Bug fix --- src/jobs/remove_slow.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/jobs/remove_slow.py b/src/jobs/remove_slow.py index 7cb6811..cbd7e2e 100644 --- a/src/jobs/remove_slow.py +++ b/src/jobs/remove_slow.py @@ -28,8 +28,7 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, if queueItem['protocol'] == 'usenet': # No need to check for speed for usenet, since there users pay for speed continue if queueItem['sizeleft'] == 0: # Skip items that are finished downloading but are still marked as downloading. May be the case when files are moving - logger.debug('remove_slow/skipping completed item marked as downloading: %s (Speed: %d KB/s, KB now: %s, KB previous: %s, Diff: %s, In Minutes: %s', \ - queueItem['title'], speed, downloadedSize, previousSize, increment, settingsDict['REMOVE_TIMER']) + logger.debug('remove_slow/skipping completed item marked as downloading (most likely moving): %s', queueItem['title']) continue # determine if the downloaded bit on average between this and the last iteration is greater than the min threshold downloadedSize, previousSize, increment, speed = await getDownloadedSize(settingsDict, queueItem, download_sizes_tracker, NAME) From d86646076686674452af551198da4e4eeae33ab5 Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Mon, 13 May 2024 20:40:27 +0200 Subject: [PATCH 13/15] bugfix --- src/jobs/remove_slow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobs/remove_slow.py b/src/jobs/remove_slow.py index cbd7e2e..c096f25 100644 --- a/src/jobs/remove_slow.py +++ b/src/jobs/remove_slow.py @@ -28,7 +28,7 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, if queueItem['protocol'] == 'usenet': # No need to check for speed for usenet, since there users pay for speed continue if queueItem['sizeleft'] == 0: # Skip items that are finished downloading but are still marked as downloading. May be the case when files are moving - logger.debug('remove_slow/skipping completed item marked as downloading (most likely moving): %s', queueItem['title']) + logger.info('>>> Detected %s download that has completed downloading - skipping from %s (likely moving): %s',failType, failType, queueItem['title']) continue # determine if the downloaded bit on average between this and the last iteration is greater than the min threshold downloadedSize, previousSize, increment, speed = await getDownloadedSize(settingsDict, queueItem, download_sizes_tracker, NAME) From 6958be1279ecb27c68263ce59c548d57b919f9bd Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Mon, 13 May 2024 22:20:25 +0200 Subject: [PATCH 14/15] reword log entry --- src/jobs/remove_slow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobs/remove_slow.py b/src/jobs/remove_slow.py index c096f25..57996cf 100644 --- a/src/jobs/remove_slow.py +++ b/src/jobs/remove_slow.py @@ -28,7 +28,7 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, if queueItem['protocol'] == 'usenet': # No need to check for speed for usenet, since there users pay for speed continue if queueItem['sizeleft'] == 0: # Skip items that are finished downloading but are still marked as downloading. May be the case when files are moving - logger.info('>>> Detected %s download that has completed downloading - skipping from %s (likely moving): %s',failType, failType, queueItem['title']) + logger.info('>>> Detected %s download that has completed downloading - skipping check (torrent files likely in process of being moved): %s',failType, queueItem['title']) continue # determine if the downloaded bit on average between this and the last iteration is greater than the min threshold downloadedSize, previousSize, increment, speed = await getDownloadedSize(settingsDict, queueItem, download_sizes_tracker, NAME) From 3b4b12de0fb9316f3765293f14f5e48893d3f8fe Mon Sep 17 00:00:00 2001 From: Benjamin Harder Date: Mon, 13 May 2024 23:19:49 +0200 Subject: [PATCH 15/15] bug fix --- src/jobs/remove_slow.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/jobs/remove_slow.py b/src/jobs/remove_slow.py index 57996cf..b511f55 100644 --- a/src/jobs/remove_slow.py +++ b/src/jobs/remove_slow.py @@ -27,18 +27,17 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, alreadyCheckedDownloadIDs.append(queueItem['downloadId']) # One downloadId may occur in multiple queueItems - only check once for all of them per iteration if queueItem['protocol'] == 'usenet': # No need to check for speed for usenet, since there users pay for speed continue - if queueItem['sizeleft'] == 0: # Skip items that are finished downloading but are still marked as downloading. May be the case when files are moving - logger.info('>>> Detected %s download that has completed downloading - skipping check (torrent files likely in process of being moved): %s',failType, queueItem['title']) - continue - # determine if the downloaded bit on average between this and the last iteration is greater than the min threshold - downloadedSize, previousSize, increment, speed = await getDownloadedSize(settingsDict, queueItem, download_sizes_tracker, NAME) - if queueItem['status'] == 'downloading' and \ - queueItem['downloadId'] in download_sizes_tracker.dict and \ - speed is not None: - if speed < settingsDict['MIN_DOWNLOAD_SPEED']: - affectedItems.append(queueItem) - logger.debug('remove_slow/slow speed detected: %s (Speed: %d KB/s, KB now: %s, KB previous: %s, Diff: %s, In Minutes: %s', \ - queueItem['title'], speed, downloadedSize, previousSize, increment, settingsDict['REMOVE_TIMER']) + if queueItem['status'] == 'downloading': + if queueItem['sizeleft'] == 0: # Skip items that are finished downloading but are still marked as downloading. May be the case when files are moving + logger.info('>>> Detected %s download that has completed downloading - skipping check (torrent files likely in process of being moved): %s',failType, queueItem['title']) + continue + # determine if the downloaded bit on average between this and the last iteration is greater than the min threshold + downloadedSize, previousSize, increment, speed = await getDownloadedSize(settingsDict, queueItem, download_sizes_tracker, NAME) + if queueItem['downloadId'] in download_sizes_tracker.dict and speed is not None: + if speed < settingsDict['MIN_DOWNLOAD_SPEED']: + affectedItems.append(queueItem) + logger.debug('remove_slow/slow speed detected: %s (Speed: %d KB/s, KB now: %s, KB previous: %s, Diff: %s, In Minutes: %s', \ + queueItem['title'], speed, downloadedSize, previousSize, increment, settingsDict['REMOVE_TIMER']) affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs,