Merge pull request #108 from ManiMatter/dev

Refinements

README.md

@@ -184,6 +184,7 @@ Steers which type of cleaning is applied to the downloads queue
 **REMOVE_SLOW**
 - Steers whether slow downloads are removed from the queue
 - Slow downloads are added to the blocklist, so that they are not re-requested in the future
+- Note: Does not apply to usenet downloads (since there users pay for certain speed, slowness should not occurr)
 - Type: Boolean
 - Permissible Values: True, False
 - Is Mandatory: No (Defaults to False)

docker/Dockerfile

@@ -18,8 +18,8 @@ ENV IS_IN_DOCKER 1
 
 WORKDIR /app
 
-COPY . .
+COPY ./docker/requirements.txt ./docker/requirements.txt
 RUN pip install --no-cache-dir -r docker/requirements.txt
-
+COPY . .
 
 CMD ["python", "main.py"]

src/jobs/remove_missing_files.py

@@ -13,11 +13,19 @@ async def remove_missing_files(settingsDict, BASE_URL, API_KEY, NAME, deleted_do
         # Find items affected
         affectedItems = []
         for queueItem in queue['records']: 
-            if 'errorMessage' in queueItem and 'status' in queueItem:
+            if 'status' in queueItem:
-                if (queueItem['status']        == 'warning' and 
+                # case to check for failed torrents
-                    (queueItem['errorMessage']  == 'DownloadClientQbittorrentTorrentStateMissingFiles' or
+                if (queueItem['status'] == 'warning' and 'errorMessage' in queueItem and 
+                    (queueItem['errorMessage'] == 'DownloadClientQbittorrentTorrentStateMissingFiles' or
                     queueItem['errorMessage'] == 'The download is missing files')):
                     affectedItems.append(queueItem)
+                # case to check for failed nzb's/bad files/empty directory
+                if queueItem['status'] == 'completed' and 'statusMessages' in queueItem:
+                    for statusMessage in queueItem['statusMessages']:
+                        if 'messages' in statusMessage:
+                            for message in statusMessage['messages']:
+                                if message.startswith("No files found are eligible for import in"):
+                                    affectedItems.append(queueItem)
         affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, 
                                             addToBlocklist = False, 
                                             doPrivateTrackerCheck = True, 

src/jobs/remove_slow.py

@@ -1,6 +1,7 @@
 from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download)
 import sys, os, traceback
 import logging, verboselogs
+from src.utils.rest import (rest_get)
 logger = verboselogs.VerboseLogger(__name__)
 
 async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, protectedDownloadIDs, privateDowloadIDs, download_sizes_tracker):
@@ -13,19 +14,30 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads,
         # Find items affected
         affectedItems = []
         alreadyCheckedDownloadIDs = []
+
+        if settingsDict['QBITTORRENT_URL']:
+            qBitConnectionStatus = (await rest_get(settingsDict['QBITTORRENT_URL']+'/sync/maindata', cookies=settingsDict['QBIT_COOKIE']))['server_state']['connection_status']
+            if qBitConnectionStatus == 'disconnected':
+                logger.warning('>>> qBittorrent is disconnected. Skipping %s queue cleaning failed on %s.',failType, NAME)
+                return 0
+
         for queueItem in queue['records']:
             if 'downloadId' in queueItem and 'size' in queueItem and 'sizeleft' in queueItem and 'status' in queueItem:
                 if queueItem['downloadId'] not in alreadyCheckedDownloadIDs:
                     alreadyCheckedDownloadIDs.append(queueItem['downloadId']) # One downloadId may occur in multiple queueItems - only check once for all of them per iteration
-                    # determine if the downloaded bit on average between this and the last iteration is greater than the min threshold
+                    if queueItem['protocol'] == 'usenet': # No need to check for speed for usenet, since there users pay for speed
-                    downloadedSize, previousSize, increment, speed = await getDownloadedSize(settingsDict, queueItem, download_sizes_tracker, NAME)
+                        continue
-                    if  queueItem['status'] == 'downloading' and \
+                    if queueItem['status'] == 'downloading':
-                        queueItem['downloadId'] in download_sizes_tracker.dict and \
+                        if queueItem['sizeleft'] == 0: # Skip items that are finished downloading but are still marked as downloading. May be the case when files are moving
-                        speed is not None:
+                            logger.info('>>> Detected %s download that has completed downloading - skipping check (torrent files likely in process of being moved): %s',failType, queueItem['title'])    
-                        if speed < settingsDict['MIN_DOWNLOAD_SPEED']:
+                            continue
-                            affectedItems.append(queueItem)
+                        # determine if the downloaded bit on average between this and the last iteration is greater than the min threshold
-                            logger.debug('remove_slow/slow speed detected: %s (Speed: %d KB/s, KB now: %s, KB previous: %s, Diff: %s, In Minutes: %s', \
+                        downloadedSize, previousSize, increment, speed = await getDownloadedSize(settingsDict, queueItem, download_sizes_tracker, NAME)
-                                queueItem['title'], speed, downloadedSize, previousSize, increment, settingsDict['REMOVE_TIMER'])
+                        if queueItem['downloadId'] in download_sizes_tracker.dict and speed is not None:
+                            if speed < settingsDict['MIN_DOWNLOAD_SPEED']:
+                                affectedItems.append(queueItem)
+                                logger.debug('remove_slow/slow speed detected: %s (Speed: %d KB/s, KB now: %s, KB previous: %s, Diff: %s, In Minutes: %s', \
+                                    queueItem['title'], speed, downloadedSize, previousSize, increment, settingsDict['REMOVE_TIMER'])
 
 
         affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, 
@@ -38,7 +50,6 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads,
         errorDetails(NAME, error)
         return 0
 
-from src.utils.rest import (rest_get)
 async def getDownloadedSize(settingsDict, queueItem, download_sizes_tracker, NAME):
     try:
         # Determines the speed of download

src/utils/loadScripts.py

@@ -34,6 +34,9 @@ def showSettings(settingsDict):
     logger.info('#' * 50)
     logger.info('Decluttarr - Application Started!')
     logger.info('')      
+    logger.info('Like this app? Thanks for giving it a ⭐️ on GitHub!')      
+    logger.info('https://github.com/ManiMatter/decluttarr/')   
+    logger.info('')      
     logger.info('*** Current Settings ***') 
     logger.info('Version: %s', settingsDict['IMAGE_TAG']) 
     logger.info('Commit: %s', settingsDict['SHORT_COMMIT_ID'])    
@@ -89,7 +92,7 @@ async def instanceChecks(settingsDict):
                         logger.error('!! %s Error: !!', settingsDict[instance + '_NAME'])
                         logger.error('Please update %s to at least version %s. Current version: %s',  settingsDict[instance + 'MIN_VERSION'],current_version)
             if not error_occured:
-                logger.info('OK  | %s', settingsDict[instance + '_NAME'])     
+                logger.info('OK | %s', settingsDict[instance + '_NAME'])     
                 logger.debug('Current version of %s: %s', instance, current_version)  
 
     # Check Bittorrent