90 lines
4.4 KiB
Bash
Executable file
90 lines
4.4 KiB
Bash
Executable file
#! Name: SkipUrlsInDownloadsCompletedTxt.sh
|
|
#! Author: kittykat / beautfar
|
|
#! Version: 2024.11.06
|
|
#! Desc: Skips urls that already exist in downloads_completed.txt
|
|
#! This is similar to the SkipOkUrlsInResultsTxt.sh script, except it is better --
|
|
#! - Creates "./data/downloads_completed.txt with only completed urls (less noise / clutter).
|
|
#! - Stores more detail including the date
|
|
#! Usage: Edit LoadPlugin="" line in mad.sh or mad.config
|
|
#! LoadPlugin="SkipUrlsInDownloadsCompletedTxt.sh"
|
|
#!
|
|
#! Notes:
|
|
#! * Return 0 (true), continue processing this url
|
|
#! * Return 1 (false), stop processing this url and move to next
|
|
#!
|
|
#! Plugin System: Additional Hook Functions added for easy of overriding:
|
|
#!
|
|
#! To Hook a Function, your function must begin with the hookable function name, and be unique
|
|
#! Best practice: HooKFuncName_<yourfilename>
|
|
#! ie.
|
|
#! PostSuccessfulDownload_myfilename() {
|
|
#! }
|
|
#!
|
|
#! Available Hook Functions:
|
|
#! -------------------------
|
|
#! * OnLoad(): Occurs after load mad.config / load plugins (prior to processing).
|
|
#! * BeginProcessing(): Occurs immediately after beginning processin of urls.txt (loops with Catnaps).
|
|
#! * PreProcessUrl(): occurs immediately after reading in an unprocessed url (^http) line to process.
|
|
#! * PostSuccessfulDownload(): occurs after a download success (is marked #OK# in the urls.txt).
|
|
#! * PostFailedDownload(): occurs after a download fails (is marked #FAIL# in the urls.txt).
|
|
#! * PostFailRetryDownload(): occurs after a download fails with a retry (is marked #RETRY# in the urls.txt).
|
|
#! * DoneProcessingAllUrls(): occurs after all the urls have finished processing (no flocks or other terms downloading).
|
|
#! * PostSuccessfulUpload(): occurs after an upload success (after upload completed ticket is created in ./downloads/).
|
|
#! * PostFailedUpload(): occurs after an upload fails definitively -- #FAIL# in the temp_upload_handler.txt
|
|
#! * PostFailRetryUpload(): occurs after an upload fails with a retry (network drop, unexpected result)
|
|
#! * DoneProcessingAllUploads: occurs after alll the files have finished processing
|
|
#!
|
|
PreProcessUrl_SkipUrlsInDownloadsCompletedTxt() {
|
|
local plugName='SkipUrlsInDownloadsCompletedTxt'
|
|
local plugFunc='PreProcessUrl_SkipUrlsInDownloadsCompletedTxt'
|
|
if [ "${DebugPluginsEnabled}" == "true" ]; then
|
|
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
|
|
fi
|
|
local pFullLine="$1"
|
|
local pUrlOnly="${1%%\ *}"
|
|
if grep -Eqi '|' <<< "${pUrlOnly}" ; then
|
|
pUrlOnly="${pUrlOnly%%\|*}"
|
|
fi
|
|
if grep -Eqi '^direct=' <<< "${pUrlOnly}" ; then
|
|
pUrlOnly=${pUrlOnly/direct=/}
|
|
fi
|
|
if [ -z "$pFullLine" ]; then
|
|
return 0
|
|
fi
|
|
local tUrl="${pUrlOnly##*\:\/\/}"
|
|
if [ "${DebugPluginsEnabled}" == "true" ]; then
|
|
echo -e "[${PINK}DEBUG${NC}]: pFullLine: $pFullLine${NC}"
|
|
echo -e "[${PINK}DEBUG${NC}]: pUrlOnly: $pUrlOnly${NC}"
|
|
echo -e "[${PINK}DEBUG${NC}]: tUrl: $tUrl${NC}"
|
|
fi
|
|
if [ ! -z "$tUrl" ]; then
|
|
linematch=""
|
|
if [ -f "${WorkDir}/data/downloads_completed.txt" ]; then
|
|
linematch=$(grep -Eni "[OK].*url:.*${tUrl}.*\$" "${WorkDir}/data/downloads_completed.txt")
|
|
else
|
|
if [ "${DebugPluginsEnabled}" == "true" ]; then
|
|
echo -e "[${PINK}DEBUG${NC}]: File not found: ${BLUE}${WorkDir}/data/downloads_completed.txt${NC}"
|
|
fi
|
|
fi
|
|
if [ ! -z "$linematch" ] ; then
|
|
echo -e ""
|
|
echo -e "${GREEN}$pUrlOnly${NC} has already been downloaded in ${GREEN}downloads_completed.txt${NC}"
|
|
echo -e "${BLUE}line${NC}:${NC}"
|
|
echo -e "${linematch//, /\\n}"
|
|
successDownloadExists "$pUrlOnly" "URL found in ./data/downloads_completed.txt (line#: ${linematch%%:*})"
|
|
return 1
|
|
fi
|
|
linematch=""
|
|
if [ -f "${WorkDir}/data/downloads_completed.txt" ]; then
|
|
linematch=$(grep -Eni "[REMOVED].*url:.*${tUrl}.*\$" "${WorkDir}/data/downloads_completed.txt")
|
|
fi
|
|
if [ ! -z "$linematch" ] ; then
|
|
echo -e ""
|
|
echo -e "${RED}$pUrlOnly${NC} already mareked removed in ${GREEN}downloads_completed.txt${NC}"
|
|
echo -e "${BLUE}line${NC}: ${linematch//, /\\n}"
|
|
removedDownload "$pUrlOnly" "URL found in ./data/downloads_completed.txt (line#: ${linematch%%:*})"
|
|
return 1
|
|
fi
|
|
fi
|
|
return 0
|
|
}
|