#!    Name: SkipUrlsInDownloadsCompletedTxt.sh
#!  Author: kittykat / beautfar
#! Version: 2024.11.06
#!    Desc: Skips urls that already exist in downloads_completed.txt
#!          This is similar to the SkipOkUrlsInResultsTxt.sh script, except it is better --
#!          - Creates "./data/downloads_completed.txt with only completed urls (less noise / clutter).
#!          - Stores more detail including the date
#!   Usage: Edit LoadPlugin="" line in mad.sh or mad.config
#!          LoadPlugin="SkipUrlsInDownloadsCompletedTxt.sh"
#!
#! Notes:
#!   * Return 0 (true), continue processing this url
#!   * Return 1 (false), stop processing this url and move to next
#!
#! Plugin System: Additional Hook Functions added for easy of overriding:
#!
#! To Hook a Function, your function must begin with the hookable function name, and be unique
#! Best practice: HooKFuncName_<yourfilename>
#! ie. 
#!   PostSuccessfulDownload_myfilename() {
#!   }
#!
#! Available Hook Functions:
#! -------------------------
#!  * OnLoad(): Occurs after load mad.config / load plugins (prior to processing).
#!  * BeginProcessing(): Occurs immediately after beginning processin of urls.txt (loops with Catnaps).
#!  * PreProcessUrl(): occurs immediately after reading in an unprocessed url (^http) line to process.
#!  * PostSuccessfulDownload(): occurs after a download success (is marked #OK# in the urls.txt).
#!  * PostFailedDownload(): occurs after a download fails (is marked #FAIL# in the urls.txt).
#!  * PostFailRetryDownload(): occurs after a download fails with a retry (is marked #RETRY# in the urls.txt).
#!  * DoneProcessingAllUrls(): occurs after all the urls have finished processing (no flocks or other terms downloading).
#!  * PostSuccessfulUpload(): occurs after an upload success (after upload completed ticket is created in ./downloads/).
#!  * PostFailedUpload(): occurs after an upload fails definitively -- #FAIL# in the temp_upload_handler.txt
#!  * PostFailRetryUpload(): occurs after an upload fails with a retry (network drop, unexpected result)
#!  * DoneProcessingAllUploads: occurs after alll the files have finished processing
#!
PreProcessUrl_SkipUrlsInDownloadsCompletedTxt() {
    local plugName='SkipUrlsInDownloadsCompletedTxt'
    local plugFunc='PreProcessUrl_SkipUrlsInDownloadsCompletedTxt'
    if [ "${DebugPluginsEnabled}" == "true" ]; then
        echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
    fi
    local pFullLine="$1"
    local pUrlOnly="${1%%\ *}"
    if grep -Eqi '|' <<< "${pUrlOnly}" ; then
        pUrlOnly="${pUrlOnly%%\|*}"
    fi
    if grep -Eqi '^direct=' <<< "${pUrlOnly}" ; then
        pUrlOnly=${pUrlOnly/direct=/}
    fi
    if [ -z "$pFullLine" ]; then
        return 0
    fi
    local tUrl="${pUrlOnly##*\:\/\/}"
    if [ "${DebugPluginsEnabled}" == "true" ]; then
        echo -e "[${PINK}DEBUG${NC}]: pFullLine: $pFullLine${NC}"
        echo -e "[${PINK}DEBUG${NC}]: pUrlOnly: $pUrlOnly${NC}"
        echo -e "[${PINK}DEBUG${NC}]: tUrl: $tUrl${NC}"
    fi
    if [ ! -z "$tUrl" ]; then
        linematch=""
        if [ -f "${WorkDir}/data/downloads_completed.txt" ]; then
            linematch=$(grep -Eni "[OK].*url:.*${tUrl}.*\$" "${WorkDir}/data/downloads_completed.txt")
        else
            if [ "${DebugPluginsEnabled}" == "true" ]; then
                echo -e "[${PINK}DEBUG${NC}]: File not found: ${BLUE}${WorkDir}/data/downloads_completed.txt${NC}"
            fi
        fi
        if [ ! -z "$linematch" ] ; then
            echo -e ""
            echo -e "${GREEN}$pUrlOnly${NC} has already been downloaded in ${GREEN}downloads_completed.txt${NC}"
            echo -e "${BLUE}line${NC}:${NC}"
            echo -e "${linematch//, /\\n}"
		    successDownloadExists "$pUrlOnly" "URL found in ./data/downloads_completed.txt (line#: ${linematch%%:*})"
		    return 1
	    fi
        linematch=""
        if [ -f "${WorkDir}/data/downloads_completed.txt" ]; then
            linematch=$(grep -Eni "[REMOVED].*url:.*${tUrl}.*\$" "${WorkDir}/data/downloads_completed.txt")
        fi
        if [ ! -z "$linematch" ] ; then
            echo -e ""
            echo -e "${RED}$pUrlOnly${NC} already mareked removed in ${GREEN}downloads_completed.txt${NC}"
            echo -e "${BLUE}line${NC}: ${linematch//, /\\n}"
		    removedDownload "$pUrlOnly" "URL found in ./data/downloads_completed.txt (line#: ${linematch%%:*})"
		    return 1
	    fi
	fi
	return 0
}