# 2025.02.18 - [uploadhive] Add handling of the new /cgi-bin/dl.cgi/ url tickets (WIP)

#               (unfortunately, this is tied to the requesting ip, so downloads get "Wrong IP")
# 2025.02.18 - [up_oshi] Add Manage url as comment on uploads
# 2025.02.18 - [up_oshi / oshi] use /nossl/ url and http
# 2025.02.17 - [gofile] Add a random sleep if 429 response detected (too many requests)
# 2025.02.17 - [*ALL] Audit and update all single bracket operations
# 2025.02.17 - [filehaus] Fix downloading from fh
# 2025.02.15 - [uploadbay] Update urls regex for acceptable alternate
# 2025.02.15 - [up_sendnow] Add send.now as upload host
# 2025.02.15 - [sendnow] Fix handling of filenames with special characters in url
This commit is contained in:
kittykat 2025-02-19 13:41:07 +00:00
parent d48116dbe3
commit d62376f7a8
Signed by: kittykat
GPG key ID: E3F1556620F70C3C
103 changed files with 3541 additions and 3517 deletions

16
plugins/AutoResetAndRetryDownloads.sh Executable file → Normal file
View file

@ -1,6 +1,6 @@
#! Name: AutoResetAndRetryDownloads.sh
#! Author: kittykat
#! Version: 2024.09.27
#! Version: 2025.02.18
#! Desc: Runs mad.sh reset after processing all urls and then relaunches MAD
#! Usage: Edit LoadPlugin="" line in mad.sh or mad.config
#! LoadPlugin="AutoResetAndRetryDownloads.sh"
@ -30,25 +30,25 @@
DoneProcessingAllUrls_AutoResetAndRetryDownloads() {
local plugName='AutoResetAndRetryDownloads'
local plugFunc='DoneProcessingAllUrls_AutoResetAndRetryDownloads'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pInputFile="$1"
arard_ticketactivetimeout=60
arard_ticketexpirytimeout=540
if [ "$LoopThroughFileUntilComplete" == "true" ] && [ ! -z $CatnapDuration ] && ((CatnapDuration > 0)); then
if [[ "$LoopThroughFileUntilComplete" == "true" ]] && [[ ! -z $CatnapDuration ]] && ((CatnapDuration > 0)); then
arard_ticketactivetimeout=$(( arard_ticketactivetimeout + (CatnapDuration * 60) ))
arard_ticketexpirytimeout=$(( arard_ticketexpirytimeout + (CatnapDuration * 60) ))
fi
if grep -Eqi '^#(http.*#RETRY#|direct=http.*#RETRY#).*$' "$pInputFile" ; then
if [ ! -f "${WorkDir}/.temp/AutoResetAndRetryDownloadsarard_startTime.txt" ]; then
if [[ ! -f "${WorkDir}/.temp/AutoResetAndRetryDownloadsarard_startTime.txt" ]]; then
echo $(date +%s) > "${WorkDir}/.temp/AutoResetAndRetryDownloadsarard_startTime.txt"
else
arard_startTime=$(cat "${WorkDir}/.temp/AutoResetAndRetryDownloadsarard_startTime.txt")
arard_currTime=$(date +%s)
arard_durationseconds=$((arard_currTime - arard_startTime))
if ((arard_durationseconds > arard_ticketexpirytimeout)); then
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Ticket expiry occurred ($arard_durationseconds > $arard_ticketexpirytimeout)..${NC}"
fi
echo $(date +%s) > "${WorkDir}/.temp/AutoResetAndRetryDownloadsarard_startTime.txt"
@ -56,7 +56,7 @@ DoneProcessingAllUrls_AutoResetAndRetryDownloads() {
arard_currTime=$(date +%s)
arard_durationseconds=$((arard_currTime - arard_startTime))
elif ((arard_durationseconds > arard_ticketactivetimeout)); then
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Ticket inactive ($arard_durationseconds > $arard_ticketactivetimeout)..${NC}"
fi
for ((a=1; a<=2; a++)); do
@ -67,7 +67,7 @@ DoneProcessingAllUrls_AutoResetAndRetryDownloads() {
echo -e "${YELLOW}Reset&Retry${NC}: Done processing... ${BLUE}$((arard_durationseconds/60))${NC} min(s). Expiry ${PINK}$(( (arard_ticketexpirytimeout - arard_durationseconds)/60 ))${NC} min(s).${NC}"
return 0
fi
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Ticket active ($arard_durationseconds)..${NC}"
fi
for ((a=1; a<=2; a++)); do
@ -78,7 +78,7 @@ DoneProcessingAllUrls_AutoResetAndRetryDownloads() {
echo -e "${YELLOW}Reset&Retry${NC}: Dozing ${BLUE}1${NC} min. Processing for ${BLUE}$((arard_durationseconds/60))${NC} of ${BLUE}$((arard_ticketactivetimeout/60))${NC} min(s)${NC}. Expiry ${PINK}$(( (arard_ticketexpirytimeout - arard_durationseconds)/60 ))${NC} min(s).${NC}"
sleep 1m
fi
if [ ! -f "${WorkDir}/.temp/AutoResetAndRetryDownloadsCnt.lk" ]; then
if [[ ! -f "${WorkDir}/.temp/AutoResetAndRetryDownloadsCnt.lk" ]]; then
touch "${WorkDir}/.temp/AutoResetAndRetryDownloadsCnt.lk"
trap "rm -f ${WorkDir}/.temp/AutoResetAndRetryDownloadsCnt.lk; echo ""; tput cnorm; exit" 0 1 2 3 6 15
madReset "$pInputFile" "true" "false" "false" "true"

4
plugins/CatnapCtrlC.sh Executable file → Normal file
View file

@ -1,6 +1,6 @@
#! Name: CatnapCtrlC.sh
#! Author: kittykat
#! Version: 2024.09.21
#! Version: 2025.02.18
#! Desc: Keeps mad.sh running until Ctrl-C, waiting for urls.txt updates
#! Usage: Edit LoadPlugin="" line in mad.sh or mad.config
#! LoadPlugin="CatnapCtrlC.sh"
@ -34,7 +34,7 @@
DoneProcessingAllUrls_CatnapCtrlC() {
local plugName='CatnapCtrlC'
local plugFunc='DoneProcessingAllUrls_CatnapCtrlC'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pInputFile="$1"

22
plugins/SkipUrlsInDownloadsCompletedTxt.sh Executable file → Normal file
View file

@ -1,6 +1,6 @@
#! Name: SkipUrlsInDownloadsCompletedTxt.sh
#! Author: kittykat / beautfar
#! Version: 2024.11.06
#! Version: 2025.02.18
#! Desc: Skips urls that already exist in downloads_completed.txt
#! This is similar to the SkipOkUrlsInResultsTxt.sh script, except it is better --
#! - Creates "./data/downloads_completed.txt with only completed urls (less noise / clutter).
@ -37,7 +37,7 @@
PreProcessUrl_SkipUrlsInDownloadsCompletedTxt() {
local plugName='SkipUrlsInDownloadsCompletedTxt'
local plugFunc='PreProcessUrl_SkipUrlsInDownloadsCompletedTxt'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pFullLine="$1"
@ -48,25 +48,25 @@ PreProcessUrl_SkipUrlsInDownloadsCompletedTxt() {
if grep -Eqi '^direct=' <<< "${pUrlOnly}" ; then
pUrlOnly=${pUrlOnly/direct=/}
fi
if [ -z "$pFullLine" ]; then
if [[ -z "$pFullLine" ]]; then
return 0
fi
local tUrl="${pUrlOnly##*\:\/\/}"
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: pFullLine: $pFullLine${NC}"
echo -e "[${PINK}DEBUG${NC}]: pUrlOnly: $pUrlOnly${NC}"
echo -e "[${PINK}DEBUG${NC}]: tUrl: $tUrl${NC}"
fi
if [ ! -z "$tUrl" ]; then
if [[ ! -z "$tUrl" ]]; then
linematch=""
if [ -f "${WorkDir}/data/downloads_completed.txt" ]; then
if [[ -f "${WorkDir}/data/downloads_completed.txt" ]]; then
linematch=$(grep -Eni "[OK].*url:.*${tUrl}.*\$" "${WorkDir}/data/downloads_completed.txt")
else
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: File not found: ${BLUE}${WorkDir}/data/downloads_completed.txt${NC}"
fi
fi
if [ ! -z "$linematch" ] ; then
if [[ ! -z "$linematch" ]] ; then
echo -e ""
echo -e "${GREEN}$pUrlOnly${NC} has already been downloaded in ${GREEN}downloads_completed.txt${NC}"
echo -e "${BLUE}line${NC}:${NC}"
@ -75,12 +75,12 @@ PreProcessUrl_SkipUrlsInDownloadsCompletedTxt() {
return 1
fi
linematch=""
if [ -f "${WorkDir}/data/downloads_completed.txt" ]; then
if [[ -f "${WorkDir}/data/downloads_completed.txt" ]]; then
linematch=$(grep -Eni "[REMOVED].*url:.*${tUrl}.*\$" "${WorkDir}/data/downloads_completed.txt")
fi
if [ ! -z "$linematch" ] ; then
if [[ ! -z "$linematch" ]] ; then
echo -e ""
echo -e "${RED}$pUrlOnly${NC} already mareked removed in ${GREEN}downloads_completed.txt${NC}"
echo -e "${RED}$pUrlOnly${NC} already marked removed in ${GREEN}downloads_completed.txt${NC}"
echo -e "${BLUE}line${NC}: ${linematch//, /\\n}"
removedDownload "$pUrlOnly" "URL found in ./data/downloads_completed.txt (line#: ${linematch%%:*})"
return 1

View file

@ -1,6 +1,6 @@
#! Name: ExampleAddNewFuncAndCallOnSuccessfulDownload.sh
#! Author: kittykat
#! Version: 2024.09.21
#! Version: 2025.02.18
#! Desc: Add a new function to MAD and call it whenever a file is successfully downloaded
#! Usage: Edit LoadPlugin="" line in mad.sh or mad.config
#! LoadPlugin="ExampleAddNewFuncAndCallOnSuccessfulDownload.sh"
@ -40,7 +40,7 @@
PostSuccessfulDownload_mysha256() {
local plugName='ExampleAddNewFuncAndCallOnSuccessfulDownload'
local plugFunc='PostSuccessfulDownload_mysha256'
if [ "${DebugAllEnabled}" == "true" ]; then
if [[ "${DebugAllEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pRemoteUrl="$1"
@ -59,7 +59,7 @@ PostSuccessfulDownload_mysha256() {
hookHashFile() {
local plugName='ExampleAddNewFuncAndCallOnSuccessfulDownload'
local plugFunc='hookHashFile'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pFilePath="$1"

24
plugins/examples/ExamplesMainHooks.sh Executable file → Normal file
View file

@ -1,6 +1,6 @@
#! Name: ExamplesMainHooks.sh
#! Author: kittykat
#! Version: 2024.09.21
#! Version: 2025.02.18
#! Desc: Just some examples on easy plugins...
#! Usage: Edit LoadPlugin="" line in mad.sh or mad.config
#! LoadPlugin="ExamplesMainHooks.sh"
@ -33,7 +33,7 @@
OnLoad_ExamplesMainHooks() {
local plugName='ExamplesMainHooks'
local plugFunc='OnLoad_ExamplesMainHooks'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pScriptSourceDir="$1"
@ -43,7 +43,7 @@ OnLoad_ExamplesMainHooks() {
BeginProcessing_ExamplesMainHooks() {
local plugName='ExamplesMainHooks'
local plugFunc='BeginProcessing_ExamplesMainHooks'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pInputFile="$1"
@ -52,7 +52,7 @@ BeginProcessing_ExamplesMainHooks() {
PreProcessUrl_ExamplesMainHooks() {
local plugName='ExamplesMainHooks'
local plugFunc='PreProcessUrl_ExamplesMainHooks'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pFullLine="$1"
@ -69,7 +69,7 @@ PreProcessUrl_ExamplesMainHooks() {
PostSuccessfulDownload_ExamplesMainHooks() {
local plugName='ExamplesMainHooks'
local plugFunc='PostSuccessfulDownload_ExamplesMainHooks'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pRemoteUrl="$1"
@ -82,7 +82,7 @@ PostSuccessfulDownload_ExamplesMainHooks() {
PostFailedDownload_ExamplesMainHooks() {
local plugName='ExamplesMainHooks'
local plugFunc='PostFailedDownload_ExamplesMainHooks'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pRemoteUrl="$1"
@ -93,7 +93,7 @@ PostFailedDownload_ExamplesMainHooks() {
PostFailRetryDownload_ExamplesMainHooks() {
local plugName='ExamplesMainHooks'
local plugFunc='PostFailRetryDownload_ExamplesMainHooks'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pRemoteUrl="$1"
@ -104,7 +104,7 @@ PostFailRetryDownload_ExamplesMainHooks() {
DoneProcessingAllUrls_ExamplesMainHooks() {
local plugName='ExamplesMainHooks'
local plugFunc='DoneProcessingAllUrls_ExamplesMainHooks'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pInputFile="$1"
@ -118,7 +118,7 @@ DoneProcessingAllUrls_ExamplesMainHooks() {
PostSuccessfulUpload_ExamplesMainHooks() {
local plugName='ExamplesMainHooks'
local plugFunc='PostSuccessfulUpload_ExamplesMainHooks'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pFilePath="$1"
@ -131,7 +131,7 @@ PostSuccessfulUpload_ExamplesMainHooks() {
PostFailedUpload_ExamplesMainHooks() {
local plugName='ExamplesMainHooks'
local plugFunc='PostFailedUpload_ExamplesMainHooks'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pFilePath="$1"
@ -143,7 +143,7 @@ PostFailedUpload_ExamplesMainHooks() {
PostFailRetryUpload_ExamplesMainHooks() {
local plugName='ExamplesMainHooks'
local plugFunc='PostFailRetryUpload_ExamplesMainHooks'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pFilePath="$1"
@ -155,7 +155,7 @@ PostFailRetryUpload_ExamplesMainHooks() {
DoneProcessingAllUploads_ExamplesMainHooks() {
local plugName='ExamplesMainHooks'
local plugFunc='DoneProcessingAllUploads_ExamplesMainHooks'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
local pInputFile="$1"

View file

@ -1,6 +1,6 @@
#! Name: ocr_captcha.sh
#! Author: kittykat
#! Version: 2025.01.14
#! Version: 2025.02.18
#! Desc: Script to extract captcha from image using tesseract-ocr and imagemagick
#! Usage: Edit LoadPlugin="" line in mad.sh or mad.config
#! LoadPlugin="ocr_captcha.sh"
@ -38,13 +38,13 @@
CaptchaOcrImage() {
local plugName='ocr_captcha'
local plugFunc='CaptchaOcrImage'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
DEPENDENCIES=(convert)
for DEPENDENCY in ${DEPENDENCIES[@]} ; do
if [ -z $(which $DEPENDENCY) ] ; then
if [ "$DEPENDENCY" == "convert" ]; then
if [[ -z $(which $DEPENDENCY) ]] ; then
if [[ "$DEPENDENCY" == "convert" ]]; then
echo "imagemagick not installed. Aborting"
else
echo "$DEPENDENCY not installed. Aborting"
@ -56,7 +56,7 @@ CaptchaOcrImage() {
data_type="$2"
imagemagick_extra_params="$3"
local captcha_image_filename="${captcha_image_filepath##*/}"
if [ ! -f "$captcha_image_filepath" ]; then
if [[ ! -f "$captcha_image_filepath" ]]; then
echo -e "Image not found."
return 1
fi
@ -93,13 +93,13 @@ CaptchaOcrImage() {
CaptchaOcrImageTesseract() {
local plugName='ocr_captcha'
local plugFunc='CaptchaOcrImageTesseract'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
DEPENDENCIES=(tesseract convert)
for DEPENDENCY in ${DEPENDENCIES[@]} ; do
if [ -z $(which $DEPENDENCY) ] ; then
if [ "$DEPENDENCY" == "convert" ]; then
if [[ "$DEPENDENCY" == "convert" ]]; then
echo "imagemagick not installed. Aborting"
else
echo "$DEPENDENCY not installed. Aborting"
@ -113,7 +113,7 @@ CaptchaOcrImageTesseract() {
data_type="$2"
imagemagick_extra_params="$3"
local captcha_image_filename="${captcha_image_filepath##*/}"
if [ ! -f "$captcha_image_filepath" ]; then
if [[ ! -f "$captcha_image_filepath" ]]; then
echo -e "Image not found."
return 1
fi

10
plugins/pjscloud.sh Executable file → Normal file
View file

@ -1,6 +1,6 @@
#! Name: pjscloud.sh
#! Author: kittykat
#! Version: 2024.10.14
#! Version: 2025.02.18
#! Desc: Wrapper to use PhantomJSCloud to retrieve url response
#! Usage: Edit LoadPlugin="" line in mad.sh or mad.config
#! LoadPlugin="pjscloud.sh"
@ -27,21 +27,21 @@
pjscloud_tor_request() {
local plugName='pjscloud'
local plugFunc='pjscloud_tor_request'
if [ "${DebugPluginsEnabled}" == "true" ]; then
if [[ "${DebugPluginsEnabled}" == "true" ]]; then
echo -e "[${PINK}DEBUG${NC}]: Running ${PINK}$plugFunc${NC} in ${BLUE}$plugName${NC} ...${NC}"
fi
pjs_targeturl="$1"
pjs_data="$2"
if [ -z "$pjs_targeturl" ]; then
if [[ -z "$pjs_targeturl" ]]; then
echo "Aborting: No target url specified."
fi
GetRandomPjsKey
if [ ! -z "$pjs_targeturl" ] && [ ! -z "$pjs_data" ] ; then
if [[ ! -z "$pjs_targeturl" ]] && [[ ! -z "$pjs_data" ]] ; then
pjs_urldata='{"url":"'"$pjs_targeturl"'", urlSettings:{"operation":"POST", "data":"'"$pjs_data"'"}, "renderType":"plainText"}'
else
pjs_urldata='{"url":"'"$pjs_targeturl"'", "renderType":"plainText"}'
fi
if [ "${UseTorCurlImpersonate}" == "true" ]; then
if [[ "${UseTorCurlImpersonate}" == "true" ]]; then
response=$("${curl_impersonate[@]}" --proxy "socks5h://${tor_identity}@${TorIp}:${torPort}" \
--connect-timeout ${ConnectTimeout} \
--insecure -k -s \