diff --git a/hosts/torup.sh b/hosts/torup.sh new file mode 100644 index 0000000..7c2d423 --- /dev/null +++ b/hosts/torup.sh @@ -0,0 +1,324 @@ +#! Name: torup.sh +#! Author: kittykat +#! Version: 2024.11.09 +#! Desc: Add support for downloading and processing of urls for a new host +#! Usage: Copy this file into the ./${ScriptDir}/hosts/ folder +#! +#! +#! ------------ REQUIRED SECTION --------------- +#! @[UPDATE] HostAndDomainRegexes: This string is loaded into mad.sh and allows dynamic handling of new url data +#! Format: '/HostCode/HostNick/HostFuncPrefix:HostDomainRegex@' +#! HostCode: (ie. 'fh' for filehaus -- cannot be used by other hosts) +#! HostNick: What is displayed throughout MAD output (ie. 'filehaus' -- "urls.txt has 10 filehaus.." will be displayed) +#! HostFuncPrefix: (ie. 'fh' -- fh_DownloadFile(), fh_FetchFileInfo() .. ) +#! * Note: Must begin with a letter a-z (functions beginning with numbers are no bueno) +#! HostDomainRegex: The regex used to verify matching urls +HostCode='torp' +HostNick='torup' +HostFuncPrefix='torp' +HostUrls='ktgzpea2b76u7fgemiibp4a76onyybo4fw5gbsagtm6jrjzmgivppyyd.onion' +HostDomainRegex='^(http|https)://ktgzpea2b76u7fgemiibp4a76onyybo4fw5gbsagtm6jrjzmgivppyyd\.onion/' +#! +#! !! DO NOT UPDATE OR REMOVE !! +#! This merges the Required HostAndDomainRegexes into mad.sh +ListHostAndDomainRegexes=${ListHostAndDomainRegexes}'/'${HostCode}'/'${HostNick}'/'${HostFuncPrefix}'/'${HostUrls}':'${HostDomainRegex}'@' +#! +#! +#! ------------ (1) Host Main Download Function --------------- # +#! +#! This is a direct= download host, so all the functions are already in mad.sh +#! Since the HostFuncPrefix is defined above as "direct", nothing further needs to be done as it will +#! call the direct_DownloadFile() function already in mad.sh +torp_DownloadFile() { + local remote_url=${1} + local file_url=${1} + local filecnt=${2} + warnAndRetryUnknownError=false + exitDownloadError=false + exitDownloadNotAvailable=false + fileAlreadyDone=false + download_inflight_path="${WorkDir}/.inflight/" + mkdir -p "$download_inflight_path" + completed_location="${WorkDir}/downloads/" + tor_identity="${RANDOM}" + finalAttempt="false" + for ((z=0; z<=$MaxUrlRetries; z++)); do + if [ $z -eq $MaxUrlRetries ] ; then + finalAttempt="true" + fi + CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} + trap "rm -f ${WorkDir}/.flocks/${CLEANSTRING}; echo ""; tput cnorm; exit" 0 1 2 3 6 15 + if torp_FetchFileInfo $finalAttempt && torp_GetFile "${filecnt}" $((z+1)) $finalAttempt ; then + return 0 + elif [ $z -lt $MaxUrlRetries ]; then + if [ "${fileAlreadyDone}" == "true" ] ; then + break + fi + if [[ "${warnAndRetryUnknownError}" == "true" ]] ; then + if [ "${DebugAllEnabled}" == "true" ] ; then + debugHtml "${remote_url##*/}" "error" "Retry due to an unknown issue: attempt #$((z+1)) of ${MaxUrlRetries}" + fi + fi + if [[ "${exitDownloadError}" == "true" || "${exitDownloadNotAvailable}" == "true" ]] ; then + if [ "${DebugAllEnabled}" == "true" ] ; then + debugHtml "${remote_url##*/}" "error" "Exit due to unrecoverable issue" + fi + rm -f "${WorkDir}/.flocks/${remote_url//[^a-zA-Z0-9]/}" + break + fi + echo -e "\n${YELLOW}A recoverable error occurred, retry attempt $((z+1))/${MaxUrlRetries}${NC}" + sleep 3 + fi + done + rm -f "${WorkDir}/.flocks/${remote_url//[^a-zA-Z0-9]/}" +} +#! +#! ------------- (2) Fetch File Info Function ----------------- # +#! +torp_FetchFileInfo() { + finalAttempt=$1 + maxfetchretries=5 + fixed_url="${remote_url/https:/http:}" + fixed_url="${fixed_url//\/file}" + echo -e "${GREEN}# Fetching download url…${NC}" + for ((i=1; i<=$maxfetchretries; i++)); do + mkdir -p "${WorkDir}/.temp" + rm -f "${torp_cookie_jar}"; + torp_cookie_jar=$(mktemp "${WorkDir}/.temp/torp_cookies""${instance_no}"".XXXXXX") + printf " ." + tor_identity="${RANDOM}" + CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} + trap "rm -f ${WorkDir}/.flocks/${CLEANSTRING}; rm -f "${torp_cookie_jar}"; echo ""; tput cnorm; exit" 0 1 2 3 6 15 + response=$(tor_curl_request --insecure -L -s \ + -c "${fdot_cookie_jar}" \ + "$fixed_url") + if [ "${DebugAllEnabled}" == "true" ] ; then + debugHtml "${remote_url##*/}" "torp_fetch$i" "${response}" + fi + if [[ -z $response ]] ; then + if [ $i == $maxfetchretries ] ; then + printf "\\n" + echo -e "${RED}| Failed to extract download url [1]${NC}" + warnAndRetryUnknownError=true + if [ "${finalAttempt}" == "true" ] ; then + failedRetryDownload "${remote_url}" "Failed to extract download url [1]" "" + fi + return 1 + else + continue + fi + fi + if grep -Eqi "There is no such file|File was deleted because" <<< "$response"; then + printf "\\n" + echo -e "${RED}| The file was not found. It could be deleted or expired.${NC}" + exitDownloadError=true + removedDownload "${remote_url}" + return 1 + fi + if [ "$filename_override" == "" ] ; then + filename=$(grep -oP '(?<=h3 class\="h5 text-white mb-3">).*?(?=.*$)' <<< "$response") + fi + file_size_bytes=$(grep -oP '(?<=\(Raw: ).*?(?=\).*$)' <<< "$response") + file_size_bytes=${file_size_bytes//[$'\t\r\n']} + if grep -Eqi '/file" class\="btn btn-primary btn-lg">Download Now' <<< "$response"; then + printf "\\n" + echo -e "${GREEN}| Download url found${NC}" + download_url="${fixed_url}/file" + break + else + if [ $i == $maxfetchretries ] ; then + printf "\\n" + echo -e "${RED}| Failed to extract download url [2]${NC}" + warnAndRetryUnknownError=true + if [ "${finalAttempt}" == "true" ] ; then + failedRetryDownload "${remote_url}" "Failed to extract download url [2]" "" + fi + return 1 + else + continue + fi + fi + break #Good to go here + done + touch ${WorkDir}/.flocks/${remote_url//[^a-zA-Z0-9]/} + if [ ! "$filename_override" == "" ] ; then + filename="$filename_override" + fi + filename=$(sanitize_file_or_folder_name "${filename}") + printf "\\n" + echo -e "${YELLOW}| File name:${NC}\t\"${filename}\"" + if [ -z $file_size_bytes ] ; then + if [ "${finalAttempt}" == "true" ] ; then + failedRetryDownload "${remote_url}" "Filesize not found!" "" + fi + echo -e "${YELLOW}| Filesize not found… retry${NC}" + return 1 + else + file_size_readable="$(numfmt --to=iec --from=auto --format "%.2f" <<< "$file_size_bytes")" + fi + echo -e "${YELLOW}| File size:${NC}\t${file_size_readable}" + file_path="${download_inflight_path}${filename}" + flockDownload="${WorkDir}/.flocks/${filename//[^a-zA-Z0-9\.\_\-]/}.flock" + if CheckFileSize "${remote_url}" "${file_size_bytes}" ; then + return 1 + fi + if CheckDownloadExists "$remote_url" "$MoveToFolder" "$filecnt" "$filename" "$file_path" "$completed_location" ; then + return 1 + fi + echo "${remote_url//[^a-zA-Z0-9]/}" > $flockDownload +} +#! +#! ----------- (3) Fetch File / Download File Function --------------- # +#! +torp_GetFile() { + echo -e "${GREEN}# Downloading…" + echo -e "${YELLOW}| File path:${NC}\t./.inflight/${filename}\n" + fileCnt=$1 + retryCnt=$2 + finalAttempt=$3 + flockDownload="${WorkDir}/.flocks/${filename//[^a-zA-Z0-9\.\_\-]/}.flock" + for ((j=1; j<=$MaxDownloadRetries; j++)); do + pd_presize=0 + if [ -f "$file_path" ] ; then + pd_presize=$(stat --format="%s" "$file_path" | tr -d '[:space:]') + fi + GetRandomUA + CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} + trap "rm -f ${WorkDir}/.flocks/${CLEANSTRING}; rm -f ${torp_cookie_jar}; rm -f $flockDownload; echo ""; tput cnorm; exit" 0 1 2 3 6 15 + if [ "${UseTorCurlImpersonate}" == "true" ]; then + if [ "${RateMonitorEnabled}" == "true" ]; then + tor_curl_request --insecure -L -G --no-alpn \ + --speed-limit $DownloadSpeedMin --speed-time $DownloadTimeoutInterval \ + -b "${torp_cookie_jar}" -c "${torp_cookie_jar}" \ + -H "Referer: $fixed_url" \ + "$download_url" --continue-at - --output "$file_path" + else + tor_curl_request --insecure -L -G --no-alpn \ + -b "${torp_cookie_jar}" -c "${torp_cookie_jar}" \ + -H "Referer: $fixed_url" \ + "$download_url" --continue-at - --output "$file_path" + fi + else + if [ "${RateMonitorEnabled}" == "true" ]; then + tor_curl_request --insecure -L -G --no-alpn \ + --speed-limit $DownloadSpeedMin --speed-time $DownloadTimeoutInterval \ + -H "User-Agent: $RandomUA" \ + -H "Referer: $fixed_url" \ + -H "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8" \ + -H "Accept-Language: en-US,en;q=0.5" \ + -H "Accept-Encoding: gzip, deflate, br" \ + -H "Connection: keep-alive" \ + -H "Upgrade-Insecure-Requests: 1" \ + -H "Sec-Fetch-Dest: document" \ + -H "Sec-Fetch-Mode: navigate" \ + -H "Sec-Fetch-Site: same-origin" \ + -H "Sec-Fetch-User: ?1" \ + -b "${torp_cookie_jar}" -c "${torp_cookie_jar}" \ + "$download_url" --continue-at - --output "$file_path" + else + tor_curl_request --insecure -L -G --no-alpn \ + -H "User-Agent: $RandomUA" \ + -H "Referer: $fixed_url" \ + -H "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8" \ + -H "Accept-Language: en-US,en;q=0.5" \ + -H "Accept-Encoding: gzip, deflate, br" \ + -H "Connection: keep-alive" \ + -H "Upgrade-Insecure-Requests: 1" \ + -H "Sec-Fetch-Dest: document" \ + -H "Sec-Fetch-Mode: navigate" \ + -H "Sec-Fetch-Site: same-origin" \ + -H "Sec-Fetch-User: ?1" \ + -b "${torp_cookie_jar}" -c "${torp_cookie_jar}" \ + "$download_url" --continue-at - --output "$file_path" + fi + fi + received_file_size=0 + if [ -f "$file_path" ] ; then + received_file_size=$(stat --format="%s" "$file_path" | tr -d '[:space:]') + fi + if CheckNoHtml "$remote_url" "$filename" "$file_path" ; then + containsHtml=false + else + containsHtml=true + fi + downDelta=$(( received_file_size - pd_presize )) + if [[ "${received_file_size}" -ne "${file_size_bytes}" ]] || [ "$containsHtml" == "true" ]; then + if [ "${AutoRepairBadPartials}" == "true" ] && (( downDelta > 0 && downDelta < 1024 )) ; then + if [ -f "${file_path}" ] ; then + if ((pd_presize > 0)); then + echo -e "${YELLOW}Bad node / HTML found:${NC} reverting to previous file..." + truncateDownload "$remote_url" "$filename" "$pd_presize" "$received_file_size" + truncate -s $pd_presize "${file_path}" + else + echo -e "${YELLOW}Bad node / HTML found:${NC} tainted partial removed..." + rm -f "${file_path}" + fi + fi + if ((j >= $MaxDownloadRetries)) ; then + rm -f "$flockDownload"; + if [ "${finalAttempt}" == "true" ] ; then + droppedSizeBadDownload "${remote_url}" "${filename}" "${received_file_size}" + fi + return 1 + else + continue + fi + elif [ "${AutoRepairBadPartials}" == "true" ] && [ "$containsHtml" == "true" ] ; then + if [ -f "${file_path}" ] ; then + if ((pd_presize > 0)); then + echo -e "${YELLOW}Bad node / HTML found:${NC} reverting to previous file..." + truncateDownload "$remote_url" "$filename" "$pd_presize" "$received_file_size" + truncate -s $pd_presize "${file_path}" + else + echo -e "${YELLOW}Bad node / HTML found:${NC} tainted partial removed..." + rm -f "${file_path}" + fi + fi + if ((j >= $MaxDownloadRetries)) ; then + rm -f "$flockDownload"; + if [ "${finalAttempt}" == "true" ] ; then + droppedSizeBadDownload "${remote_url}" "${filename}" "${received_file_size}" + fi + return 1 + else + continue + fi + elif (( downDelta > 0 && downDelta < 1024 )) || [ "$containsHtml" == "true" ] ; then + if [ -f "$file_path" ] ; then + rm -rf "$file_path" + fi + echo -e "\n${YELLOW}Bad node / HTML found:${NC} tainted partial removed..." + if ((j >= $MaxDownloadRetries)) ; then + rm -f "$flockDownload"; + if [ "${finalAttempt}" == "true" ] ; then + droppedSizeBadDownload "${remote_url}" "${filename}" "${received_file_size}" + fi + return 1 + else + continue + fi + fi + if [[ "${received_file_size}" -ne "${file_size_bytes}" ]]; then + echo -e "\n${RED}Download failed, file is incomplete.${NC}" + if ((j >= $MaxDownloadRetries)) ; then + rm -f "$flockDownload"; + if [ "${finalAttempt}" == "true" ] ; then + droppedSizeBadDownload "${remote_url}" "${filename}" "${received_file_size}" + fi + return 1 + else + continue + fi + fi + else + break + fi + done + rm -f "$flockDownload"; + rm -f "${torp_cookie_jar}"; + ProcessCompletedDownload "$remote_url" "$MoveToFolder" "$filecnt" "$filename" "$file_size_bytes" "$completed_location" "$file_path" + return 0 +} +#! +#! --------------- Host Extra Functions ------------------- # +#! diff --git a/mad.sh b/mad.sh index ddcbdd9..6bed4cd 100644 --- a/mad.sh +++ b/mad.sh @@ -31,9 +31,12 @@ # * klonkerz - feedback and suggestions, url only processing # * Everyone who provided feedback and helped test.. and those who wish to remain anonymous -ScriptVersion=2024.11.08 +ScriptVersion=2024.11.09 #================================================= # Recent Additions +# 2024.11.09 - [torup] Add TorUp as a download host +# (ktgzpea2b76u7fgemiibp4a76onyybo4fw5gbsagtm6jrjzmgivppyyd.onion) +# 2024.11.08 - [nippy] Fix nippydrive.com. Update detection of temporarily unavailable response. # 2024.11.08 - [nippy] Fix nippydrive.com. Update detection of temporarily unavailable response. # 2024.11.08 - [up2share] Fix download url (https) # 2024.11.08 - [up2share] Fix advertised filesize on a redirection