#!/bin/bash # Copyright 2023-2024 kittykat (kittykat@morke.org) | (luvherpurr@torbox3uiot6wchz.onion) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS # OF ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. # # * Some lines were taken and modified from the following scripts: # - 1fichier.sh by SoupeAuLait@Rindexxx # - download1f.sh by GotGap # - hexlove.sh by AnonymousCapybara # - 1flove.sh provided by Sweeties and the NK crew # - Developers of ZTools for tesing hosts and apis, and providing a helpful resource # # Special thanks for contributions and collaboration: # * beautfar - code fixes, bad html detection, several areas of code updates, and ideas # * MisterFL - unzip after completion code and ideas # # Additional thanks for contributions: # * stacktrough - click captcha workarounds and pixel websock code # * oldfart - code suggestions, helpful feedback, clipmon implementation, WorkDir, fixes # * zaire - feedback, suggestions, and encouragement # * Rexmaxx - testing and bug reports # * TinyPanties56 - feedback and suggestions # * klonkerz - feedback and suggestions, url only processing # * Everyone who provided feedback and helped test.. and those who wish to remain anonymous ScriptVersion=2024.11.08 #================================================= # Recent Additions # 2024.11.08 - [nippy] Fix nippydrive.com. Update detection of temporarily unavailable response. # 2024.11.08 - [up2share] Fix download url (https) # 2024.11.08 - [up2share] Fix advertised filesize on a redirection # 2024.11.06 - [SkipUrlsInDownloadsCompletedTxt] Fix blank url check # 2024.11.06 - [ranoz] Add ranoz.gg as download host # 2024.11.05 - [up_ranoz] Add ranoz.gg as upload host # 2024.11.02 - [innocent] Disable rate monitor on download # 2024.11.02 - [mad, innocent] Add 18s timeout on HEAD (get hack) for no response from host # 2024.11.01 - [up_uploadhive] Add uploadhive as upload host # 2024.11.01 - [innocent] Switch between 3 alternate head type attempts # 2024.10.30 - [uploadev] Add additional file removed response handling # 2024.10.30 - [anonsharing] Add AnonSharing.com as download host (only processes urls with fileid) # 2024.10.29 - [kraken] Add recaptcha response handling # 2024.10.29 - [bowfile] Add File has been removed by the site administrator response # 2024.10.29 - [up_anonsharing] Add AnonSharing.com as upload host (unique url with fileid) # 2024.10.29 - [uploadev] Add UploadEv.org as download host # 2024.10.27 - [up_uploadev] Add UploadEv.org as upload host # 2024.10.25 - [dosya] Add check for too many failed responses. try again later. # 2024.10.24 - [bedrive / up_bedrive] Add bedrive.ru as download / upload host # 2024.10.24 - [mad] Add pw: and ref: keyword values to ./data/downloads_completed.txt logging # 2024.10.24 - [mad] Add extended connection-timeout request (tor_curl_request_extended) # 2024.10.24 - [dosya] Use extended connection-timeout request for HEAD # 2024.10.23 - [mad] Fix PostFailedUpload function call # 2024.10.22 - [innocent / up_innocent] Add innocent.onion as download / upload host # * Current download does not support resume # 2024.10.22 - [mad] Few updates to direct download # 2024.10.21 - [nippy] Update nippy to handle dbree (.me, .org) # 2024.10.21 - [dbree] Add dbree.org # 2024.10.17 - [dailyuploads] Add recaptcha detection and abort (js required) # * Working on possible PJSCloud solution # 2024.10.16 - [mad] Remove tor_curl_upload duplicate connect-timeout (thanks PeachX) # 2024.10.14 - [pjscloud] Create pjscloud.sh plugin -- use PhantomJSCloud to get a response from a url # * Change pixeldrain ViewPump to use pjscloud.sh plugin. (default enabled) # * Testing usage on other javascript required response hosts (ie. daily, hexload) # 2024.10.13 - [hexload] Add detection of new download2 type: Wait x seconds to create download link # * Add detection of download2b type loop (still working on solution, exit loop for now) # 2024.10.13 - [ocr_captcha] Add tessdata_type: NUMBERONLY, ALPHAONLY, ALPHANUMERIC, and not specified # 2024.10.12 - [mad] Alternate with nohead function on direct download for hosts that do not allow head # 2024.10.12 - [turboonion/up_turboonion] Add turbo.onion as an upload and download host # 2024.10.12 - [tempfileme/up_tempfileme] Add tempfile.me as upload and download host # 2024.10.11 - [tempsh/up_tempsh] Add temp.sh as upload and download host # 2024.10.10 - [mad] Add cleanup of input file on processing (remove trailing whitespace) # 2024.10.08 - [_audit] Add _audit.sh in release bundle which creates mad-audit-*.log for http/curl calls # * ./_audit.sh # 2024.10.07 - [mad] Add mad audit. Prints all lines that use http, curl, tor_curl in script files # * Displays color hightlighted, file, line #, and matches in scripts. # * ./mad.sh audit # 2024.10.06 - [1fichier] Add new removal response "..file has been deleted following a notification .." # 2024.10.06 - [mad] Update direct download no response processing to fail / retry # 2024.10.05 - [dailyuploads] Fix head request, filename fallback, and add head retries # 2024.10.05 - [dailyuploads] Add wrong captcha response # 2024.10.04 - Add README-uploads.txt in ./documentation (cheat sheet of hosts, retention, max size) # 2024.10.02 - [up_kouploader] Add kouploader.jp as upload host (240MB node) # 2024.10.01 - [up_axfc] Add axfc.net as upload host # -- See ./documentation/!Changelog (Historical).txt for further changes -- # #================================================= # SCRIPT GLOBALS Section #------------------- # TorIp (local network ip tor traffic is routed through) # @Default=127.0.0.1 TorIp=127.0.0.1 # UseTorCurlImpersonate: This is helpful to circumvent the Cloudflare script protection of hexload, filedot, and other hosts. # If "true", the curl-impersonate dependencies are required to be in the script directory [curl-impersonate-ff, curl_ff109]. # Instructions on downloading them are displayed if ran without them and this is set to "true". # if "false", the normal curl is used. # ** Both route through Tor and are safe # @Default=false UseTorCurlImpersonate=false # Overrides the location for all the working files (urls.txt, downloads, completed, _temp, _flocks, _debug) # * The mad.sh script can reside anywhere (optional mad.config must be in the same folder if being used) # * The WorkDirOverride path is where everything else will reside # @Default= (no override / use the same folder as the mad.sh script) WorkDirOverride="" # VerboseLoading: Determines Verbosity of display for Hosts / Plugins loading # If "true" all hosts and plugins will be displayed as loading # If "false" or blank, only "FAIL" status messages are shown # @Default=true VerboseLoading=true # [RateMonitor]: Enable Download / Upload Speed Rate Monitor that restarts downloads / uploads # if they are < X bytes for X seconds. # DownloadSpeedMin / DownloadTimeoutInterval / UploadSpeedMin / UploadTimeoutInterval specified below # @Default=true RateMonitorEnabled=true # Plugin System: This allows writing a script file that will be loaded / overlaid upon launch, allowing users to # add functions and to override (hook) functions already in mad.sh with additional functionality # (ie. unzip after successful download, skip urls that are already in completed.txt) ## Additional Hook Functions added for easy of overriding: # * OnLoad(): Occurs after load mad.config / load plugins (prior to processing). # * BeginProcessing(): Occurs immediately after beginning processin of urls.txt (loops with Catnaps). # * PreProcessUrl(): occurs immediately after reading in an unprocessed url (^http) line to process. # * PostSuccessfulDownload(): occurs after a download success (is marked #OK# in the urls.txt). # * PostFailedDownload(): occurs after a download fails (is marked #FAIL# in the urls.txt). # * PostFailRetryDownload(): occurs after a download fails with a retry (is marked #RETRY# in the urls.txt). # * DoneProcessingAllUrls(): occurs after all the urls have finished processing (no flocks or other terms downloading). # * PostSuccessfulUpload(): occurs after an upload success (after upload completed ticket is created in ./completed/). # * PostFailedUpload(): occurs after an upload fails definitively -- #FAIL# in the uploads_processed.txt # * PostFailRetryUpload(): occurs after an upload fails with a retry (network drop, unexpected result) # * DoneProcessingAllUploads: occurs after alll the files have finished processing # @Default="" # @Recommended="pjscloud.sh,ocr_captcha.sh" # @MyCurrent="pjscloud.sh,ocr_captcha.sh,SkipUrlsInDownloadsCompletedTxt.sh" LoadPlugins="" #================================================= # UPLOAD SECTION #------------------- # Upload file retries per host when uploading # @Default=4 MaxUploadRetries=4 # Selected upload hosts # @Default=1f,uhive,oshi DefaultUploadHosts='1f,uhive,oshi' # [RateMonitor - UploadSpeedMin]: Minimum required Upload Speed in bytes (used in coordination with UploadTimeoutInterval) # This helps ensure an upload doesn't go stale and hit a speed of 0 for too long. (! Requires RateMonitorEnabled=true) # ie. curl: (28) Operation too slow. Less than 10 bytes/sec transferred the last 60 seconds # @Default=10 UploadSpeedMin=10 # [RateMonitor - UploadTimeoutInterval]: Amount of time in seconds a transfer can remain below the UploadSpeedMin before it will timeout. # This helps ensure an upload doesn't go stale and hit a speed of 0 for too long. (! Requires RateMonitorEnabled=true) # ie. curl: (28) Operation too slow. Less than 5000 bytes/sec transferred the last 60 seconds # @Default=300 (5 min) UploadTimeoutInterval=300 #================================================= # DOWNLOAD SECTION #------------------- # Number of 1F circuit retries per instance to get an empty slot (# of CRs x 10 instances) # @Default=100 CircuitRetries=100 # Connection attempt timeout (tor_curl/tor_curl_impersonate) # @Default=18 ConnectTimeout=18 # Url processing retries (through drops, bad nodes, captchas, unrecognized error codes, disconnects, etc.) # This is a complete restart failure such as failure to fetch info or pre-download url info. # @Default=15 MaxUrlRetries=15 # Download file retries (through drops, bad nodes, captchas, unrecognized error codes, disconnects, etc.) # This is a quick restart of a failed download (we have cdn or download url and file info). # @Default=8 MaxDownloadRetries=8 # [RateMonitor - DownloadSpeedMin]: Minimum required Download Speed in bytes (used in coordination with DownloadTimeoutInterval) # This helps ensure a download doesn't go stale and hit a speed of 0 for too long. (! Requires RateMonitorEnabled=true) # ie. curl: (28) Operation too slow. Less than 5000 bytes/sec transferred the last 60 seconds # @Default=5000 DownloadSpeedMin=5000 # [RateMonitor - DownloadTimeoutInterval]: Amount of time in seconds a transfer can remain below the DownloadSpeedMin before it will timeout. # This helps ensure a download doesn't go stale and hit a speed of 0 for too long. (! Requires RateMonitorEnabled=true) # ie. curl: (28) Operation too slow. Less than 5000 bytes/sec transferred the last 60 seconds # @Default=60 DownloadTimeoutInterval=60 # Minimum download size in bytes: Require hosts to report > x bytes [0=disable] # @Default=100 bytes MinimumAllowedFilesize=100 # Automatically repair partial downloads with bad data (bad node, too many connections from IP, blocked, etc.) # True: Allows resuming through a bad node. # False: Restarts from the beginning on a bad node. # @Default=true AutoRepairBadPartials=true # Auto rename duplicate filenames being downloaded # This will prepend the filename with a random string ie. MyFile.rar --> 20240801124552305_renamed_MyFile.rar # @Default=false AutoRenameDuplicateFilenames=false # Loop through urls.txt until all urls are processed (# commented out..) # - Retry skipped, duplicate files from different hosts with same name, failures, etc. # @Default=true LoopThroughFileUntilComplete=true # Reload the urls.txt if it changes # @Default=true AutoReloadOnFilelistTxtChanges=true # Auto Comment out Success / Failed Downloads in urls.txt ("url" --> "#url #STATUS# filename/message") # @Default=true AutoCommentOnCompletion=true #================================================= # SCRIPT EXTRAS SECTION #------------------- # [OsType]: Used to determine how to launch terminals (multi arguments) [exo-launcher, xterm, gnome-terminal] # Whonix-exo # Linux-xterm # Linux-gnome # @Default=Linux-exo OsType="Whonix-exo" # Auto show "Mad Status" after complete # @Default=true AutoShowMadStatus=true # Clear the console on urls.txt changes # @Default=false ClearScreenOnAutoReload=false # Time to wait catnapping while waiting for urls to process or the list to complete (in minutes) # @Default=1 CatnapDuration=1 #================================================= # HOST SPECIFIC SECTION #------------------- # [Oshi]: Control BaseUrl Override (none, oshiat, oshionion) # none: Will download from whatever url base is passed in # oshiat: Will convert all oshi urls to oshi.at (clearnet, faster) # oshionion: Will convert all oshi urls to oshi.onion (onion, slower) # @Default=oshiat OshiBaseUrlOverride="oshiat" # [PixelDrain]: Use bypass method (may not be as fast, and with ViewPump enabled, not necessary) # @Default=false UsePixeldrainBypass=false # [FileDot]: Enable processing of filedot urls (must add account details below) # @Default=false EnableFiledotProcessing=false # [FileDot - Filedot User / Pass list] # ! If you wish to use filedot processing, you must create an account or accounts and enter the user/pass details below. # - Setup free accounts on the signup page (use any email address as they are not verified). Add the account information # below and remove/add any lines for accounts. # - The accounts are randomly selected for every download (each account has a 5GB/day download limit). ar_fdUP[0]="user1|pass1" #ar_fdUP[1]="user2|pass2" # Uncomment line to use a 2nd account #ar_fdUP[2]="user3|pass3" # Uncomment line to use a 3rd account #ar_fdUP[3]="user4|pass4" # Uncomment line to use a 4th account #ar_fdUP[4]="user5|pass5" # Uncomment line to use a 5th account #ar_fdUP[5]="user6|pass6" # Uncomment line to use a 6th account #ar_fdUP[6]="user7|pass7" # Uncomment line to use a 7th account #ar_fdUP[7]="user8|pass8" # Uncomment line to use a 8th account #ar_fdUP[8]="user9|pass9" # Uncomment line to use a 9th account #ar_fdUP[9]="user10|pass10" # Uncomment line to use a 10th account # [PhantomJS Keys]: pjscloud.sh plugin # ! Required for pixeldrain ViewPump and pjscloud functionality (enable / disable each in the pjscloud.sh file) # * Optional: you can create addtional / personal accounts and enter the apikey details below to ensure api resources. # - Setup free accounts on the signup page (use a temporary email, and connect from proxy or VPN). # below and remove/add any lines for apikeys. # - The keys are randomly selected for every download (each account has a 500 requests/day limit for free accounts). ar_pgsKey[0]='ak-shp9s-6zqr2-d30tt-9h64j-a0zkz' ar_pgsKey[1]='ak-2dxy6-mjhk5-ypk2m-dtgvj-4vvbb' # Uncomment line to use a 2nd key ar_pgsKey[2]='ak-akvyb-b0kpd-yrxp0-3wnkg-qjwb8' # Uncomment line to use a 3rd key ar_pgsKey[3]='ak-z5y5h-s5kpr-kkxpz-rnpxw-xpet2' # Uncomment line to use a 4th key ar_pgsKey[4]='ak-ygp2m-rhp5v-znjt2-aq9x9-pz3j9' # Uncomment line to use a 5th key ar_pgsKey[5]='ak-fgdhj-j6fad-ap0j1-ta529-v5gwh' # Uncomment line to use a 6th key ar_pgsKey[6]='ak-x2ng1-cr476-k4bph-ae8ks-9eg45' # Uncomment line to use a 7th key ar_pgsKey[7]='ak-s6k8z-wb6fz-dgb37-j268v-mgspe' # Uncomment line to use a 8th key ar_pgsKey[8]='ak-msdn7-vs5jr-4kknq-3qgw7-grj57' # Uncomment line to use a 9th key ar_pgsKey[9]='ak-77pgx-g1ge9-twmhy-em51a-p8p53' # Uncomment line to use a 10th key #ar_pgsKey[10]='aa-bbbbb-ccccc-ddddd-eeeee-fffff' # Uncomment line to use a 11th key #ar_pgsKey[11]='aa-bbbbb-ccccc-ddddd-eeeee-fffff' # Uncomment line to use a 12th key #ar_pgsKey[12]='aa-bbbbb-ccccc-ddddd-eeeee-fffff' # Uncomment line to use a 13th key #ar_pgsKey[13]='aa-bbbbb-ccccc-ddddd-eeeee-fffff' # Uncomment line to use a 14th key #ar_pgsKey[14]='aa-bbbbb-ccccc-ddddd-eeeee-fffff' # Uncomment line to use a 15th key #ar_pgsKey[15]='aa-bbbbb-ccccc-ddddd-eeeee-fffff' # Uncomment line to use a 16th key # Global pjscloud enabled hosts PJSCloud_pixeldrain=true # Enables pixeldrain ViewPump (pjscloud.sh plugin required) PJSCloud_hexload=false # Enables hexload download2 javascript response handling (WIP) PJSCloud_daily=false # Enables dailyuploads recaptcha javascript response handling (WIP) #================================================= # DEBUGGING #------------------- # DebugAllEnabled: Debug host response output to ./_debug folder # @Default=false DebugAllEnabled=false # DebugPluginsEnabled: Debug plugin hook entry and flow to screen # @Default=false DebugPluginsEnabled=false #================================================= # Other Script Globals (don't change) #------------------------------------------------- ScriptDir="$( cd "$( dirname "$(realpath "$0")" )" && pwd )" ScriptPid=$$ WorkDir=${ScriptDir} ListHostAndDomainRegexes="" ListUploadHosts="" HashFilelistTxt="" MoveToFolder="" #! Global var for passing Key/Value pairs to plugins / functions (accessible in functions) #! Can add any #key=value in the urls.txt file and then access them in the $UrlsVars variable #! UrlsVars[$key]="$val" ie. ${UrlsVars[pw]} would be the current line #pw=value declare -A UrlsVars InputFile="" warnAndRetryUnknownError=false exitDownloadNotAvailable=false fileAlreadyDone=false file_id="" file_url="" filename="" filename_override="" file_path="" download_url="" listFdotLimitReached="" checkTor() { local torPort= for port in 9050 9150 ; do echo "" 2>/dev/null > /dev/tcp/${TorIp}/$port if [ "$?" == "0" ] ; then torPort=$port fi done printf "%d" $torPort } tor_curl_request() { if [ "${UseTorCurlImpersonate}" == "true" ]; then "${curl_impersonate[@]}" --proxy "socks5h://${tor_identity}@${TorIp}:${torPort}" --connect-timeout ${ConnectTimeout} --compressed --globoff "$@" else curl --proxy "socks5h://${tor_identity}@${TorIp}:${torPort}" --connect-timeout ${ConnectTimeout} --compressed --globoff "$@" fi } tor_curl_request_extended() { if [ "${UseTorCurlImpersonate}" == "true" ]; then "${curl_impersonate[@]}" --proxy "socks5h://${tor_identity}@${TorIp}:${torPort}" --connect-timeout 60 --compressed --globoff "$@" else curl --proxy "socks5h://${tor_identity}@${TorIp}:${torPort}" --connect-timeout 60 --compressed --globoff "$@" fi } tor_curl_upload() { if [ "${UseTorCurlImpersonate}" == "true" ]; then if [ "${RateMonitorEnabled}" == "true" ]; then "${curl_impersonate[@]}" --proxy "socks5h://${tor_identity}@${TorIp}:${torPort}" --connect-timeout ${ConnectTimeout} --speed-limit $UploadSpeedMin --speed-time $UploadTimeoutInterval --compressed --globoff "$@" else "${curl_impersonate[@]}" --proxy "socks5h://${tor_identity}@${TorIp}:${torPort}" --connect-timeout ${ConnectTimeout} --compressed --globoff "$@" fi else if [ "${RateMonitorEnabled}" == "true" ]; then curl --proxy "socks5h://${tor_identity}@${TorIp}:${torPort}" --connect-timeout ${ConnectTimeout} --speed-limit $UploadSpeedMin --speed-time $UploadTimeoutInterval -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'Origin: null' -H 'Connection: keep-alive' -H 'Upgrade-Insecure-Requests: 1' -H 'Sec-Fetch-Dest: document' -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-Site: cross-site' -H 'Sec-Fetch-User: ?1' -H 'TE: trailers' --compressed --globoff "$@" else curl --proxy "socks5h://${tor_identity}@${TorIp}:${torPort}" --connect-timeout ${ConnectTimeout} -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'Origin: null' -H 'Connection: keep-alive' -H 'Upgrade-Insecure-Requests: 1' -H 'Sec-Fetch-Dest: document' -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-Site: cross-site' -H 'Sec-Fetch-User: ?1' -H 'TE: trailers' --compressed --globoff "$@" fi fi } GetRandomFiledotUser() { arrSize=${#ar_fdUP[@]} index=$(($RANDOM % $arrSize)) RandomFdotUser=${ar_fdUP[$index]} local tFdotUser=${RandomFdotUser%%\|*} if [ -f "${WorkDir}/.temp/_fdot-limitreached-accounts.txt" ]; then lastModSeconds=$(date -r "${WorkDir}/.temp/_fdot-limitreached-accounts.txt" +%s) currSeconds=$(date +%s) elapsedSeconds=$((currSeconds - lastModSeconds)) elapsedHours=$((elapsedSeconds / 60 / 60)) if ((elapsedHours >= 24)); then rm -f "${WorkDir}/.temp/_fdot-limitreached-accounts.txt" fi fi if [ -f "${WorkDir}/.temp/_fdot-limitreached-accounts.txt" ]; then listFdotLimitReached=$(cat "${WorkDir}/.temp/_fdot-limitreached-accounts.txt") else listFdotLimitReached="" fi maxAccCheckCnt=0 while [[ "${listFdotLimitReached}" == *"${tFdotUser}|"* ]]; do maxAccCheckCnt=$((maxAccCheckCnt + 1)) index=$(($RANDOM % $arrSize)) RandomFdotUser=${ar_fdUP[$index]} tFdotUser=${RandomFdotUser%%\|*} if (( maxAccCheckCnt >= 1000 )); then RandomFdotUser="#ALLACCOUNTSMAXED#|##" break fi done } urlencode_literal_grouped_case () { string=$1; format=; set -- while literal=${string%%[!-._~0-9A-Za-z]*} case "$literal" in ?*) format=$format%s set -- "$@" "$literal" string=${string#$literal};; esac case "$string" in "") false;; esac do tail=${string#?} head=${string%$tail} format=$format%%%02x set -- "$@" "'$head" string=$tail done printf "$format\\n" "$@" } urlencode_literal_grouped_case_urlendingonly () { echo "${1%/*}""/""$(urlencode_literal_grouped_case ${1##*/})" } urldecode() { : "${*//+/ }"; echo -e "${_//%/\\x}"; } urlencode_spaces() { echo "${1// /%20}" } sanitize_file_or_folder_name() { local var="$*" var="${var//\%20/ }" var="${var//[\*\"\/\\\<\>\:\|\?$'\t\r\n']/_}" var=$(trim_string $var) echo $var } sanitize_html_tags() { local var="$*" var=$(sed -e 's/<[^>]*>//g' <<< "${var}") echo $var } trim_string() { local var="$*" while [[ $var == ' '* ]]; do var="${var## }" done while [[ $var == *' ' ]]; do var="${var%% }" done echo "$var" } literalize_string() { lit_string=$1 lit_string=${lit_string//\&/\\\&} lit_string=${lit_string//\|/\\\|} lit_string=${lit_string//\[/\\\[} lit_string=${lit_string//\]/\\\]} lit_string="${lit_string//[$'\t\r\n']}" echo "${lit_string}" } GetFileSize() { local filepath=$1 local humanreadable=$2 if [ -f "$filepath" ]; then if [[ "$humanreadable" == "true" ]]; then echo $(wc -c < "$filepath" | numfmt --to=iec) else echo $(wc -c < "$filepath") fi else echo 0 fi } TrimWhitespace() { echo $(sed 's/ *$//' <<< "$1") } CleanInputFile() { sed -i 's/[ \t]*$//' $1 } LaunchTerminal() { selhost=$1 selinfile=$2 script_source="$0" if ! grep -Eqi "^(./|/)" <<< "$script_source" ; then script_source="${ScriptDir}/$script_source" fi if [ -f $script_source ] ; then printf "Spawing terminal for $script_source $selhost $selinfile\\n" if [ "$OsType" == "Whonix-exo" ]; then if [ "$selhost" == "allhosts" ]; then exo-open --launch TerminalEmulator bash -c "$script_source $selinfile" >/dev/null 2>&1 else exo-open --launch TerminalEmulator bash -c "$script_source $selhost $selinfile" >/dev/null 2>&1 fi elif [ "$OsType" == "Linux-xterm" ]; then if [ "$selhost" == "allhosts" ]; then xterm -e /bin/bash -c "$script_source $selinfile" >/dev/null 2>&1 else xterm -e /bin/bash -c "$script_source $selhost $selinfile" >/dev/null 2>&1 fi elif [ "$OsType" == "Linux-gnome" ]; then if [ "$selhost" == "allhosts" ]; then gnome-terminal -- /bin/bash -c "$script_source $selinfile" >/dev/null 2>&1 else gnome-terminal -- /bin/bash -c "$script_source $selhost $selinfile" >/dev/null 2>&1 fi else if [ "$selhost" == "allhosts" ]; then exo-open --launch TerminalEmulator /bin/bash -c "$script_source $selinfile" >/dev/null 2>&1 else exo-open --launch TerminalEmulator /bin/bash -c "$script_source $selhost $selinfile" >/dev/null 2>&1 fi fi fi } ReloadScript() { script_source="$0" passedArgs=$@ if [ -z "$passedArgs" ] && [ ! -f ${InputFile} ] ; then InputFile="${WorkDir}/${InputFile}" fi if ! grep -Eqi "^(./|/)" <<< "$script_source"; then script_source="${ScriptDir}/$script_source" fi if [ -f $script_source ]; then if [ ! -z "$passedArgs" ] ; then exec "$script_source" $@ else exec "$script_source" "${InputFile}" fi fi } check_file_extension() { if [[ "$filename" == "*.zip" ]]; then echo -e "${RED}The zip file format is considered dangerous, aborting download." return 1 elif [[ "$filename" == "*.exe" ]]; then echo -e "${RED}The file is an executable program (.exe), aborting download." return 1 else return 0 fi } debugHtml() { local fileCntName="$1" local functionName="$2" local downloadHtml=$3 if [ ! -d "${WorkDir}/.debug" ] ; then mkdir -p "${WorkDir}/.debug" fi echo -e "${downloadHtml}" >> "${WorkDir}/.debug/`date +%y%m%d-%H%M%S`_${fileCntName}_${functionName}.txt" } addResultsNote() { local url="${1//[$'\t\r\n']}" local note="$2" mkdir -p "${WorkDir}/downloads" echo -e "[NOTE] ${url}, $note" >> "${WorkDir}/downloads/results.txt" } successDownload() { local url="${1//[$'\t\r\n']}" local filename=$(literalize_string "$2") local folder=$(literalize_string "$3") local filesize=$(literalize_string "$4") local filepath=$(literalize_string "$5") mkdir -p "${WorkDir}/data" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [OK] url: ${url}, name: ${filename}, size: ${filesize}, pw: ${UrlsVars[pw]}, path: ${filepath}" >> "${WorkDir}/data/downloads_completed.txt" mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [OK] url: ${url}, name: ${filename}, size: ${filesize}, path: ${filepath}" >> "${WorkDir}/downloads/results.txt" if [ ! "$UrlOnly" == "true" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^$url.*>#& #OK# ${filename}>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#& #OK# ${filename}>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#& #OK# ${filename}>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#& #OK# ${filename}>g" "${InputFile}" #direct url https else sed -i -e "s>^$url.*>#&>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#&>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#&>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#&>g" "${InputFile}" #direct url https fi fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi PostSuccessfulDownload "${url}" "${filepath}" "${filename}" "${folder}" "${filesize}" } successDownloadExists() { local url="${1//[$'\t\r\n']}" local filename=$(literalize_string "$2") local folder=$(literalize_string "$3") local filesize=$(literalize_string "$4") local filepath=$(literalize_string "$5") mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [EXISTS] url: ${url}, name: ${filename}, size: ${filesize}, path: ${filepath}" >> "${WorkDir}/downloads/results.txt" if [ ! "$UrlOnly" == "true" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^$url.*>#& #OK# ${filename} (File exists)>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#& #OK# ${filename} (File exists)>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#& #OK# ${filename} (File exists)>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#& #OK# ${filename} (File exists)>g" "${InputFile}" #direct url https else sed -i -e "s>^$url.*>#&>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#&>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#&>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#&>g" "${InputFile}" #direct url https fi fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi } failedRetryDownload() { local url="${1//[$'\t\r\n']}" local message=$(literalize_string "$2") local message2=$(literalize_string "$3") if [ ! -z "$message2" ]; then message="$message, $message2" fi mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [RETRY] ${url}, ${message}" >> "${WorkDir}/downloads/results.txt" if [ ! "$UrlOnly" == "true" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^$url.*>#& #RETRY# ${message}>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#& #RETRY# ${message}>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#& #RETRY# ${message}>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#& #RETRY# ${message}>g" "${InputFile}" #direct url https else sed -i -e "s>^$url.*>#&>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#&>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#&>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#&>g" "${InputFile}" #direct url https fi fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi PostFailRetryDownload "${url}" "${message}" "${message2}" } failedDownload() { local url="${1//[$'\t\r\n']}" local message=$(literalize_string "$2") local message2=$(literalize_string "$3") if [ ! -z "$message2" ]; then message="$message, $message2" fi mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [FAIL] ${url}, ${message}" >> "${WorkDir}/downloads/results.txt" if [ ! "$UrlOnly" == "true" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^$url.*>#& #FAIL# $message>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#& #FAIL# $message>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#& #RETRY# $message>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#& #FAIL# $message>g" "${InputFile}" #direct url https else sed -i -e "s>^$url.*>#&>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#&>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#&>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#&>g" "${InputFile}" #direct url https fi fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi PostFailedDownload "${url}" "${message}" "${message2}" } removedDownload() { local url="${1//[$'\t\r\n']}" local message=$(literalize_string "$2") mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [REMOVED] ${url}" >> "${WorkDir}/downloads/results.txt" mkdir -p "${WorkDir}/data" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [REMOVED] url: ${url}, message: $message" >> "${WorkDir}/data/downloads_completed.txt" if [ ! "$UrlOnly" == "true" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^$url.*>#& #REMOVED#>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#& #REMOVED#>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#& #REMOVED#>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#& #REMOVED#>g" "${InputFile}" #direct url https else sed -i -e "s>^$url.*>#&>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#&>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#&>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#&>g" "${InputFile}" #direct url https fi fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi } skipUrlDownload() { local url="${1//[$'\t\r\n']}" local filename="$2" local flockshortname=$3 local flockpath=$4 flockpathcontents=$(cat $flockpath) CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi } renameDuplicateDownload() { local url=$(literalize_string "${1//[$'\t\r\n']}") local filename=$(literalize_string "$2") local cTimestamp=$(date +"%Y%m%d%H%M%S%3N") local newfilename="${cTimestamp}_renamed_${filename}" if [ ! "$UrlOnly" == "true" ]; then sed -i -e "s>^$url.*>${url}|${newfilename}>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>${url}|${newfilename}>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>direct=${url}|${newfilename}>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>direct=${url}|${newfilename}>g" "${InputFile}" #direct url https fi mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [RENAME] ${url}, ${filename}, (new filename: ${newfilename})" >> "${WorkDir}/downloads/results.txt" CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi if [ "$UrlOnly" == "true" ]; then echo -e "${RED}| [FAILED]: A file already exists with the specified name. Rename it and try again.${NC}" echo -e "| Filename: \"./downloads/$filename\"" fi } droppedSizeBadDownload() { local url="${1//[$'\t\r\n']}" local filename=$(literalize_string "$2") local filesize=$(literalize_string "$3") local altsize=$(literalize_string "$4") mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo "$dateStamp [DROP/BADSIZE] ${url}, ${filename}, size: ${filesize}, advertisedsize: ${altsize}" >> "${WorkDir}/downloads/results.txt" if [ ! "$UrlOnly" == "true" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^$url.*>#& #RETRY# ${filename} (dropped or bad size)>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#& #RETRY# ${filename} (dropped or bad size)>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#& #RETRY# ${filename} (dropped or bad size)>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#& #RETRY# ${filename} (dropped or bad size)>g" "${InputFile}" #direct url https else sed -i -e "s>^$url.*>#&>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#&>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#&>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#&>g" "${InputFile}" #direct url https fi fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi } noCdnDownload() { local url="${1//[$'\t\r\n']}" mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [NOCDN] ${url}" >> "${WorkDir}/downloads/results.txt" if [ ! "$UrlOnly" == "true" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^$url.*>#& #RETRY# (No CDN found)>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#& #RETRY# (No CDN found)>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#& #RETRY# (No CDN found)>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#& #RETRY# (No CDN found)>g" "${InputFile}" #direct url https else sed -i -e "s>^$url.*>#&>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#&>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#&>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#&>g" "${InputFile}" #direct url https fi fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi } passwordProtectedDownload() { local url="${1//[$'\t\r\n']}" mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [PASSWORD] ${url}" >> "${WorkDir}/downloads/results.txt" mkdir -p "${WorkDir}/data" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [PASSWORD] url: ${url}, name: ${filename}, size: ${filesize}, path: ${filepath}" >> "${WorkDir}/data/downloads_completed.txt" if [ ! "$UrlOnly" == "true" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^${url}.*>#& #PASSWORD#>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#& #PASSWORD#>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=${url}.*>#& #PASSWORD#>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#& #PASSWORD#>g" "${InputFile}" #direct url https else sed -i -e "s>^$url.*>#&>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#&>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#&>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#&>g" "${InputFile}" #direct url https fi fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi } badUrlDownload() { local url="${1//[$'\t\r\n']}" local message=$(literalize_string "$2") if [ ! -z $message ]; then message=" $message" fi mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [BADURL] ${url}" >> "${WorkDir}/downloads/results.txt" if [ ! "$UrlOnly" == "true" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^${url}.*>#& #BAD-URL#${message}>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#& #BAD-URL#${message}>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=${url}.*>#& #BAD-URL#${message}>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#& #BAD-URL#${message}>g" "${InputFile}" #direct url https else sed -i -e "s>^$url.*>#&>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#&>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#&>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#&>g" "${InputFile}" #direct url https fi fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi } filenameOrSizeNotExistDownload() { local url="${1//[$'\t\r\n']}" local filename=$(literalize_string "$2") local message=$(literalize_string "$3") mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [FAIL] ${url}, No File / Bad size (filename: ${filename}, ${message})" >> "${WorkDir}/downloads/results.txt" if [ ! "$UrlOnly" == "true" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^${url}.*>#& #RETRY# ${filename} (No filename or size. Try again later)>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#& #RETRY# ${filename} (No filename or size. Try again later)>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$${url}.*>#& #RETRY# ${filename} (No filename or size. Try again later)>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#& #RETRY# ${filename} (No filename or size. Try again later)>g" "${InputFile}" #direct url https else sed -i -e "s>^$url.*>#&>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#&>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#&>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#&>g" "${InputFile}" #direct url https fi fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi } fileExistsButSizeTooLargeDownload() { local url="${1//[$'\t\r\n']}" local filename=$(literalize_string "$2") local filesize=$(literalize_string "$3") local downloadFilesize=$(literalize_string "$4") mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [EXISTS/TOOLARGE] ${url}, ${filename}, sizeOnDisk: ${filesize} downSize: ${downloadFilesize}" >> "${WorkDir}/downloads/results.txt" if [ ! "$UrlOnly" == "true" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^$url.*>#& #RETRY# ${filename} (File exists and is too large--retry or delete from downloads)>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#& #RETRY# ${filename} (File exists and is too large--retry or delete from downloads)>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#& #RETRY# ${filename} (File exists and is too large--retry or delete from downloads)>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#& #RETRY# ${filename} (File exists and is too large--retry or delete from downloads)>g" "${InputFile}" #direct url https else sed -i -e "s>^$url.*>#&>g" "${InputFile}" #processed url sed -i -e "s>^${url/https:/http:}.*>#&>g" "${InputFile}" #http (if changed) sed -i -e "s>^direct=$url.*>#&>g" "${InputFile}" #direct url http sed -i -e "s>^direct=${url/https:/http:}.*>#&>g" "${InputFile}" #direct url https fi fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ]; then rm -f "${WorkDir}/.flocks/${CLEANSTRING}" fi } truncateDownload() { local url="${1//[$'\t\r\n']}" local filename="$2" local truncsize=$3 local partialsize=$4 mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [TRUNC] ${url}, ${filename}, tsize:$truncsize, psize: $partialsize" >> "${WorkDir}/downloads/results.txt" } pumpDownload() { local url="${1//[$'\t\r\n']}" local preViews=$2 local postViews=$3 mkdir -p "${WorkDir}/downloads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "$dateStamp [PUMP] ${url}, previews:$preViews, postviews: $postViews" >> "${WorkDir}/downloads/results.txt" } successUpload() { local pLine="${1//[$'\t\r\n']}" local filepath="${2//[$'\t\r\n']}" local HostCode=$(literalize_string "$3") local filesize=$(literalize_string "$4") local downloadLink="$5" local responseHtml=$6 local filename="${filepath##*/}" mkdir -p "${WorkDir}/uploads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "[OK] ${filename}, ${HostCode}, ${downloadLink}" >> "${WorkDir}/uploads/results.txt" mkdir -p "${WorkDir}/uploads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "${filename}, ${HostCode}, ${downloadLink}" >> "${WorkDir}/uploads/result-links.txt" mkdir -p "${WorkDir}/uploads/_tickets" cTicket="${WorkDir}/uploads/_tickets/`date +%y%m%d-%H%M%S`_${filename}_${HostCode}_upload.txt" echo -e "${downloadLink}\\n\\nResponse:\\n${responseHtml}" > "$cTicket" echo -e "${BLUE}| Ticket${NC}: "$cTicket"" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') mkdir -p "${WorkDir}/data" echo -e "$dateStamp [OK] file: ${filename}, host: ${HostCode}, dl: ${downloadLink}, ticket: ${cTicket}, size: ${filesize}, path: ${filepath}" >> "${WorkDir}/data/uploads_completed.txt" if [ ! -z "$InputFile" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^${pLine}.*>#& #OK# $downloadLink>g" "${InputFile}" #processed line else sed -i -e "s>^$pLine.*>#& $downloadLink>g" "${InputFile}" #processed line fi else dateStamp=$(date '+%Y/%m/%d %H:%M:%S') mkdir -p "${WorkDir}/uploads" echo -e "$dateStamp [OK] file: ${filename}, host: ${HostCode}, dl: ${downloadLink}, ticket: ${cTicket}, size: ${filesize}, path: ${filepath}" >> "${WorkDir}/uploads/temp_upload_handler.txt" fi UploadTicket="${WorkDir}/.flocks/upload_${HostCode}_${filepath//[^a-zA-Z0-9]/}" if [ -f "${UploadTicket}" ]; then rm -f "${UploadTicket}" fi PostSuccessfulUpload "${filepath}" "${HostCode}" "${filename}" "${filesize}" "${downloadLink}" } successUploadExists() { local pLine="${1//[$'\t\r\n']}" local filepath="${2//[$'\t\r\n']}" local HostCode=$(literalize_string "$3") local message=$(literalize_string "$4") local message2=$(literalize_string "$5") local filename="${filepath##*/}" if [ ! -z "$message2" ]; then message="$message, $message2" fi mkdir -p "${WorkDir}/uploads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "[EXISTS] ${filename}, ${HostCode}, ${downloadLink}" >> "${WorkDir}/uploads/results.txt" if [ ! -z "$InputFile" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^${pLine}.*>#& #OK# (Upload exists) ${message}>g" "${InputFile}" #processed line else sed -i -e "s>^$pLine.*>#&>g" "${InputFile}" #processed line fi fi UploadTicket="${WorkDir}/.flocks/upload_${HostCode}_${filepath//[^a-zA-Z0-9]/}" if [ -f "${UploadTicket}" ]; then rm -f "${UploadTicket}" fi } failedUpload() { local pLine="${1//[$'\t\r\n']}" local filepath="${2//[$'\t\r\n']}" local HostCode=$(literalize_string "$3") local message=$(literalize_string "$4") local message2=$(literalize_string "$5") local filename="${filepath##*/}" if [ ! -z "$message2" ]; then message="$message, $message2" fi mkdir -p "${WorkDir}/uploads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "[FAIL] ${HostCode}, ${filename}, ${message}" >> "${WorkDir}/uploads/results.txt" if [ ! -z "$InputFile" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^${pLine}.*>#& #FAIL# ${message}>g" "${InputFile}" #processed line else sed -i -e "s>^$pLine.*>#&>g" "${InputFile}" #processed line fi else dateStamp=$(date '+%Y/%m/%d %H:%M:%S') mkdir -p "${WorkDir}/uploads" echo -e "$dateStamp [FAIL] file: ${filename}, host: ${HostCode}, msg: ${message}, path: ${filepath}" >> "${WorkDir}/uploads/temp_upload_handler.txt" fi UploadTicket="${WorkDir}/.flocks/upload_${HostCode}_${filepath//[^a-zA-Z0-9]/}" if [ -f "${UploadTicket}" ]; then rm -f "${UploadTicket}" fi PostFailedUpload "$pline" "${filepath}" "${HostCode}" "${message1}" "${message2}" } failedRetryUpload() { local pLine="${1//[$'\t\r\n']}" local filepath="${2//[$'\t\r\n']}" local HostCode=$(literalize_string "$3") local message=$(literalize_string "$4") local message2=$(literalize_string "$5") if [ ! -z "$message2" ]; then message="$message, $message2" fi mkdir -p "${WorkDir}/uploads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "[RETRY] ${HostCode}, ${filename}, ${message}" >> "${WorkDir}/uploads/results.txt" if [ ! -z "$InputFile" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^${pLine}.*>#& #RETRY# ${message}>g" "${InputFile}" #processed line else sed -i -e "s>^$pLine.*>#&>g" "${InputFile}" #processed line fi else dateStamp=$(date '+%Y/%m/%d %H:%M:%S') mkdir -p "${WorkDir}/uploads" echo -e "$dateStamp [RETRY] file: ${filename}, host: ${HostCode}, msg: ${message}, path: ${filepath}" >> "${WorkDir}/uploads/temp_upload_handler.txt" fi UploadTicket="${WorkDir}/.flocks/upload_${HostCode}_${filepath//[^a-zA-Z0-9]/}" if [ -f "${UploadTicket}" ]; then rm -f "${UploadTicket}" fi PostFailRetryUpload "${url}" "${message}" "${message2}" } skipFailedUpload() { local pLine="${1//[$'\t\r\n']}" local filepath="${2//[$'\t\r\n']}" local HostCode=$(literalize_string "$3") local message=$(literalize_string "$4") local message2=$(literalize_string "$5") local filename="${filepath##*/}" if [ ! -z "$message2" ]; then message="$message, $message2" fi mkdir -p "${WorkDir}/uploads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "[SKIP/FAIL] ${HostCode}, ${filename}, ${message}" >> "${WorkDir}/uploads/results.txt" if [ ! -z "$InputFile" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^${pLine}.*>#& #FAIL# (Skip) ${message}>g" "${InputFile}" #processed line else sed -i -e "s>^$pLine.*>#&>g" "${InputFile}" #processed line fi else dateStamp=$(date '+%Y/%m/%d %H:%M:%S') mkdir -p "${WorkDir}/uploads" echo -e "$dateStamp [FAIL] file: ${filename}, host: ${HostCode}, msg: ${message}, path: ${filepath}" >> "${WorkDir}/uploads/temp_upload_handler.txt" fi UploadTicket="${WorkDir}/.flocks/upload_${HostCode}_${filepath//[^a-zA-Z0-9]/}" if [ -f "${UploadTicket}" ]; then rm -f "${UploadTicket}" fi } uploadBadInputLine() { local pLine="${1//[$'\t\r\n']}" local message=$(literalize_string "$2") if [ ! -z $message ]; then message=" $message" fi mkdir -p "${WorkDir}/uploads" dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "[BADLINE] ${pLine}${message}" >> "${WorkDir}/uploads/results.txt" if [ ! -z "$InputFile" ]; then if [ "${AutoCommentOnCompletion}" == "true" ] ; then sed -i -e "s>^${pLine}.*>#& #RETRY# (Bad Line)${message}>g" "${InputFile}" #processed line else sed -i -e "s>^$pLine.*>#&>g" "${InputFile}" #processed line fi else dateStamp=$(date '+%Y/%m/%d %H:%M:%S') mkdir -p "${WorkDir}/uploads" echo -e "$dateStamp [FAIL] file: ${filename}, host: ${HostCode}, msg: ${message}, path: ${filepath}" >> "${WorkDir}/uploads/temp_upload_handler.txt" fi } yes_or_no() { while true; do printf "${YELLOW}" read -p "$* [y/n]: " yn printf "${NC}" case $yn in [Yy]*) return 0 ;; [Nn]*) echo -e "Aborted" ; return 1 ;; esac done } sleepRandomSecs() { minWait=$1 maxWait=$2 sleep $((minWait + RANDOM % (maxWait - minWait)))s } sleepRandomMins() { minWait=$1 maxWait=$2 sleep $((minWait + RANDOM % (maxWait - minWait)))m } GetRandomUA() { ar_UA[0]="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.3" ar_UA[1]="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.3" ar_UA[2]="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0" ar_UA[3]="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0" ar_UA[4]="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.3" ar_UA[5]="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.4" ar_UA[6]="Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0" ar_UA[7]="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0" ar_UA[8]="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0" ar_UA[9]="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0" ar_UA[10]="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0" arrSize=${#ar_UA[@]} index=$(($RANDOM % $arrSize)) RandomUA=${ar_UA[$index]} } LoadMadDownloadHosts() { if [ -d "${ScriptDir}/hosts/" ]; then echo -e "${GREEN}Loading Download Hosts...${NC}" tHostFuncPrefixes="" cnthostsloaded=0 for fil in "${ScriptDir}"/hosts/*.sh ; do if [ -f "$fil" ]; then if grep -Eq '^HostFuncPrefix='"'" "$fil" ; then tfilename="${fil##*/}" if [[ "$tfilename" == "up_"* ]] ; then continue fi _hostfuncprefix=$(grep -oP -m 1 '^HostFuncPrefix='"'"'\K.*?(?='"'"')' "$fil") if [[ ! "${_hostfuncprefix}" == "direct" ]] && \ [ ! -z "${tHostFuncPrefixes}" ] && \ grep -Eqi ":${_hostfuncprefix}:" <<< "$tHostFuncPrefixes" ; then echo -e "[${RED}FAIL${NC}] ${BLUE}${tfilename}${NC} (dupe HostFuncPrefix ${PINK}${_hostfuncprefix}${NC} detected)${NC}" else source "$fil" cnthostsloaded=$((cnthostsloaded + 1)) tHostFuncPrefixes="${tHostFuncPrefixes}:${_hostfuncprefix}:" if [ "$VerboseLoading" == "true" ]; then echo -e "[${GREEN}OK${NC}] ${BLUE}${tfilename}${NC}" fi fi else echo -e "[${RED}FAIL${NC}] ${BLUE}${tfilename}${NC} (not a valid host file)${NC}" fi fi done echo -e " (${YELLOW}$cnthostsloaded${NC}) loaded${NC}" echo -e "" fi } LoadMadUploadHosts() { if [ -d "${ScriptDir}/hosts/" ]; then echo -e "${GREEN}Loading Upload Hosts...${NC}" tHostFuncPrefixes="" cnthostsloaded=0 for fil in "${ScriptDir}"/hosts/up_*.sh ; do if [ -f "$fil" ]; then if grep -Eq '^HostFuncPrefix='"'" "$fil" ; then tfilename="${fil##*/}" _hostfuncprefix=$(grep -oP -m 1 '^HostFuncPrefix='"'"'\K.*?(?='"'"')' "$fil") if [[ ! "${_hostfuncprefix}" == "direct" ]] && \ [ ! -z "${tHostFuncPrefixes}" ] && \ grep -Eqi ":${_hostfuncprefix}:" <<< "$tHostFuncPrefixes" ; then echo -e "[${RED}FAIL${NC}] ${BLUE}${tfilename}${NC} (dupe HostFuncPrefix ${PINK}${_hostfuncprefix}${NC} detected)${NC}" else source "$fil" cnthostsloaded=$((cnthostsloaded + 1)) tHostFuncPrefixes="${tHostFuncPrefixes}:${_hostfuncprefix}:" if [ "$VerboseLoading" == "true" ]; then echo -e "[${GREEN}OK${NC}] ${BLUE}${tfilename}${NC}" fi fi else echo -e "[${RED}FAIL${NC}] ${BLUE}${tfilename}${NC} (not a valid host file)${NC}" fi fi done echo -e " (${YELLOW}$cnthostsloaded${NC}) loaded${NC}" echo -e "" fi } LoadMadPlugins() { if [ ! -z "${LoadPlugins}" ]; then echo -e "${GREEN}Loading Plugins...${NC}" loadedPluginFuncsUsed="" cntplugsloaded=0 readarray -d "," -t arrPlugins <<< "${LoadPlugins}" for plugin in "${arrPlugins[@]}"; do plg="${plugin//[$'\t\r\n']}" if [ ! -z "${plg}" ] ; then plg=$(literalize_string "$plugin") if [ -f "${ScriptDir}/plugins/${plg}" ]; then currPluginFuncsUsed=$(grep '()' "${ScriptDir}/plugins/${plg}" | awk '!/#/ {print $1}') currPluginFuncsUsed="${currPluginFuncsUsed//$'\n'/, }" readarray -d ", " -t arrCurrPluginFuncsUsed <<< "$currPluginFuncsUsed" local isDupeFunc=false for cplg in "${arrCurrPluginFuncsUsed[@]}"; do if [ ! -z "${loadedPluginFuncsUsed}" ] && [[ *"${cplg}"* == "$loadedPluginFuncsUsed" ]] ; then isDupeFunc=true echo -e "[${RED}FAIL${NC}] ${BLUE}${plg}${NC} (dupe hook detected)${NC}" echo -e " Function already overriden: ${YELLOW}$cplg${NC}" break else if grep -Eq '^OnLoad' <<< "$cplg" ; then lstOnLoad="$lstOnLoad""@""${cplg%%(*}" elif grep -Eq '^BeginProcessing' <<< "$cplg" ; then lstBeginProcessing="$lstBeginProcessing""@""${cplg%%(*}" elif grep -Eq '^PreProcessUrl' <<< "$cplg" ; then lstPreProcessUrl="$lstPreProcessUrl""@""${cplg%%(*}" elif grep -Eq '^PostSuccessfulDownload' <<< "$cplg" ; then lstPostSuccessfulDownload="$lstPostSuccessfulDownload""@""${cplg%%(*}" elif grep -Eq '^PostFailedDownload' <<< "$cplg" ; then lstPostFailedDownload="$lstPostFailedDownload""@""${cplg%%(*}" elif grep -Eq '^PostFailRetryDownload' <<< "$cplg" ; then lstPostFailRetryDownload="$lstPostFailRetryDownload""@""${cplg%%(*}" elif grep -Eq '^DoneProcessingAllUrls' <<< "$cplg" ; then lstDoneProcessingAllUrls="$lstDoneProcessingAllUrls""@""${cplg%%(*}" elif grep -Eq '^PostSuccessfulUpload' <<< "$cplg" ; then lstPostSuccessfulUpload="$lstPostSuccessfulUpload""@""${cplg%%(*}" elif grep -Eq '^PostFailedUpload' <<< "$cplg" ; then lstPostFailedUpload="$lstPostFailedUpload""@""${cplg%%(*}" elif grep -Eq '^PostFailRetryUpload' <<< "$cplg" ; then lstPostFailRetryUpload="$lstPostFailRetryUpload""@""${cplg%%(*}" elif grep -Eq '^DoneProcessingAllUploads' <<< "$cplg" ; then lstDoneProcessingAllUploads="$lstDoneProcessingAllUploads""@""${cplg%%(*}" fi fi done if [ "$isDupeFunc" == "false" ]; then source "${ScriptDir}/plugins/${plg}" cntplugsloaded=$((cntplugsloaded + 1)) if [ "$VerboseLoading" == "true" ]; then echo -e "[${GREEN}OK${NC}] ${BLUE}${plg}${NC}" echo -e " --> Hooks: ${PINK}$currPluginFuncsUsed${NC}" fi loadedPluginFuncsUsed="${loadedPluginFuncsUsed}${currPluginFuncsUsed}" else continue fi else echo -e "[${RED}FAIL${NC}] ${BLUE}${plg}${NC} (not found)${NC}" fi fi done echo -e " (${YELLOW}$cntplugsloaded${NC}) loaded${NC}" echo -e "" fi } install_curl_impersonate() { clear echo -e "${BLUE}lwthiker curl_impersonate${NC} is the original dev, but it is relatively inactive." echo -e "- Currently uses curl v8.1.1." echo -e "+ Supports Firefox, Chrome, Edge, and Safari impersonation." echo -e "+ (Recommended)" echo -e "" echo -e "${BLUE}lexiforest curl_impersonate${NC} is an active fork of curl_impersonate." echo -e "+ Currently uses curl v8.7.1" echo -e "+ Supports Chrome, Edge, and Safari impersonation." echo -e "+ (beta, not tested)" echo -e "" PS3='Please select which curl_impersonate to install: ' options=("lwthiker (orig, ff/chrome/edge/safari, not active) *Currently recommended*" \ "lexiforest (fork, chrome/edge/safari, active)" \ "Quit") select opt in "${options[@]}" do case $opt in "lwthiker (orig, ff/chrome/edge/safari, not active) *Currently recommended*") install_curl_impersonate_lwthiker_orig break ;; "lexiforest (fork, chrome/edge/safari, active)") install_curl_impersonate_lexiforest_fork break ;; "Quit") exit 0 ;; *) echo "Invalid option $REPLY";; esac done } install_curl_impersonate_lwthiker_orig() { clear mkdir -p "${WorkDir}/downloads" mkdir -p "${WorkDir}/.temp" echo -e "${BLUE}lwthiker curl_impersonate${NC} is the original curl_impersonate." echo -e "+ Currently uses curl v8.1.1, and has low activity for updates" echo -e "+ Supports FireFox, Chrome, Edge, and Safari impersonation." echo -e "" echo -e "${GREEN}| Fetching:${NC} latest ${BLUE}lwthiker curl_impersonate${NC} info from github...${NC}" latestBinaryUrl="" for ((j=1; j<=4; j++)); do response=$(tor_curl_request --insecure -L -s https://github.com/lwthiker/curl-impersonate/releases/latest) if [ "${DebugAllEnabled}" == "true" ] ; then debugHtml "github" "lbf_inst_curlimp$j" "$response" fi if [ ! -z "$response" ]; then latestTag=$(grep -oPi -m 1 '(?<=/curl-impersonate/releases/tag/).*?(?=")' <<< "$response") latestBinaryDate=$(grep -oPi -m 1 '(?<= file_size_bytes)); then echo -e "${RED}| ERROR: File exists in downloads folder but is larger than expected.${NC}\nThis could be due to several instances saving the same file, an old download using the same name, or host experiencing a temporary issue." exit 1 fi else for ((j=1; j<=$MaxDownloadRetries; j++)); do tor_identity="${RANDOM}" trap "echo ""; tput cnorm; exit" 0 1 2 3 6 15 tor_curl_request --insecure -L "$download_url" --continue-at - --output "$file_path" received_file_size=0 if [ -f "$file_path" ] ; then received_file_size=$(stat --format="%s" "$file_path" | tr -d '[:space:]') fi if ((received_file_size == file_size_bytes)) ; then break elif ((received_file_size < file_size_bytes)) ; then if ((j >= MaxDownloadRetries)) ; then echo -e "${RED}| FAILED: Size mismatch after downloading${NC}" exit 1 else echo -e "${YELLOW}| RETRY: Size mismatch after downloading.${NC}" continue fi elif ((received_file_size > file_size_bytes)) ; then echo -e "${RED}| FAILED: Size is too large after downloading.${NC}" exit 1 else echo -e "${RED}| FAILED: Unknown error downloading.${NC}" exit 1 fi done fi if [ -f $file_path ] ; then echo -e "" echo -e "${GREEN}| [OK]${NC} Download complete.${NC}" mv "${file_path}" "$extract_location" final_tarpath="$extract_location/$filename" echo -e "| Extracting curl_impersonate..." tarOutput=$(tar -xvzf ${final_tarpath} -C $extract_location) rm -f "${ScriptDir}"/curl* mv "$extract_location/curl-impersonate-ff" "${ScriptDir}/" mv "$extract_location/curl_ff109" "${ScriptDir}/" echo -e "${GREEN}| Done.${NC} Update ${BLUE}\"UseTorCurlImpersonate=true\"${NC} in script to use..." exit 0 else echo -e "${RED}| Failed: Unable to find ./.temp/${filename}${NC}" exit 1 fi } } install_curl_impersonate_lexiforest_fork() { clear mkdir -p "${WorkDir}/downloads" mkdir -p "${WorkDir}/.temp" echo -e "${BLUE}lexiforest curl_impersonate${NC} is an active fork of curl_impersonate." echo -e "+ Currently uses curl v8.7.1, and is patched for latest CVEs" echo -e "+ Supports Chrome, Edge, and Safari impersonation." echo -e "" echo -e "${GREEN}| Fetching:${NC} latest ${BLUE}lexiforest curl_impersonate fork${NC} info from github...${NC}" latestBinaryUrl="" for ((j=1; j<=4; j++)); do response=$(tor_curl_request --insecure -L -s https://github.com/lexiforest/curl-impersonate/releases/latest) if [ "${DebugAllEnabled}" == "true" ] ; then debugHtml "github" "lbf_inst_curlimp$j" "$response" fi if [ ! -z "$response" ]; then latestTag=$(grep -oPi -m 1 '(?<=/curl-impersonate/releases/tag/).*?(?=")' <<< "$response") latestBinaryDate=$(grep -oPi -m 1 '(?<= file_size_bytes)); then echo -e "${RED}| ERROR: File exists in downloads folder but is larger than expected.${NC}\nThis could be due to several instances saving the same file, an old download using the same name, or host experiencing a temporary issue." exit 1 fi else for ((j=1; j<=$MaxDownloadRetries; j++)); do tor_identity="${RANDOM}" trap "echo ""; tput cnorm; exit" 0 1 2 3 6 15 tor_curl_request --insecure -L "$download_url" --continue-at - --output "$file_path" received_file_size=0 if [ -f "$file_path" ] ; then received_file_size=$(stat --format="%s" "$file_path" | tr -d '[:space:]') fi if ((received_file_size == file_size_bytes)) ; then break elif ((received_file_size < file_size_bytes)) ; then if ((j >= MaxDownloadRetries)) ; then echo -e "${RED}| FAILED: Size mismatch after downloading${NC}" exit 1 else echo -e "${YELLOW}| RETRY: Size mismatch after downloading.${NC}" continue fi elif ((received_file_size > file_size_bytes)) ; then echo -e "${RED}| FAILED: Size is too large after downloading.${NC}" exit 1 else echo -e "${RED}| FAILED: Unknown error downloading.${NC}" exit 1 fi done fi if [ -f $file_path ] ; then echo -e "" echo -e "${GREEN}| [OK]${NC} Download complete.${NC}" mv "${file_path}" "$extract_location" final_tarpath="$extract_location/$filename" echo -e "| Extracting curl_impersonate..." tarOutput=$(tar -xvzf ${final_tarpath} -C $extract_location) rm -f "${ScriptDir}"/curl* mv "$extract_location/curl-impersonate-chrome" "${ScriptDir}/" mv "$extract_location/curl_chrome124" "${ScriptDir}/" echo -e "${GREEN}| Done.${NC} Update ${BLUE}\"UseTorCurlImpersonate=true\"${NC} in script to use..." exit 0 else echo -e "${RED}| Failed: Unable to find ./.temp/${filename}${NC}" exit 1 fi } } madReset() { local InputFile="$1" local noprompt=false local autoclean=false local autoclose=true local noclear=false if [ "$2" == "true" ]; then noprompt=true fi if [ "$3" == "true" ]; then autoclean=true fi if [ "$4" == "false" ]; then autoclose=false fi if [ "$5" == "true" ]; then noclear=true fi if [ "$noclear" == "false" ]; then clear fi echo -e "${BLD}" echo -e "${PINK}:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:" echo -e ":${NC} ${GREEN}MAD${PINK} Reset${NC} : Revert URLs marked with #RETRY#${PINK}${BLD} :" echo -e ":-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:${NC}\\n" if [ ! -f "${InputFile}" ] && [ -f "${WorkDir}/${InputFile}" ]; then InputFile="${WorkDir}/${InputFile}" fi if [ ! -f "${InputFile}" ]; then printf "Unable to read file %s! [mr]\\n" "${InputFile}" else fileCount=1 lineCnt=1 while IFS= read -r line || [[ -n $line ]]; do if [[ "$line" =~ ^\#(http|https|direct=http|direct=https):// ]] && [[ "$line" =~ '#RETRY#' ]]; then remote_url=$(grep -oPi '^#\K.*?(?= #RETRY#)' <<< "$line") printf "Reset Line ${GREEN}#${lineCnt}${NC}: %s\\n" "${remote_url}" sed -i -e "s|^$(literalize_string "${line}")|$(literalize_string "${remote_url}")|gi" "${InputFile}" #processed url fi lineCnt=$((lineCnt+1)) done < ${InputFile} fi if [ -d "${WorkDir}/.flocks" ] || [ -d "${WorkDir}/.temp" ] || [ -d "${WorkDir}/.debug" ] ; then if [ "$autoclean" == "true" ]; then if [ -d "${WorkDir}/.flocks" ]; then rm -rf "${WorkDir}/.flocks" fi if [ -d "${WorkDir}/.temp" ]; then rm -rf "${WorkDir}/.temp" fi if [ -d "${WorkDir}/.debug" ]; then rm -rf "${WorkDir}/.debug" fi else if [ "$noprompt" == "false" ]; then printf "\\n" yes_or_no "Cleanup temp folders? (choose 'n' if currently processing in other terminals)" && { if [ -d "${WorkDir}/.flocks" ]; then rm -rf "${WorkDir}/.flocks" fi if [ -d "${WorkDir}/.temp" ]; then rm -rf "${WorkDir}/.temp" fi if [ -d "${WorkDir}/.debug" ]; then rm -rf "${WorkDir}/.debug" fi } fi fi fi if [ "$autoclose" == "true" ]; then exit 0 fi } madResetUploads() { local InputFile="$1" local noprompt=false local autoclean=false local autoclose=true local noclear=false if [ "$2" == "true" ]; then noprompt=true fi if [ "$3" == "true" ]; then autoclean=true fi if [ "$4" == "false" ]; then autoclose=false fi if [ "$5" == "true" ]; then noclear=true fi if [ "$noclear" == "false" ]; then clear fi echo -e "${BLD}" echo -e "${PINK}:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:" echo -e ":${NC} ${GREEN}MAD${PINK} Reset${NC} : Revert Upload files marked with #RETRY#${PINK}${BLD} :" echo -e ":-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:${NC}\\n" if [ ! -f "${InputFile}" ] && [ -f "${WorkDir}/${InputFile}" ]; then InputFile="${WorkDir}/${InputFile}" fi if [ ! -f "${InputFile}" ]; then printf "Unable to read file %s! [mr]\\n" "${InputFile}" exit 1 fi fileCount=1 lineCnt=1 while IFS= read -r line || [[ -n $line ]]; do if grep -Eqi '^\#.*\|.*#RETRY#' <<< "$line" ; then fline=$(grep -oPi '^#\K.*?(?= #RETRY#)' <<< "$line") printf "Reset Line ${GREEN}#${lineCnt}${NC}: %s\\n" "${fline}" sed -i -e "s|^$(literalize_string "${line}")|$(literalize_string "${fline}")|gi" "${InputFile}" #processed url fi lineCnt=$((lineCnt+1)) done < ${InputFile} if [ -d "${WorkDir}/.flocks" ] || [ -d "${WorkDir}/.temp" ] || [ -d "${WorkDir}/.debug" ] ; then if [ "$autoclean" == "true" ]; then if [ -d "${WorkDir}/.flocks" ]; then rm -rf "${WorkDir}/.flocks" fi if [ -d "${WorkDir}/.temp" ]; then rm -rf "${WorkDir}/.temp" fi if [ -d "${WorkDir}/.debug" ]; then rm -rf "${WorkDir}/.debug" fi else if [ "$noprompt" == "false" ]; then printf "\\n" yes_or_no "Cleanup temp folders? (choose 'n' if currently processing in other terminals)" && { if [ -d "${WorkDir}/.flocks" ]; then rm -rf "${WorkDir}/.flocks" fi if [ -d "${WorkDir}/.temp" ]; then rm -rf "${WorkDir}/.temp" fi if [ -d "${WorkDir}/.debug" ]; then rm -rf "${WorkDir}/.debug" fi } fi fi fi if [ "$autoclose" == "true" ]; then exit 0 fi } madAudit() { clear echo -e "${BLD}" echo -e "${PINK}:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:" echo -e ":${NC} ${GREEN}MAD${PINK} Audit${NC} : Reports usage of http & curl in scripts${PINK}${BLD} :" echo -e ":-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:${NC}\\n" if [ "$ScriptDir" == "$WorkDir" ]; then cd "$ScriptDir" readarray -d $'' arrFiles < <(find . -name "*.sh" -printf '%p\n' | sort -Vk1) for fil in "${arrFiles[@]}"; do maud_http=$(grep -n -vxE '[[:blank:]]*([#].*)?' $fil | grep --color='always' -Ei '(http|https):') maud_curl=$(grep -n -vxE '[[:blank:]]*([#].*)?' $fil | grep --color='always' -Ei 'curl') maud_torcurl=$(grep -n -vxE '[[:blank:]]*([#].*)?' $fil | grep -A 12 --color='always' -Ei 'tor_curl') echo -e "Files:" echo -e "${BLUE}${fil}${NC}" echo -e "" echo -e "" echo -e "${PINK}MAD Audit of http lines:${NC} (${GREEN}grep \"http:\" or \"https:\"${NC})" echo -e "_________________________________________________________________________" echo -e "$maud_http" echo -e "" echo -e "${PINK}MAD Audit of curl:${NC} (${GREEN}grep \"curl\"${NC})" echo -e "_________________________________________________________________________" echo -e "$maud_curl" echo -e "" echo -e "${PINK}MAD Audit of tor_curl (+12 lines after):${NC} (${GREEN}grep \"tor_curl\"${NC})" echo -e "_________________________________________________________________________" echo -e "$maud_torcurl" echo -e "" echo -e "" done else cd "$ScriptDir" readarray -d $'' arrFiles < <(find . -name "*.sh" -printf '%p\n' | sort -Vk1) cd "$WorkDir" readarray -d $'' arrFiles2 < <(find . -name "*.sh" -printf '%p\n' | sort -Vk1) for fil in "${arrFiles[@]}"; do maud_http=$(grep -n -vxE '[[:blank:]]*([#].*)?' $fil | grep --color='always' -Ei '(http|https):') maud_curl=$(grep -n -vxE '[[:blank:]]*([#].*)?' $fil | grep --color='always' -Ei 'curl') maud_torcurl=$(grep -n -vxE '[[:blank:]]*([#].*)?' $fil | grep -A 12 --color='always' -Ei 'tor_curl') echo -e "Files:" echo -e "${BLUE}${fil}${NC}" echo -e "" echo -e "" echo -e "${PINK}MAD Audit of http lines:${NC} (${GREEN}grep \"http:\" or \"https:\"${NC})" echo -e "_________________________________________________________________________" echo -e "$maud_http" echo -e "" echo -e "${PINK}MAD Audit of curl:${NC} (${GREEN}grep \"curl \"${NC})" echo -e "_________________________________________________________________________" echo -e "$maud_curl" echo -e "" echo -e "${PINK}MAD Audit of tor_curl (+12 lines after):${NC} (${GREEN}grep \"tor_curl\"${NC})" echo -e "_________________________________________________________________________" echo -e "$maud_torcurl" echo -e "" done for fil in "${arrFiles2[@]}"; do maud_http=$(grep -n -vxE '[[:blank:]]*([#].*)?' $fil | grep --color='always' -Ei '(http|https):') maud_curl=$(grep -n -vxE '[[:blank:]]*([#].*)?' $fil | grep --color='always' -Ei 'curl') maud_torcurl=$(grep -n -vxE '[[:blank:]]*([#].*)?' $fil | grep -A 12 --color='always' -Ei 'tor_curl') echo -e "Files:" echo -e "${BLUE}${fil}${NC}" echo -e "" echo -e "" echo -e "${PINK}MAD Audit of http lines:${NC} (${GREEN}grep \"http:\" or \"https:\"${NC})" echo -e "_________________________________________________________________________" echo -e "$maud_http" echo -e "" echo -e "${PINK}MAD Audit of curl:${NC} (${GREEN}grep \"curl\"${NC})" echo -e "_________________________________________________________________________" echo -e "$maud_curl" echo -e "" echo -e "${PINK}MAD Audit of tor_curl (+12 lines after):${NC} (${GREEN}grep \"tor_curl\"${NC})" echo -e "_________________________________________________________________________" echo -e "$maud_torcurl" echo -e "" done fi } madStatus() { local InputFile="$1" if [ "$arg1" == "status" ] ; then clear fi echo -e "${BLD}" echo -e "${PINK}:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:" echo -e ":${NC} ${GREEN}MAD${PINK} Status${NC} : Reports status of urls in file${PINK}${BLD} :" echo -e ":-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:${NC}\\n" if [ ! -f "${InputFile}" ] && [ -f "${WorkDir}/${InputFile}" ]; then InputFile="${WorkDir}/${InputFile}" fi if [ ! -f "${InputFile}" ]; then printf "Unable to read file %s! [ms]\\n" "${InputFile}" exit 1 fi lineCnt=1 listTodo="" cntTodo=0 listOk="" cntOk=0 listRetry="" cntRetry=0 listRemoved="" cntRemoved=0 listOther="" cntOther=0 if [ "${AutoCommentOnCompletion}" == "true" ] ; then while IFS= read -r line || [[ -n $line ]]; do if ! grep -Eqi '^(http|direct=http|#http|#direct=http)' <<< "${line}" > /dev/null ; then lineCnt=$((lineCnt+1)) continue fi tline=$line if grep -Eqi '^(http|direct=http)' <<< "${line}" > /dev/null ; then if grep -Eqi '^direct=' <<< "${line}" > /dev/null ; then tline=${tline/direct=/} fi if grep -Eqi '\|' <<< "${line}" > /dev/null ; then listTodo="$listTodo$lineCnt\t${NC}${tline%%\|*}${NC}\\n" else listTodo="$listTodo$lineCnt\t${NC}${tline//[$'\t\r\n']}${NC}\\n" fi cntTodo=$((cntTodo+1)) elif grep -Pqi '^(#http|#direct=http).*(?=.*#OK#)' <<< "${line}" > /dev/null ; then if grep -Eqi '^#direct=' <<< "${line}" > /dev/null ; then tline=${tline/direct=/} fi if grep -Eqi '\|' <<< "${tline}" > /dev/null ; then tline=${tline#*#} listOk="$listOk$lineCnt\t${GREEN}${tline%%\|*}${NC}\\n" else tline=${tline%%' #OK#'*} tline=${tline#*#} listOk="$listOk$lineCnt\t${GREEN}${tline//[$'\t\r\n']}${NC}\\n" fi cntOk=$((cntOk+1)) elif grep -Pqi '^(#http|#direct=http).*(?=.*#RETRY#)' <<< "${line}" > /dev/null ; then if grep -Eqi '^#direct=' <<< "${line}" > /dev/null ; then tline=${tline/direct=/} fi if grep -Eqi '\|' <<< "${line}" > /dev/null ; then tline=${tline##*/#} listRetry="$listRetry$lineCnt\t${YELLOW}${tline%%\|*}${NC}\\n" else tline=${tline##*/#} tline=${tline%%\ #RETRY#*} listRetry="$listRetry$lineCnt\t${YELLOW}${tline//[$'\t\r\n']}${NC}\\n" fi cntRetry=$((cntRetry+1)) elif grep -Pqi '^(#http|#direct=http).*(?=.*#REMOVED#)' <<< "${line}" > /dev/null ; then if grep -Eqi '^#direct=' <<< "${line}" > /dev/null ; then tline=${tline/direct=/} fi if grep -Eqi '\|' <<< "${line}" > /dev/null ; then tline=${tline##*/#} listRemoved="$listRemoved$lineCnt\t${RED}${tline%%\|*}${NC}\\n" else tline=${tline##*/#} tline=${tline%%\ #REMOVED#*} listRemoved="$listRemoved$lineCnt\t${RED}${tline//[$'\t\r\n']}${NC}\\n" fi cntRemoved=$((cntRemoved+1)) elif grep -Pqi '^(#http|#direct=http).*(?=.*#FAIL#)' <<< "${line}" > /dev/null ; then if grep -Eqi '^#direct=' <<< "${line}" > /dev/null ; then tline=${tline/direct=/} fi if grep -Eqi '\|' <<< "${line}" > /dev/null ; then tline=${tline##*/#} listOther="$listOther$lineCnt\t${RED}${tline%%\|*}${NC}\\n" else tline=${tline##*/#} tline=${tline%%\ #BAD-URL#*} listOther="$listOther$lineCnt\t${RED}${tline//[$'\t\r\n']}${NC}\\n" fi cntOther=$((cntOther+1)) elif grep -Pqi '^(#http|#direct=http).*(?=.*#BAD-URL#)' <<< "${line}" > /dev/null ; then if grep -Eqi '^#direct=' <<< "${line}" > /dev/null ; then tline=${tline/direct=/} fi if grep -Eqi '\|' <<< "${line}" > /dev/null ; then tline=${tline##*/#} listOther="$listOther$lineCnt\t${RED}${tline%%\|*}${NC}\\n" else tline=${tline##*/#} tline=${tline%%\ #BAD-URL#*} listOther="$listOther$lineCnt\t${RED}${tline//[$'\t\r\n']}${NC}\\n" fi cntOther=$((cntOther+1)) elif grep -Pqi '^(#http|#direct=http).*(?=.*#PASSWORD#)' <<< "${line}" > /dev/null ; then if grep -Eqi '^#direct=' <<< "${line}" > /dev/null ; then tline=${tline/direct=/} fi if grep -Eqi '\|' <<< "${line}" > /dev/null ; then tline=${tline##*/#} listOther="$listOther$lineCnt\t${RED}${tline%%\|*}${NC}\\n" else tline=${tline##*/#} tline=${tline%%\ #PASSWORD#*} listOther="$listOther$lineCnt\t${RED}${tline//[$'\t\r\n']}${NC}\\n" fi cntOther=$((cntOther+1)) fi lineCnt=$((lineCnt+1)) done < ${InputFile} if [ -d "${WorkDir}/downloads" ]; then size_complete=$(du -hs --apparent-size "${WorkDir}/downloads" | awk '{print $1}') else size_complete="" fi echo -e "${PINK}Line #\tUrl${NC}" echo -e "------------------------------------------------" echo -e "[Todo] (${BLUE}$cntTodo${NC}) - URLs to process" echo -e "$listTodo" if [ ! -z $size_complete ]; then echo -e "[OK] (${GREEN}$cntOk${NC}) - Downloads completed [size: $size_complete]" else echo -e "[OK] (${GREEN}$cntOk${NC}) - Downloads completed" fi echo -e "$listOk" echo -e "[Retry] (${YELLOW}$cntRetry${NC}) - Downloads that failed and need a reset and retry" echo -e "$listRetry" echo -e "[Removed] (${RED}$cntRemoved${NC}) - Downloads that have been removed from the host" echo -e "$listRemoved" echo -e "[Other] (${RED}$cntOther${NC}) - Downloads with a bad url, require a password, or are unavailable" echo -e "$listOther" else while IFS= read -r line || [[ -n $line ]]; do if ! grep -Eqi '^(http|direct=http|#http|#direct=http)' <<< "${line}" > /dev/null ; then continue fi tline=$line if grep -q -e '^(http|direct=http)' <<< "${line}" > /dev/null ; then if grep -q -e '^direct=' <<< "${line}" > /dev/null ; then tline=${tline/direct=/} fi if grep -q -e '\|' <<< "${line}" > /dev/null ; then listTodo="$listTodo$lineCnt\t${NC}${tline%%\|*}${NC}\\n" else listTodo="$listTodo$lineCnt\t${NC}${tline//[$'\t\r\n']}${NC}\\n" fi cntTodo=$((cntTodo+1)) elif grep -q -P '^(#http|#direct=http)' <<< "${line}" > /dev/null ; then if grep -q -e '^#direct=' <<< "${line}" > /dev/null ; then tline=${tline/direct=/} fi if grep -q -e '\|' <<< "${line}" > /dev/null ; then tline=${tline#*#} listOk="$listOk$lineCnt\t${GREEN}${tline%%\|*}${NC}\\n" else tline=${tline%%' #OK#'*} tline=${tline#*#} listOk="$listOk$lineCnt\t${GREEN}${tline//[$'\t\r\n']}${NC}\\n" fi cntOk=$((cntOk+1)) fi done < ${InputFile} if [ -d "${WorkDir}/downloads" ]; then size_complete=$(du -hs --apparent-size "${WorkDir}/downloads" | awk '{print $1}') else size_complete="" fi echo -e "${PINK}Line #\tUrl${NC}" echo -e "------------------------------------------------" echo -e "[Todo] ($cntTodo${NC}) - URLs to process" echo -e "$listTodo" if [ ! -z $size_complete ]; then echo -e "[OK] (${GREEN}$cntOk${NC}) - URLs completed (commented out) [size: $size_complete]" else echo -e "[OK] (${GREEN}$cntOk${NC}) - URLs completed (commented out)" fi echo -e "$listOk" fi exit 0 } madStatusUploads() { local InputFile="$1" if [ "$arg1" == "status" ] ; then clear fi echo -e "${BLD}" echo -e "${PINK}:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:" echo -e ":${NC} ${GREEN}MAD${PINK} Status${NC} : Report status of uploads in file${PINK}${BLD} :" echo -e ":-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:${NC}\\n" if [ ! -f "${InputFile}" ] && [ -f "${WorkDir}/${InputFile}" ]; then InputFile="${WorkDir}/${InputFile}" fi if [ ! -f "${InputFile}" ]; then printf "Unable to read file %s! [ms]\\n" "${InputFile}" exit 1 fi lineCnt=1 listTodo="" cntTodo=0 listOk="" cntOk=0 listRetry="" cntRetry=0 listOther="" cntOther=0 if [ "${AutoCommentOnCompletion}" == "true" ] ; then while IFS= read -r line || [[ -n $line ]]; do if grep -Eqi '^$|^ ' <<< "${line}" > /dev/null || grep -Eqvi '\|' <<< "${line}" > /dev/null; then lineCnt=$((lineCnt+1)) continue fi tline="$line" if grep -Eqi '^[^#]' <<< "${line}" > /dev/null && grep -Eqi '\|' <<< "${line}" > /dev/null ; then tline="${tline//|/ [}""]" listTodo="$listTodo$lineCnt\t${NC}${tline//[$'\t\r\n']}${NC}\\n" cntTodo=$((cntTodo+1)) elif grep -Pqi '^#.*(?=#OK#)' <<< "${line}" > /dev/null ; then tline="${tline#*\#}" tline="${tline//|/ [}" tline="${tline// #OK#/]}" listOk="$listOk$lineCnt\t${NC}${GREEN}${tline//[$'\t\r\n']}${NC}\\n" cntOk=$((cntOk+1)) elif grep -Pqi '^#.*(?=#RETRY#)' <<< "${line}" > /dev/null ; then tline="${tline#*\#}" tline="${tline//|/ [}" tline="${tline// #RETRY#/]}" listRetry="$listRetry$lineCnt\t${NC}${YELLOW}${tline//[$'\t\r\n']}${NC}\\n" cntRetry=$((cntRetry+1)) elif grep -Pqi '^#.*(?=#FAIL#)' <<< "${line}" > /dev/null ; then tline="${tline#*\#}" tline="${tline//|/ [}" tline="${tline// #FAIL#/]}" listOther="$listOther$lineCnt\t${RED}${tline//[$'\t\r\n']}${NC}\\n" cntOther=$((cntOther+1)) elif grep -Pqi '^#.*(?=#BAD-LINE#)' <<< "${line}" > /dev/null ; then tline="${tline#*\#}" tline="${tline//|/ [}" tline="${tline// #BAD-LINE#/]}" listOther="$listOther$lineCnt\t${RED}${tline//[$'\t\r\n']}${NC}\\n" cntOther=$((cntOther+1)) fi lineCnt=$((lineCnt+1)) done < ${InputFile} echo -e "${PINK}Line #\tFilename\tHostCode${NC}" echo -e "------------------------------------------------" echo -e "[Todo] (${BLUE}$cntTodo${NC}) - Uploads to process" echo -e "$listTodo" echo -e "[OK] (${GREEN}$cntOk${NC}) - Uploads completed" echo -e "$listOk" echo -e "[Retry] (${YELLOW}$cntRetry${NC}) - Uploads that failed and need a reset and retry" echo -e "$listRetry" echo -e "[Fail] (${RED}$cntOther${NC}) - Uploads that failed are have a bad line in file" echo -e "$listOther" else while IFS= read -r line || [[ -n $line ]]; do if grep -Eqi '^$|^ ' <<< "${line}" > /dev/null || grep -Eqvi '\|' <<< "${line}" > /dev/null; then lineCnt=$((lineCnt+1)) continue fi tline=$line if grep -Eqi '^[^#]' <<< "${line}" > /dev/null && grep -Eqi '\|' <<< "${line}" > /dev/null ; then listTodo="$listTodo$lineCnt\t${NC}${tline//[$'\t\r\n']}${NC}\\n" cntTodo=$((cntTodo+1)) elif grep -Pqi '^#.*(?=\|)' <<< "${line}" > /dev/null ; then listOk="$listOk$lineCnt\t${GREEN}${tline//[$'\t\r\n']}${NC}\\n" cntOk=$((cntOk+1)) fi done < ${InputFile} echo -e "${PINK}Line #\tFilename\tHostCode${NC}" echo -e "------------------------------------------------" echo -e "[Todo] ($cntTodo${NC}) - Uploads to process" echo -e "$listTodo" echo -e "[OK] (${GREEN}$cntOk${NC}) - Uploads completed (commented out)" echo -e "$listOk" fi exit 0 } madHostDetails() { clear echo -e "${BLD}" echo -e "${PINK}:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:" echo -e ":${NC} ${GREEN}MAD${PINK} Host Details${NC} :: Available hosts & functions${PINK}${BLD} :" echo -e ":-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:${NC}\\n" echo -e "" echo -e "${YELLOW}Download Hosts:${NC}" echo -e "----------------------------------------------------------------${NC}" echo -e "" for fil in "${ScriptDir}"/hosts/*.sh ; do if [ -f "$fil" ]; then tfilename="${fil##*/}" if [[ "$tfilename" == "up_"* ]] ; then continue fi local _hostcode=$(grep -oPi -m 1 '(?<=^HostCode='"'"').*?(?='"'"')' "$fil") local _hostnick=$(grep -oPi -m 1 '(?<=^HostNick='"'"').*?(?='"'"')' "$fil") local _hostfuncprefix=$(grep -oPi -m 1 '(?<=^HostFuncPrefix='"'"').*?(?='"'"')' "$fil") local _hosturls=$(grep -oPi -m 1 '(?<=^HostUrls='"'"').*?(?='"'"')' "$fil") local _hostdomainregex=$(grep -oPi -m 1 '(?<=^HostDomainRegex='"'"').*?(?='"'"')' "$fil") local _currHostsFuncsUsed=$(grep '()' "${fil}" | awk '!/#/ {print $1}') local chfuCleaned=${_currHostsFuncsUsed//$'\n'/, } echo -e "[${YELLOW}$_hostcode${NC}] ${GREEN}$_hostnick${NC}" echo -e "Module: ./hosts/${BLUE}${tfilename}${NC}" echo -e "Urls: ${NC}$_hosturls${NC}" echo -e "Regex: ${RED}${_hostdomainregex}${NC}" echo -e "Functions: ${PINK}$chfuCleaned${NC}" echo -e "Entrypoint: ${BLUE}${_hostfuncprefix}_DownloadFile()${NC}" echo -e "" fi done echo -e "" echo -e "" echo -e "${YELLOW}Upload Hosts:${NC}" echo -e "----------------------------------------------------------------${NC}" echo -e "" for fil in "${ScriptDir}"/hosts/up_*.sh ; do if [ -f "$fil" ]; then tfilename="${fil##*/}" local _hostcode=$(grep -oPi -m 1 '(?<=^HostCode='"'"').*?(?='"'"')' "$fil") local _hostnick=$(grep -oPi -m 1 '(?<=^HostNick='"'"').*?(?='"'"')' "$fil") local _hostfuncprefix=$(grep -oPi -m 1 '(?<=^HostFuncPrefix='"'"').*?(?='"'"')' "$fil") local _info=$(grep -oPi -m 1 '(?<=^#! Info: ).*$' "$fil") local _msize=$(grep -oPi -m 1 '(?<=^#! MaxSize: ).*$' "$fil") local _expire=$(grep -oPi -m 1 '(?<=^#! Expire: ).*$' "$fil") local _currHostsFuncsUsed=$(grep '()' "${fil}" | awk '!/#/ {print $1}') local chfuCleaned=${_currHostsFuncsUsed//$'\n'/, } echo -e "[${YELLOW}$_hostcode${NC}] ${GREEN}$_hostnick${NC}" echo -e "Module: ./hosts/${BLUE}${tfilename}${NC}" echo -e "Info: ${NC}$_info${NC}" echo -e "MaxSize: ${BLUE}${_msize}${NC}" echo -e "Expire: ${RED}${_expire}${NC}" echo -e "Functions: ${PINK}$chfuCleaned${NC}" echo -e "Entrypoint: ${BLUE}${_hostfuncprefix}_UploadFile()${NC}" echo -e "" fi done } madPluginDetails() { clear echo -e "${BLD}" echo -e "${PINK}:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:" echo -e ":${NC} ${GREEN}MAD${PINK} Plugin Details${NC} :: Available plugins & functions${PINK}${BLD} :" echo -e ":-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:${NC}\\n" for fil in "${ScriptDir}"/plugins/*.sh ; do if [ -f "$fil" ]; then tfilename="${fil##*/}" local _description=$(grep -oPzi '(?<=Desc: ).*\n.*(?=#.*Usage:)' "$fil" | tr '\0' ' ') local _currPluginFuncsUsed=$(grep '()' "${fil}" | awk '!/#/ {print $1}') local chfuCleaned=${_currPluginFuncsUsed//$'\n'/, } echo -e "Plugin: ${YELLOW}${tfilename}${NC}${NC}" echo -e "Functions: ${PINK}$chfuCleaned${NC}" if [[ ! -z $_description ]]; then echo -e "Desc: ${BLUE}${_description}${NC}" else echo -e "" fi fi done } clipboard_monitor() { InputFile="$1" if [ ! -f "${InputFile}" ] && [ -f "${WorkDir}/${InputFile}" ]; then InputFile="${WorkDir}/${InputFile}" fi if [ ! -f "${InputFile}" ]; then touch "${InputFile}" fi if ! hash xclip 2>/dev/null; then echo -e "${YELLOW}xclip is required for clipmon. Use \"sudo apt install xclip\" to install.${NC}" exit 1 fi echo -e "${BLD}" echo -e "${GREEN}MAD${PINK} Clipboard Monitor: ${GREEN}Enabled${NC}" echo -e "| Capture File: ${BLUE}${InputFile}${NC}" echo -e "| Log file: ${WorkDir}/.temp/_clipmonlog.txt${NC}" echo -e clipold=$(xclip -o 2>/dev/null) exitcode=$? if ! [ $exitcode -eq 0 ]; then echo "" | xclip clipold=$(xclip -o 2>/dev/null) fi while true; do clipnew=$(xclip -o 2>/dev/null) if ! [ "$clipold" == "$clipnew" ]; then mkdir -p "${WorkDir}/.temp" echo "$clipnew" >> "${WorkDir}/.temp/_clipmonlog.txt" clipold="$clipnew" sed 's/^[[:space:]]*// ; s/[[:space:]]*$//' <<< "${clipnew}" | while IFS= read -r line do line="${line//[$'\t\r\n']}" isHostMatchFound=false _hostcode="" _hostnick="" _hostfuncprefix="" _hostdomainregex="" readarray -d '@' -t arrHostAndDomainRegexes <<< "${ListHostAndDomainRegexes}" for hline in "${arrHostAndDomainRegexes[@]}"; do if [[ "$hline" == "" ]] || [[ ! "$hline" == *":"* ]]; then continue fi chnk1="${hline%%:*}" _hostcode=$(echo $chnk1|cut -f2 -d '/') _hostnick=$(echo $chnk1|cut -f3 -d '/') _hostfuncprefix=$(echo $chnk1|cut -f4 -d '/') _hostdomainregex="${hline#*\:}" if [[ "$line" =~ $_hostdomainregex ]]; then isHostMatchFound=true break fi done if [ "$isHostMatchFound" == "true" ]; then download_url=$(urlencode_literal_grouped_case_urlendingonly "$line") echo -e "${GREEN}$_hostnick url found:${NC} $line${NC}" echo -e "$line" >> ${InputFile} fi done fi sleep 1s done } lstOnLoad="" lstBeginProcessing="" lstPreProcessUrl="" lstPostSuccessfulDownload="" lstPostFailedDownload="" lstPostFailRetryDownload="" lstDoneProcessingAllUrls="" lstPostSuccessfulUpload="" lstPostFailedUpload="" lstPostFailRetryUpload="" lstDoneProcessingAllUploads="" OnLoad() { local pScriptSourceDir="$1" local pScriptArguments="$2" if [[ -z "$lstOnLoad" ]]; then return 0 fi isHookMatch=false hookReturnCode=0 readarray -d '@' -t arrHookList <<< "${lstOnLoad}" for hookFunc in "${arrHookList[@]}"; do if [[ "$hookFunc" == "" ]]; then continue fi $hookFunc "$pScriptSourceDir" "$pScriptArguments" ret=$? if ((ret != 0)) && ((ret >hookReturnCode)); then hookReturnCode=$ret fi done return $hookReturnCode } BeginProcessing() { local pInputFile="$1" if [[ -z "$lstBeginProcessing" ]]; then return 0 fi isHookMatch=false hookReturnCode=0 readarray -d '@' -t arrHookList <<< "${lstBeginProcessing}" for hookFunc in "${arrHookList[@]}"; do if [[ "$hookFunc" == "" ]]; then continue fi $hookFunc "$pInputFile" ret=$? if ((ret != 0)) && ((ret >hookReturnCode)); then hookReturnCode=$ret fi done return $hookReturnCode } PreProcessUrl() { local pFullLine="$1" local pUrlOnly="${1%%\ *}" if grep -Eqi '\|' <<< "${pUrlOnly}" ; then pUrlOnly="${pUrlOnly%%\|*}" fi if grep -Eqi '^direct=' <<< "${pUrlOnly}" ; then pUrlOnly="${pUrlOnly%%\=*}" fi if [[ -z "$lstPreProcessUrl" ]]; then return 0 fi isHookMatch=false hookReturnCode=0 readarray -d '@' -t arrHookList <<< "${lstPreProcessUrl}" for hookFunc in "${arrHookList[@]}"; do if [[ "$hookFunc" == "" ]]; then continue fi $hookFunc "$pFullLine" "$pUrlOnly" ret=$? if ((ret != 0)) && ((ret >hookReturnCode)); then hookReturnCode=$ret fi done return $hookReturnCode } PostSuccessfulDownload() { local pRemoteUrl="$1" local pFilePath="$2" local pFileName="$3" local pFolderName="$4" local pFileSize="$5" isHookMatch=false hookReturnCode=0 if [[ -z "$lstPostSuccessfulDownload" ]]; then return 0 fi readarray -d '@' -t arrHookList <<< "${lstPostSuccessfulDownload}" for hookFunc in "${arrHookList[@]}"; do if [[ -z "$hookFunc" ]]; then continue fi $hookFunc "$pRemoteUrl" "$pFilePath" "$pFileName" "$pFolderName" "$pFileSize" ret=$? if ((ret != 0)) && ((ret >hookReturnCode)); then hookReturnCode=$ret fi done return $hookReturnCode } PostFailedDownload() { local pRemoteUrl="$1" local pMessage1="$2" local pMessage2="$3" isHookMatch=false hookReturnCode=0 if [[ -z "$lstPostFailedDownload" ]]; then return 0 fi readarray -d '@' -t arrHookList <<< "${lstPostFailedDownload}" for hookFunc in "${arrHookList[@]}"; do if [[ -z "$hookFunc" ]]; then continue fi $hookFunc "$pRemoteUrl" "$pMessage1" "$pMessage2" ret=$? if ((ret != 0)) && ((ret >hookReturnCode)); then hookReturnCode=$ret fi done return $hookReturnCode } PostFailRetryDownload() { local pRemoteUrl="$1" local pFilePath="$2" local pFileName="$3" local pFolderName="$4" isHookMatch=false hookReturnCode=0 if [[ -z "$lstPostFailRetryDownload" ]]; then return 0 fi readarray -d '@' -t arrHookList <<< "${lstPostFailRetryDownload}" for hookFunc in "${arrHookList[@]}"; do if [[ -z "$hookFunc" ]]; then continue fi $hookFunc "$pRemoteUrl" "$pFilePath" "$pFileName" "$pFolderName" ret=$? if ((ret != 0)) && ((ret >hookReturnCode)); then hookReturnCode=$ret fi done return $hookReturnCode } DoneProcessingAllUrls() { local pInputFile="$1" isHookMatch=false hookReturnCode=0 if [[ -z "$lstDoneProcessingAllUrls" ]]; then return 0 fi readarray -d '@' -t arrHookList <<< "${lstDoneProcessingAllUrls}" for hookFunc in "${arrHookList[@]}"; do if [[ -z "$hookFunc" ]]; then continue fi $hookFunc "$pInputFile" ret=$? if ((ret != 0)) && ((ret >hookReturnCode)); then hookReturnCode=$ret fi done return $hookReturnCode } PostSuccessfulUpload() { local pFilePath="$1" local pHostCode="$2" local pFileName="$3" local pFileSize="$4" local pDownloadLink="$5" isHookMatch=false hookReturnCode=0 if [[ -z "$lstPostSuccessfulUpload" ]]; then return 0 fi readarray -d '@' -t arrHookList <<< "${lstPostSuccessfulUpload}" for hookFunc in "${arrHookList[@]}"; do if [[ -z "$hookFunc" ]]; then continue fi $hookFunc "$pFilePath" "$pHostCode" "$pFileName" "$pFileSize" "$pDownloadLink" ret=$? if ((ret != 0)) && ((ret >hookReturnCode)); then hookReturnCode=$ret fi done return $hookReturnCode } PostFailedUpload() { local pFilePath="$1" local pHostCode="$2" local pMessage1="$3" local pMessage2="$4" isHookMatch=false hookReturnCode=0 if [[ -z "$lstPostFailedUpload" ]]; then return 0 fi readarray -d '@' -t arrHookList <<< "${lstPostFailedUpload}" for hookFunc in "${arrHookList[@]}"; do if [[ -z "$hookFunc" ]]; then continue fi $hookFunc "$pFilePath" "$pHostCode" "$pMessage1" "$pMessage2" ret=$? if ((ret != 0)) && ((ret >hookReturnCode)); then hookReturnCode=$ret fi done return $hookReturnCode } PostFailRetryUpload() { local pFilePath="$1" local pHostCode="$2" local pMessage1="$3" local pMessage2="$4" isHookMatch=false hookReturnCode=0 if [[ -z "$lstPostFailRetryUpload" ]]; then return 0 fi readarray -d '@' -t arrHookList <<< "${lstPostFailRetryUpload}" for hookFunc in "${arrHookList[@]}"; do if [[ -z "$hookFunc" ]]; then continue fi $hookFunc "$pFilePath" "$pHostCode" "$pMessage1" "$pMessage2" ret=$? if ((ret != 0)) && ((ret >hookReturnCode)); then hookReturnCode=$ret fi done return $hookReturnCode } DoneProcessingAllUploads() { local pInputFile="$1" isHookMatch=false hookReturnCode=0 if [[ -z "$lstDoneProcessingAllUploads" ]]; then return 0 fi readarray -d '@' -t arrHookList <<< "${lstDoneProcessingAllUploads}" for hookFunc in "${arrHookList[@]}"; do if [[ -z "$hookFunc" ]]; then continue fi $hookFunc "$pInputFile" ret=$? if ((ret != 0)) && ((ret >hookReturnCode)); then hookReturnCode=$ret fi done return $hookReturnCode } CheckDownloadExists () { local cde_remote_url=$1 local cde_MoveToFolder=$2 local cde_filecnt=$3 local cde_filename=$4 local cde_file_path=$5 local cde_completed_location=$6 local cde_flockDownload="${WorkDir}/.flocks/${cde_filename//[^a-zA-Z0-9\.\_\-]/}.flock" if [[ -e "$cde_flockDownload" ]]; then fContents=$(cat $cde_flockDownload) fContents=${fContents//[$'\t\r\n']} if [ "$AutoRenameDuplicateFilenames" == "true" ]; then if [ "$fContents" == "${cde_remote_url//[^a-zA-Z0-9]/}" ]; then echo -e "${YELLOW}[SKIP]${NC} Lock exists for filename. Is it downloading in another terminal?${NC}" echo -e "${YELLOW}File: ${NC}${cde_filename}" echo -e "${YELLOW}Lock: ${NC}./.flocks/${cde_filename//[^a-zA-Z0-9\.\_\-]/}.flock" echo -e "URL: ${cde_remote_url}" printf "%s\\n" "================================================================================" fileAlreadyDone=true return 0 else echo -e "${YELLOW}[RENAME]${NC} Lock exists for filename. Renaming download filename.${NC}" echo -e "${YELLOW}File: ${NC}${cde_filename}" echo -e "${YELLOW}Lock: ${NC}./.flocks/${cde_filename//[^a-zA-Z0-9\.\_\-]/}.flock" echo -e "URL: ${cde_remote_url}" printf "%s\\n" "================================================================================" renameDuplicateDownload "${cde_remote_url}" "${cde_filename}" fileAlreadyDone=true return 0 fi else echo -e "${YELLOW}[SKIP]${NC} Lock exists for filename. Is it downloading in another terminal?${NC}" echo -e "${YELLOW}File: ${NC}${cde_filename}" echo -e "${YELLOW}Lock: ${NC}./.flocks/${cde_filename//[^a-zA-Z0-9\.\_\-]/}.flock" echo -e "URL: ${cde_remote_url}" printf "%s\\n" "================================================================================" if [ ! "$fContents" == "${cde_remote_url//[^a-zA-Z0-9]/}" ]; then skipUrlDownload "${cde_remote_url}" "${cde_filename}" "./.flocks/${cde_filename//[^a-zA-Z0-9\.\_\-]/}.flock" "$cde_flockDownload" fi fileAlreadyDone=true return 0 fi fi if [[ -e "${cde_file_path}" ]]; then existing_file_size=$(stat --format="%s" "${cde_file_path}" | tr -d '[:space:]') if [[ "${existing_file_size}" -eq "${file_size_bytes}" ]]; then echo -e "${GREEN}File exists in downloads and the size matches the expected size.\nNo need to re-download.${NC}" mkdir -p "${WorkDir}/downloads" if [ ! "${cde_MoveToFolder}" == "" ] ; then if [ ! -d "${cde_completed_location}${cde_MoveToFolder}" ]; then mkdir -p "${cde_completed_location}${cde_MoveToFolder}" fi mv "${cde_file_path}" "${cde_completed_location}${cde_MoveToFolder}/" printf "\\n%s\\n" "================================================================================" echo -e "${RED}❤${GREEN}[OK] Active download exists and is complete.${NC}" echo -e "${GREEN}File: ${NC}#${cde_filecnt}, ${cde_filename}" echo -e "URL: ${cde_remote_url}" echo -e "Location: \"./downloads/${cde_MoveToFolder}/${cde_filename}\"" printf "%s\\n" "================================================================================" successDownloadExists "${cde_remote_url}" "${cde_filename}" "${cde_MoveToFolder}" "${existing_file_size}" "${cde_completed_location}${cde_MoveToFolder}/${cde_filename}" fileAlreadyDone=true return 0 else mv "${cde_file_path}" "${cde_completed_location}/" printf "\\n%s\\n" "================================================================================" echo -e "${RED}❤${GREEN}[OK] Active download exists and is complete.${NC}" echo -e "${GREEN}File: ${NC}#${cde_filecnt}, ${cde_filename}" echo -e "URL: ${cde_remote_url}" echo -e "Location: \"./downloads/${cde_filename}\"" printf "%s\\n" "================================================================================" successDownloadExists "${cde_remote_url}" "${cde_filename}" "./" "${existing_file_size}" "${cde_completed_location}${cde_filename}" fileAlreadyDone=true return 0 fi return 0 elif [ ! -z "$file_size_bytes" ] && [[ "${existing_file_size}" -gt "${file_size_bytes}" ]]; then echo -e "${RED}ERROR: File exists in downloads folder but is larger than expected.${NC}\nThis could be due to several instances saving the same file, an old download using the same name, or host experiencing a temporary issue." fileExistsButSizeTooLargeDownload "${cde_remote_url}" "${cde_filename}" "${existing_file_size}" "${file_size_bytes}" exitDownloadError=true return 0 fi fi if [ ! "${cde_MoveToFolder}" == "" ] ; then if [[ -e "${cde_completed_location}${cde_MoveToFolder}/${cde_filename}" ]] ; then if [ "$AutoRenameDuplicateFilenames" == "true" ]; then printf "\\n%s\\n" "================================================================================" echo -e "${RED}❤${GREEN}[RENAME] Download exists and is complete. Renaming download filename.${NC}" echo -e "${GREEN}File: ${NC}#${cde_filecnt}, ${cde_filename}" echo -e "URL: ${cde_remote_url}" echo -e "Location: \"./downloads/${cde_filename}\"" printf "%s\\n" "================================================================================" renameDuplicateDownload "${cde_remote_url}" "${cde_filename}" fileAlreadyDone=true return 0 else printf "\\n%s\\n" "================================================================================" echo -e "${RED}❤${GREEN}[OK] Download exists and is complete.${NC}" echo -e "${GREEN}File: ${NC}#${cde_filecnt}, ${cde_filename}" echo -e "URL: ${cde_remote_url}" echo -e "Location: \"./downloads/${cde_MoveToFolder}/${cde_filename}\"" printf "%s\\n" "================================================================================" successDownloadExists "${cde_remote_url}" "${cde_filename}" "${cde_MoveToFolder}" "${existing_file_size}" "${cde_completed_location}${cde_MoveToFolder}/${cde_filename}" fileAlreadyDone=true return 0 fi fi else if [[ -e "${cde_completed_location}${cde_filename}" ]] ; then if [ "$AutoRenameDuplicateFilenames" == "true" ]; then printf "\\n%s\\n" "================================================================================" echo -e "${RED}❤${GREEN}[RENAME] Download exists and is complete. Renaming download filename.${NC}" echo -e "${GREEN}File: ${NC}#${cde_filecnt}, ${cde_filename}" echo -e "URL: ${cde_remote_url}" echo -e "Location: \"./downloads/${cde_filename}\"" printf "%s\\n" "================================================================================" renameDuplicateDownload "${cde_remote_url}" "${cde_filename}" fileAlreadyDone=true return 0 else printf "\\n%s\\n" "================================================================================" echo -e "${RED}❤${GREEN}[OK] Download exists and is complete.${NC}" echo -e "${GREEN}File: ${NC}#${cde_filecnt}, ${cde_filename}" echo -e "URL: ${cde_remote_url}" echo -e "Location: \"./downloads/${cde_filename}\"" printf "%s\\n" "================================================================================" successDownloadExists "${cde_remote_url}" "${cde_filename}" "./" "${existing_file_size}" "${cde_completed_location}${cde_filename}" fileAlreadyDone=true return 0 fi fi fi return 1 } CheckNoHtml() { local cde_remote_url=$1 local cde_filename=$2 local cde_file_path=$3 if [ -f "${cde_file_path}" ] ; then badHtml=$(grep -aoPi '^.*(||content-type|:error|not found|too many connections).*$' "${cde_file_path}") if [ "$badHtml" == "" ]; then return 0 else if [ "${DebugAllEnabled}" == "true" ] ; then cp "$cde_file_path" "${WorkDir}/.debug/$cde_filename.htmldebug.txt" fi if [ "${DebugAllEnabled}" == "true" ] ; then echo -e "" printf "${YELLOW}HTML found:${NC} Details in ./.debug/_err.log" mkdir -p "${WorkDir}/.debug" echo -e "-----------------------------------------------------------------" >> "${WorkDir}/.debug/_err.log" echo -e "DateTime: `date +%y%m%d-%H%M%S`" >> "${WorkDir}/.debug/_err.log" echo -e "Message: Bad html found" >> "${WorkDir}/.debug/_err.log" echo -e "RemoteUrl: ${cde_remote_url}" >> "${WorkDir}/.debug/_err.log" echo -e "Filename: ${cde_filename}" >> "${WorkDir}/.debug/_err.log" echo -e "FilePath: ${cde_file_path}" >> "${WorkDir}/.debug/_err.log" echo -e "BadHtml: ${badHtml}" >> "${WorkDir}/.debug/_err.log" echo -e "" fi return 1 fi else return 0 fi } CheckFileSize() { local cde_remote_url=$1 local cde_file_size_bytes=$2 if [ -z $cde_file_size_bytes ]; then return 1 fi if (( cde_file_size_bytes < MinimumAllowedFilesize )); then echo -e "${RED}| ERROR: File less than MinimumAllowedFilesize${NC} (${MinimumAllowedFilesize})${NC}" failedDownload "${remote_url}" "File less than MinimumAllowedFilesize. (min: ${MinimumAllowedFilesize})" "${cde_file_size_bytes}" fileAlreadyDone=true return 0 fi return 1 } ProcessCompletedDownload() { local cde_remote_url=$1 local cde_MoveToFolder=$2 local cde_filecnt=$3 local cde_filename=$4 local cde_file_size_bytes=$5 local cde_completed_location=$6 local cde_inflight_path=$7 completed_location="${WorkDir}/downloads/" if [ ! -d "${WorkDir}/downloads" ]; then mkdir -p "${WorkDir}/downloads" fi if [ ! "${cde_MoveToFolder}" == "" ] ; then if [ ! -d "${cde_completed_location}${cde_MoveToFolder}" ]; then mkdir -p "${cde_completed_location}${cde_MoveToFolder}" fi mv "${cde_inflight_path}" "${cde_completed_location}${cde_MoveToFolder}/" printf "\\n%s\\n" "================================================================================" echo -e "${RED}❤${GREEN}[OK] Download complete.${NC}" echo -e "${GREEN}File: ${NC}#${cde_filecnt}, ${cde_filename}" echo -e "URL: ${cde_remote_url}" echo -e "Location: \"./downloads/${cde_MoveToFolder}/${cde_filename}\"" printf "%s\\n" "================================================================================" successDownload "${cde_remote_url}" "${cde_filename}" "${cde_MoveToFolder}" "${cde_file_size_bytes}" "${cde_completed_location}${cde_MoveToFolder}/${cde_filename}" else mv "${cde_inflight_path}" "${WorkDir}/downloads/" printf "\\n%s\\n" "================================================================================" echo -e "${RED}❤${GREEN}[OK] Download complete.${NC}" echo -e "${GREEN}File: ${NC}#${cde_filecnt}, ${cde_filename}" echo -e "URL: ${cde_remote_url}" echo -e "Location: \"./downloads/${cde_filename}\"" printf "%s\\n" "================================================================================" successDownload "${cde_remote_url}" "${cde_filename}" "./" "${cde_file_size_bytes}" "${cde_completed_location}${cde_filename}" fi } MadUploadFromFileTxt() { local pInFile="$1" mkdir -p "${WorkDir}/uploads" clear echo -e "${BLD}" echo -e "${PINK}:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:" echo -e ":${NC} ${GREEN}MAD${PINK} Uploader${NC} :: Transfer files in uploads.txt${PINK}${BLD} :" echo -e ":-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:${NC}\\n" OnLoad $0 $@ if ((torPort >= 9050 && torPort <= 9150)); then printf "TorIp: ${GREEN}$TorIp${NC}, Tor listening on port ${GREEN}$torPort${NC}, " else printf "TorIp: ${RED}$TorIp${NC}, ${RED}Tor port not found !!${NC}.\\n" printf "Ensure Tor is setup and listening on a port between 9050 and 9150. Exiting...\\n" exit 1 fi if [ "${UseTorCurlImpersonate}" == "true" ]; then printf "client: ${GREEN}Tor${NC} + ${BLUE}curl_impersonate${NC}\\n" else printf "client: ${GREEN}Tor${NC} + ${GREEN}curl${NC}\\n" fi printf "ConnectTimeout: ${GREEN}$ConnectTimeout${NC}, UploadRetries: ${GREEN}$MaxUploadRetries${NC}, " if [ "${RateMonitorEnabled}" == "true" ]; then printf "RateMonitor: ${GREEN}${RateMonitorEnabled}${NC}" else printf "RateMonitor: ${GREY}${RateMonitorEnabled}${NC}" fi printf "\\n" echo -e "DefaultUploadHosts: ${BLUE}$DefaultUploadHosts${NC}" if [ "${DebugAllEnabled}" == "true" ] || [ "${DebugPluginsEnabled}" == "true" ] ; then bDebugMsgPrintCnt=0 if [ "${DebugAllEnabled}" == "true" ]; then printf "DebugHosts: ${BLUE}${DebugAllEnabled}${NC}" bDebugMsgPrintCnt=$((bDebugMsgPrintCnt + 1)) fi if [ "${DebugPluginsEnabled}" == "true" ]; then if ((bDebugMsgPrintCnt > 0)) ; then printf ", " fi printf "DebugPlugins: ${BLUE}${DebugPluginsEnabled}${NC}" fi printf "\\n" fi echo -e "" HashFilelistTxt=$( sha1sum ${InputFile} | awk '{print $1}' ) if [ "$VerboseLoading" == "true" ]; then printf "SHA1: %s\\n\\n" "${HashFilelistTxt}" fi mainLoopControl=true while ${mainLoopControl} do mainLoopControl="$LoopThroughFileUntilComplete" BeginProcessing "${InputFile}" qChkLineCount=$(grep -Evi '^#|^$|#OK#|#FAIL#|#RETRY#)' "${InputFile}" | wc -l | awk '{ print $1 }') if ((qChkLineCount <= 0)) ; then if DoneProcessingAllUrls "${InputFile}" ; then if [ "${AutoShowMadStatus}" == "true" ] ; then echo -e "${RED}❤${GREEN}Done! ${YELLOW}Me0W!${NC} :D" madStatus "${InputFile}" fi exit 0 else continue fi fi fileCount=1 lockCount=0 sed 's/^[[:space:]]*// ; s/[[:space:]]*$//' "${InputFile}" | while IFS= read -r line do filepath="" filename="" hostcode="" line="${line//[$'\t\r\n']}" if [[ ${line} =~ ^# ]] ; then continue elif grep -Eqvi '\|' <<< "$line" ; then continue fi if [[ $line =~ \| ]] ; then upfile_hostcode="${line##*\|}" upfile_filepath="${line%%\|*}" fi if [ ! -f "$upfile_filepath" ]; then upfile_filepath="${WorkDir}/uploads/$upfile_filepath" fi upfile_filename="${upfile_filepath##*\/}" upfile_fsize=$(GetFileSize "$upfile_filepath" "false") if [ ! -f "$upfile_filepath" ] || ((upfile_fsize <= 0)) ; then uploadBadInputLine "$line" "Bad input line (file: $upfile_filepath, size: $upfile_fsize)" continue fi if [ -z "$upfile_hostcode" ] || [ -z "$upfile_filepath" ]; then uploadBadInputLine "$line" "Bad input line (check filename and hostcode are valid)" continue fi if [ -f "${InputFile}" ] ; then currentHashFilelistTxt=$( sha1sum "${InputFile}" | awk '{print $1}' ) else currentHashFilelistTxt=$( sha1sum ${InputFile} | awk '{print $1}' ) fi UploadTicket="${WorkDir}/.flocks/upload_${upfile_hostcode}_${upfile_filepath//[^a-zA-Z0-9]/}" if [ -f "$UploadTicket" ]; then echo -e "${YELLOW}Lock Exists (Skipping)${NC}: ./.flocks/upload_${upfile_hostcode}_${upfile_filepath//[^a-zA-Z0-9]/}" fileCount=$((fileCount + 1)) continue fi _hostcode="" _hostnick="" _hostfuncprefix="" _hostdomainregex="" readarray -d '@' -t arrUploadHosts <<< "${ListUploadHosts}" for hline in "${arrUploadHosts[@]}"; do if [[ "$hline" == "" ]] || [[ ! "$hline" == *"/"* ]]; then continue fi _hostcode=$(echo $hline|cut -f2 -d '/') _hostnick=$(echo $hline|cut -f3 -d '/') _hostfuncprefix=$(echo $hline|cut -f4 -d '/') if [[ "$upfile_hostcode" == "$_hostcode" ]] ; then printf "\nUploading ${BLUE}%s${NC} to ${YELLOW}%s${NC} ${GREEN}%d${NC} of ${GREEN}%d${NC}\\n" $upfile_filename $_hostcode $fileCount $lineCount printf "|Size: %d\\n" $upfile_fsize printf "|Path: %s\\n" $upfile_filepath mkdir -p "${WorkDir}/.flocks" touch ${UploadTicket} echo -e "-----------------------------------------------------------------------" ${_hostfuncprefix}_UploadFile "${upfile_hostcode}" "${upfile_filepath}" "${fileCount}" "$line" echo -e "" break fi done done #loop through the file line by line qChkLineCount=$(grep -Evi '^#|^$|#OK#|#FAIL#|#RETRY#)' "${InputFile}" | wc -l | awk '{ print $1 }') if ((qChkLineCount > 0)) ; then if [ "$LoopThroughFileUntilComplete" == "false" ]; then echo -e "${NC}" echo -e "${YELLOW}Unprocessed / Skipped File / HostCode(s) Found:${NC}" echo -e "Most likely another terminal is uploading it, or a flock exists...${NC}" if DoneProcessingAllUploads "${InputFile}" ; then exit 0 else continue fi else for ((a=1; a<=4; a++)); do printf "\033[1A\r" printf "\33[2K\r" done echo -e "${NC}" echo -e "${YELLOW}Unprocessed / Skipped File / HostCode(s) Found:${NC}" echo -e "Most likely another terminal is uploading it, or a flock exists...${NC}" echo -e "${YELLOW}Catnapping${NC} for ${CatnapDuration} mins to allow ${YELLOW}$qChkLineCount${NC} upload(s) to finish... ${YELLOW}zZzZzZ${NC} ${BLUE}x$CatnapCount${NC}" sleep ${CatnapDuration}m continue fi else if DoneProcessingAllUploads "${InputFile}" ; then exit 0 else continue fi fi done #loop until done } MadUploadFilesInUploadsFolder() { clear echo -e "${BLD}" echo -e "${PINK}:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:" echo -e ":${NC} ${GREEN}MAD${PINK} Uploader${NC} : Transfer files in ./uploads folder${PINK}${BLD} :" echo -e ":-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:${NC}\\n" OnLoad $0 $@ if ((torPort >= 9050 && torPort <= 9150)); then printf "TorIp: ${GREEN}$TorIp${NC}, Tor listening on port ${GREEN}$torPort${NC}, " else printf "TorIp: ${RED}$TorIp${NC}, ${RED}Tor port not found !!${NC}.\\n" printf "Ensure Tor is setup and listening on a port between 9050 and 9150. Exiting...\\n" exit 1 fi if [ "${UseTorCurlImpersonate}" == "true" ]; then printf "client: ${GREEN}Tor${NC} + ${BLUE}curl_impersonate${NC}\\n" else printf "client: ${GREEN}Tor${NC} + ${GREEN}curl${NC}\\n" fi printf "ConnectTimeout: ${GREEN}$ConnectTimeout${NC}, UploadRetries: ${GREEN}$MaxUploadRetries${NC}, " if [ "${RateMonitorEnabled}" == "true" ]; then printf "RateMonitor: ${GREEN}${RateMonitorEnabled}${NC}" else printf "RateMonitor: ${GREY}${RateMonitorEnabled}${NC}" fi printf "\\n" echo -e "DefaultUploadHosts: ${BLUE}$DefaultUploadHosts${NC}" if [ "${DebugAllEnabled}" == "true" ] || [ "${DebugPluginsEnabled}" == "true" ] ; then bDebugMsgPrintCnt=0 if [ "${DebugAllEnabled}" == "true" ]; then printf "DebugHosts: ${BLUE}${DebugAllEnabled}${NC}" bDebugMsgPrintCnt=$((bDebugMsgPrintCnt + 1)) fi if [ "${DebugPluginsEnabled}" == "true" ]; then if ((bDebugMsgPrintCnt > 0)) ; then printf ", " fi printf "DebugPlugins: ${BLUE}${DebugPluginsEnabled}${NC}" fi printf "\\n" fi echo -e "" echo -e " Filesize Filename (./uploads/ folder)" echo -e "-------------------------------------------------" for fil in "${WorkDir}/uploads/"*.[Rr][aA][rR] \ "${WorkDir}/uploads/"*.[7][zZ] \ "${WorkDir}/uploads/"*.[0-9][0-9][0-9] ; do if [ -d "$fil" ] || [ ! -f "$fil" ]; then continue fi fsize=$(GetFileSize "$fil" "false") if [ -f "$fil" ] && ((fsize > 0)); then tfilename="${fil##*/}" printf "${BLUE}%10s${NC} ${GREEN}%-56s${NC}\\n" $(GetFileSize "$fil" "true") "${tfilename}" fi done printf "\\n" yes_or_no "Are you sure you wish to continue uploading the following files? " && { local lstUploadto="" echo -e "" echo -e "Available ${YELLOW}HostCodes${NC} of hosts to upload to..." _hostcode="" _hostnick="" _hostfuncprefix="" isHostMatchFound=false _helpprochostcodes="" readarray -d '@' -t arrListUploadHosts <<< "${ListUploadHosts}" local itemcount=1 for hline in "${arrListUploadHosts[@]}"; do if [ -z "$hline" ] ; then continue fi _hostcode=$(echo $hline|cut -f2 -d '/') _hostnick=$(echo $hline|cut -f3 -d '/') _hostfuncprefix=$(echo $hline|cut -f4 -d '/') if [ -z "$_hostcode" ] ; then continue fi if [[ ! "$_helpprochostcodes" =~ *"$_hostcode"* ]]; then _helpprochostcodes="${_helpprochostcodes}:${_hostcode}:" local pad12=" " if ((itemcount <= 2)); then printf "${YELLOW}%9s${NC} %-15s${NC}" "${_hostcode}" "${_hostnick}" itemcount=$((itemcount + 1)) else printf "${YELLOW}%9s${NC} %-15s${NC}\\n" "${_hostcode}" "${_hostnick}" itemcount=1 fi fi done if ((itemcount <= 1)); then printf "\\n" else printf "\\n\\n" fi echo -e "Enter hostcodes for the hosts you wish to upload to (comma separated list):" echo -e "ie. ${YELLOW}1f,kraken,oshi${NC}', '${RED}q${NC}' or '${RED}e${NC}' to quit, or '${BLUE}d${NC}' to use the default hosts" echo -e "${BLUE}Default Hosts: ${YELLOW}${DefaultUploadHosts}${NC}" read -p "--> " lstUploadto if [[ "$lstUploadto" =~ '(q|Q|quit|e|E|exit)' ]] ; then exit 0 fi if [[ "$lstUploadto" == 'd' ]] || [[ "$lstUploadto" == 'D' ]]; then lstUploadto="$DefaultUploadHosts" fi echo -e "" for fil in "${WorkDir}/uploads/"*.[Rr][aA][rR] \ "${WorkDir}/uploads/"*.[7][zZ] \ "${WorkDir}/uploads/"*.[0-9][0-9][0-9] ; do if [ -d "$fil" ] || [ ! -f "$fil" ]; then continue fi fsize=$(GetFileSize "$fil" "false") if ((fsize > 0)); then tfilename="${fil##*/}" readarray -d ',' -t arrUploadHostCodes <<< "${lstUploadto}" for hline in "${arrUploadHostCodes[@]}"; do if [[ "$hline" == "" ]] ; then continue fi userentry_hostcode=$(TrimWhitespace "$hline") linematch="" if [ -f "${WorkDir}/uploads/temp_upload_handler.txt" ]; then linematch=$(grep -Eni -m 1 "[OK].*file:.*${tfilename}.*host:.*${userentry_hostcode}.*\$" "${WorkDir}/uploads/temp_upload_handler.txt") if [ ! -z "$linematch" ] ; then echo -e "" echo -e "${GREEN}$tfilename${NC} already uploaded to ${userentry_hostcode} in ${GREEN}temp_upload_handler.txt${NC}" echo -e "${BLUE}line${NC}: ${linematch//, /\\n}" echo -e "" successUploadExists "$tfilename" "$userentry_hostcode" "File/Host (OK) found in ./uploads/temp_upload_handler.txt (line#: ${linematch%%:*})" continue fi linematch=$(grep -Eni -m 1 "[FAIL].*file:.*${tfilename}.*host:.*${userentry_hostcode}.*\$" "${WorkDir}/uploads/temp_upload_handler.txt") if [ ! -z "$linematch" ] ; then echo -e "" echo -e "${GREEN}$tfilename${NC} already failed upload to ${userentry_hostcode} in ${GREEN}temp_upload_handler.txt${NC}" echo -e "${BLUE}line${NC}: ${linematch//, /\\n}" echo -e "" skipfailedUpload "$pline" "$tfilename" "$userentry_hostcode" "File/Host (FAIL) found in ./uploads/temp_upload_handler.txt (line#: ${linematch%%:*})" continue fi linematch=$(grep -Eni -m 1 "[RETRY].*file:.*${tfilename}.*host:.*${userentry_hostcode}.*\$" "${WorkDir}/uploads/temp_upload_handler.txt") if [ ! -z "$linematch" ] ; then echo -e "" echo -e "${GREEN}$tfilename${NC} already failed upload to ${userentry_hostcode} in ${GREEN}temp_upload_handler.txt${NC}" echo -e "${BLUE}line${NC}: ${linematch//, /\\n}" echo -e "" skipfailedUpload "$pline" "$tfilename" "$userentry_hostcode" "File/Host (FAIL or RETRY) found in ./uploads/temp_upload_handler.txt (line#: ${linematch%%:*})" continue fi fi if [[ "$ListUploadHosts" == *"/$userentry_hostcode/"* ]]; then _hostcode="" _hostnick="" _hostfuncprefix="" isHostMatchFound=false _helpprochostcodes="" readarray -d '@' -t arrListUploadHosts <<< "${ListUploadHosts}" local itemcount=1 for hline in "${arrListUploadHosts[@]}"; do if [[ "$hline" == "" ]] ; then continue fi _hostcode=$(echo $hline|cut -f2 -d '/') _hostnick=$(echo $hline|cut -f3 -d '/') _hostfuncprefix=$(echo $hline|cut -f4 -d '/') if [[ "$_hostcode" == "$userentry_hostcode" ]]; then UploadTicket="${WorkDir}/.flocks/upload_${_hostcode}_${fil//[^a-zA-Z0-9]/}" if [ -f "$UploadTicket" ]; then echo -e "${YELLOW}Lock Exists (Skipping)${NC}: ./.flocks/upload_${upfile_hostcode}_${upfile_filepath//[^a-zA-Z0-9]/}" continue fi mkdir -p "${WorkDir}/.flocks" touch ${UploadTicket} echo -e "-----------------------------------------------------------------------" ${_hostfuncprefix}_UploadFile "${userentry_hostcode}" "${fil}" "${fileCount}" echo -e "" break fi done fi done fi done } exit 0 } direct_FetchFileInfo() { echo -e "${GREEN}# Fetching file info…${NC}" finalAttempt=$1 CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} trap "rm -f ${WorkDir}/.flocks/${CLEANSTRING}; echo ""; tput cnorm; exit" 0 1 2 3 6 15 download_url=${file_url} if grep -Eqi '.onion' <<< "$download_url" && grep -Eqi 'https://' <<< "$download_url" ; then echo -e "${PINK}| Reverting .onion address to http...${NC}" download_url="${download_url/https/http}" fi file_id=$(grep -oP '[^\/]*$' <<< "$file_url") maxretries=2 for ((j=1; j<=$maxretries; j++)); do mkdir -p "${WorkDir}/.temp" if ((j > 1)); then tput rc; tput el; fi tput sc filename="" file_size_bytes="" tor_identity="${RANDOM}" if ((j % 2 == 0)); then printf "| Retrieving Get/Head: attempt #$j" file_header=$(tor_curl_request --insecure -m 18 -s -D - -o /dev/null \ -H "Connection: keep-alive" \ -w 'EffectiveUrl=%{url_effective}' \ "$download_url") else printf "| Retrieving Head: attempt #$j" rm -f "${WorkDir}/.temp/directhead" file_header=$(tor_curl_request --insecure --head -H "Connection: keep-alive" -L -s -i "$download_url" | tee "${WorkDir}/.temp/directhead" & sleep 6 [ -s "${WorkDir}/.temp/directhead" ] kill $! 2>/dev/null ) if [ ! -f "${WorkDir}/.flocks/${remote_url//[^a-zA-Z0-9]/}" ]; then touch "${WorkDir}/.flocks/${remote_url//[^a-zA-Z0-9]/}" fi rm -f "${WorkDir}/.temp/directhead" fi if [ "${DebugAllEnabled}" == "true" ] ; then urllen=${#remote_url} if ((urllen > 64)); then debugHtml "${remote_url##*/}" "direct_${RANDOM}_head" "download_url: ${download_url}"$'\n'"${file_header}" else debugHtml "${remote_url##*/}" "direct_${remote_url//[^a-zA-Z0-9]/}_head" "download_url: ${download_url}"$'\n'"${file_header}" fi fi if [ ! -z "$file_header" ] ; then if grep -Eqi 'HTTP/.*404' <<< "${file_header}" ; then printf "\\n" echo -e "${RED}| 404. The file was not found or has been removed.${NC}" removedDownload "${remote_url}" exitDownloadNotAvailable=true return 1 fi if ! grep -Eqi 'HTTP/.*200|HTTP/.*302' <<< "${file_header}" ; then hResponse=$(grep -oPi 'HTTP/.* \K.*$' <<< "${file_header}") if ((j>=$maxretries)); then printf "\\n" echo -e "${RED}| Unexpected header response ($hResponse).${NC}" return 1 else printf "\\n" echo -e "${YELLOW}| Unexpected header response ($hResponse). Retrying...${NC}" continue fi fi if [ "$filename_override" == "" ] && [ "$filename" == "" ] ; then if grep -Eqi 'filename=' <<< "${file_header}" ; then filename=$(grep -oPi -m 1 'filename=\K.*?$' <<< "${file_header}") filename="${filename%%;*}" filename=${filename//filename*=UTF-8\'\'/} filename=${filename##filename} filename=${filename//\"/} filename=${filename//[$'\t\r\n']} elif [ ! -z $file_id ]; then if grep -Eqi '(.rar|.7z|.zip|.[0-9][0-9][0-9])$' <<< ${file_id} ; then printf "\\n" echo -e "${YELLOW}| Failed to extract file name, using url name.${NC}" filename="$file_id" else printf "\\n" echo -e "${YELLOW}| Failed to extract file name. Using url fileid.${NC}" filename=${file_id}".renameMe" fi else printf "\\n" echo -e "${YELLOW}| Failed to extract file name. Using url.${NC}" filename="${remote_url//[^a-zA-Z0-9]/}.renameMe" filename="${filename/https/}" fi fi if grep -Eqi 'content-length:' <<< "${file_header}" ; then file_size_bytes=$(grep -oPi '(?<=content-length: ).*?(?=$)' <<< "$file_header") file_size_bytes=${file_size_bytes//[$'\t\r\n']} else printf "\\n" echo -e "${YELLOW}| Failed to retreive filesize${NC}" if ((j>=$maxretries)); then file_size_bytes="" else continue fi fi printf "\\n" break else if ((j>=$maxretries)); then printf "\\n" echo -e "${YELLOW}| No response. Attempting direct download without head${NC}" return 1 else continue fi fi done touch ${WorkDir}/.flocks/${remote_url//[^a-zA-Z0-9]/} if [ ! "$filename_override" == "" ] ; then filename="$filename_override" fi filename=$(sanitize_file_or_folder_name "${filename}") if [ -z "$filename" ]; then printf "\\n" echo -e "${RED}| Unexpected or no header response.${NC}" return 1 fi if [ -z $file_size_bytes ] ; then file_size_readable="${RED}Unknown filesize…${NC}" else file_size_readable="$(numfmt --to=iec --from=auto --format "%.2f" <<< "$file_size_bytes")" fi echo -e "${YELLOW}| File size:${NC}\t${file_size_readable}" file_path="${download_inflight_path}${filename}" echo -e "${YELLOW}| File name:${NC}\t\"${filename}\"" flockDownload="${WorkDir}/.flocks/${filename//[^a-zA-Z0-9\.\_\-]/}.flock" if CheckDownloadExists "$remote_url" "$MoveToFolder" "$filecnt" "$filename" "$file_path" "$completed_location" ; then return 1 fi echo "${remote_url//[^a-zA-Z0-9]/}" > $flockDownload } direct_GetFile() { echo -e "${GREEN}# Downloading…" echo -e "${YELLOW}| File path:${NC}\t./.inflight/${filename}\n" fileCnt=$1 retryCnt=$2 finalAttempt=$3 flockDownload="${WorkDir}/.flocks/${filename//[^a-zA-Z0-9\.\_\-]/}.flock" for ((j=1; j<=$MaxDownloadRetries; j++)); do pd_presize=0 if [ -f "$file_path" ] ; then pd_presize=$(stat --format="%s" "$file_path" | tr -d '[:space:]') fi if [ -z $file_size_bytes ] ; then echo -e "${BLUE}| No Resume Fetch${NC} (unknown filesize)" CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} trap "rm -f ${WorkDir}/.flocks/${CLEANSTRING}; rm -f $flockDownload; echo ""; tput cnorm; exit" 0 1 2 3 6 15 tor_curl_request --insecure -L --referer "$file_url" "$download_url" --output "$file_path" rc=$? if [ $rc -ne 0 ] ; then printf "${RED}Download Failed (bad exit status).${NC}" if [ -f ${file_path} ]; then printf "${YELLOW} Partial removed...${NC}" printf "\n\n" rm -f "${file_path}" else printf "\n\n" fi if ((j >= $MaxDownloadRetries)) ; then rm -f "$flockDownload"; if [ "${finalAttempt}" == "true" ] ; then droppedSizeBadDownload "${remote_url}" "${filename}" "${received_file_size}" fi return 1 else continue fi fi if CheckNoHtml "$remote_url" "$filename" "$file_path" ; then containsHtml=false else containsHtml=true fi if [ "$containsHtml" == "true" ]; then echo -e "${YELLOW}Download Failed (contains html)${NC} partial removed..." rm -f "${file_path}" if ((j >= $MaxDownloadRetries)) ; then rm -f "$flockDownload"; if [ "${finalAttempt}" == "true" ] ; then droppedSizeBadDownload "${remote_url}" "${filename}" "${received_file_size}" fi return 1 else continue fi fi break else CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} trap "rm -f ${WorkDir}/.flocks/${CLEANSTRING}; rm -f $flockDownload; echo ""; tput cnorm; exit" 0 1 2 3 6 15 if [ "${RateMonitorEnabled}" == "true" ]; then tor_curl_request --insecure -L --speed-limit $DownloadSpeedMin --speed-time $DownloadTimeoutInterval "$download_url" --continue-at - --output "$file_path" else tor_curl_request --insecure -L --referer "$file_url" "$download_url" --continue-at - --output "$file_path" fi received_file_size=0 if [ -f "$file_path" ] ; then received_file_size=$(stat --format="%s" "$file_path" | tr -d '[:space:]') fi if CheckNoHtml "$remote_url" "$filename" "$file_path" ; then containsHtml=false else containsHtml=true fi downDelta=$(( received_file_size - pd_presize )) if [[ "${received_file_size}" -ne "${file_size_bytes}" ]] || [ "$containsHtml" == "true" ]; then if [ "${AutoRepairBadPartials}" == "true" ] && (( downDelta > 0 && downDelta < 1024 )) ; then if [ -f "${file_path}" ] ; then if ((pd_presize > 0)); then echo -e "${YELLOW}Bad node / HTML found:${NC} reverting to previous file..." truncateDownload "$remote_url" "$filename" "$pd_presize" "$received_file_size" truncate -s $pd_presize "${file_path}" else echo -e "${YELLOW}Bad node / HTML found:${NC} tainted partial removed..." rm -f "${file_path}" fi fi if ((j >= $MaxDownloadRetries)) ; then rm -f "$flockDownload"; if [ "${finalAttempt}" == "true" ] ; then droppedSizeBadDownload "${remote_url}" "${filename}" "${received_file_size}" fi return 1 else continue fi elif [ "${AutoRepairBadPartials}" == "true" ] && [ "$containsHtml" == "true" ] ; then if [ -f "${file_path}" ] ; then if ((pd_presize > 0)); then echo -e "${YELLOW}Bad node / HTML found:${NC} reverting to previous file..." truncateDownload "$remote_url" "$filename" "$pd_presize" "$received_file_size" truncate -s $pd_presize "${file_path}" else echo -e "${YELLOW}Bad node / HTML found:${NC} tainted partial removed..." rm -f "${file_path}" fi fi if ((j >= $MaxDownloadRetries)) ; then rm -f "$flockDownload"; if [ "${finalAttempt}" == "true" ] ; then droppedSizeBadDownload "${remote_url}" "${filename}" "${received_file_size}" fi return 1 else continue fi elif (( downDelta > 0 && downDelta < 1024 )) || [ "$containsHtml" == "true" ] ; then if [ -f "$file_path" ] ; then rm -rf "$file_path" fi echo -e "\n${YELLOW}Bad node / HTML found:${NC} tainted partial removed..." if ((j >= $MaxDownloadRetries)) ; then rm -f "$flockDownload"; if [ "${finalAttempt}" == "true" ] ; then droppedSizeBadDownload "${remote_url}" "${filename}" "${received_file_size}" fi return 1 else continue fi fi if [[ "${received_file_size}" -ne "${file_size_bytes}" ]]; then echo -e "\n${RED}Download failed, file is incomplete.${NC}" if ((j >= $MaxDownloadRetries)) ; then rm -f "$flockDownload"; if [ "${finalAttempt}" == "true" ] ; then droppedSizeBadDownload "${remote_url}" "${filename}" "${received_file_size}" fi return 1 else continue fi fi else break fi fi done rm -f "$flockDownload"; ProcessCompletedDownload "$remote_url" "$MoveToFolder" "$filecnt" "$filename" "$file_size_bytes" "$completed_location" "$file_path" return 0 } direct_DownloadFile() { local remote_url="${1}" # url from urls.txt local filecnt=${2} local file_url="${3}" # Override url ie. lainsafe.onion --> lainsafe if [ -z "$file_url" ]; then file_url="$remote_url" fi warnAndRetryUnknownError=false exitDownloadError=false exitDownloadNotAvailable=false fileAlreadyDone=false download_inflight_path="${WorkDir}/.inflight/" mkdir -p "$download_inflight_path" completed_location="${WorkDir}/downloads/" tor_identity="${RANDOM}" finalAttempt="false" for ((z=0; z<=$MaxUrlRetries; z++)); do if [ $z -eq $MaxUrlRetries ] ; then finalAttempt="true" fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} trap "rm -f ${WorkDir}/.flocks/${CLEANSTRING}; echo ""; tput cnorm; exit" 0 1 2 3 6 15 if direct_FetchFileInfo $finalAttempt && direct_GetFile "${filecnt}" $((z+1)) $finalAttempt ; then return 0 elif [ $z -lt $MaxUrlRetries ]; then if [ "${fileAlreadyDone}" == "true" ] ; then break fi if [[ "${warnAndRetryUnknownError}" == "true" ]] ; then if [ "${DebugAllEnabled}" == "true" ] ; then debugHtml "${remote_url##*/}" "error" "Retry due to an unknown issue: attempt #$((z+1)) of ${MaxUrlRetries}" fi fi if [[ "${exitDownloadError}" == "true" || "${exitDownloadNotAvailable}" == "true" ]] ; then if [ "${DebugAllEnabled}" == "true" ] ; then debugHtml "${remote_url##*/}" "error" "Exit due to unrecoverable issue" fi rm -f "${WorkDir}/.flocks/${remote_url//[^a-zA-Z0-9]/}" break fi echo -e "\n${YELLOW}A recoverable error occurred, retry attempt $((z+1))/${MaxUrlRetries}${NC}" sleep 3 fi done rm -f "${WorkDir}/.flocks/${remote_url//[^a-zA-Z0-9]/}" } backupIFS=$IFS IFS=$(echo -en "\n\b") RED=$(tput setaf 1) PINK=$(tput setaf 5) NC=$(tput sgr0) # No color BLD=$(tput bold) # Bold text WHITE="\033[1;37m" CYAN="\033[1;36m" BLUE="\033[1;34m" YELLOW="\033[1;33m" GREEN="\033[1;32m" RED="\033[1;31m" GREY="\033[1;30m" NC="\033[0m" # No color tput civis # Hide the cursor trap "tput el; tput cnorm; exit" 0 1 2 3 6 15 CatnapCount=0 UrlOnly=false didSourceMadConfig=false if [ -f "${ScriptDir}/mad.config" ] ; then source ${ScriptDir}/mad.config didSourceMadConfig=true fi if [ ! -z ${WorkDirOverride} ]; then WorkDir="${WorkDirOverride}" if [ ! -d "${WorkDir}" ]; then mkdir -p "${WorkDir}" fi fi if [[ "$1" == "audit" ]]; then madAudit exit 0 fi torPort=$(checkTor) if [ "$torPort" == "" ] ; then printf "%s\\n" "Tor is not running!" exit 1 fi if [ "${UseTorCurlImpersonate}" == "true" ]; then curl_impersonate=() readarray -d $'' arrFiles < <(find "$ScriptDir" -maxdepth 1 -name "curl_*" -printf '%p\n' | sort -Vk1) bFoundCurlHeader=false for fil in "${arrFiles[@]}"; do if which $fil >/dev/null; then curl_impersonate=($fil) bFoundCurlHeader=true break fi done if [ "$bFoundCurlHeader" == "false" ]; then echo -e "${RED}[ERROR] Missing dependency \"curl-impersonate\"!${NC}" echo -e "Some hosts use CloudFlare to detect and block scripts (such as hexload)." echo -e "To get around it, this script needs to impersonate a browser." echo -e "You'll need to download ${GREEN}\"curl-impersonate\"${NC}." echo -e "" echo -e "[Manual Installation]" echo -e "The latest binary can be obtained on GitHub, search for \"curl-impersonate\"" echo -e "To access the releases on GitHub without javascript, do the following:" echo -e " 1. Visit the page of curl-impersonate and add \"/releases/latest/\" at end of URL." echo -e " 2. You'll be redirected to the latest version, e.g: \"/releases/tag/v0.5.4\"" echo -e " 3. In the URL replace \"tag\" with \"expanded_assets\"" echo -e " e.g. \"/releases/expanded_assets/v0.6.1\"" echo -e " 4. Download archive ${GREEN}\"curl-impersonate-vX.Y.Z.x86_64-linux-gnu.tar.gz\"${YELLOW}." echo -e " 5. Extract files ${GREEN}\"curl-impersonate-ff\"${NC} and ${GREEN}\"curl_ff109\"${NC} next to this script." echo -e "" echo -e "[Auto Install]" echo -e "run $0 install_curl_impersonate\\n" echo -e "" yes_or_no "Do you wish to download and extract latest curl_impersonate (using tor+curl)?" && { UseTorCurlImpersonate=false install_curl_impersonate } exit 0 fi fi arg1="$1" # filelist, multi, hex, 1f, pd, reset, status arg2="$2" # auto, filelist, arg3="$3" # filelist arg4="$4" # moveToFolder arg5="$5" # fileCount arg6="$6" # lineCount if [ "$#" -ne 1 ] && [ "$#" -ne 2 ] && [ "$#" -ne 3 ] && [ "$#" -ne 4 ] && [ "$#" -ne 5 ] && [ "$#" -ne 6 ] || \ [ "$1" == "help" ] || [ "$1" == "?" ] ; then LoadMadDownloadHosts echo -e "${BLUE}MAD Help -------------------------${NC}" echo -e "Supported Hosts + Keyword:" _hostcode="" _hostnick="" _hostfuncprefix="" _hosturls="" _hostdomainregex="" _helpprochostcodes="" readarray -d '@' -t arrHostAndDomainRegexes <<< "${ListHostAndDomainRegexes}" for hline in "${arrHostAndDomainRegexes[@]}"; do if [[ "$hline" == "" ]] || [[ ! "$hline" == *":"* ]]; then continue fi chnk1="${hline%%:*}" _hostcode=$(echo $chnk1|cut -f2 -d '/') _hostnick=$(echo $chnk1|cut -f3 -d '/') _hostfuncprefix=$(echo $chnk1|cut -f4 -d '/') _hosturls=$(echo $chnk1|cut -f5 -d '/') _hostdomainregex="${hline#*\:}" if [[ ! "$_helpprochostcodes" =~ *"$_hostcode"* ]]; then echo -e " [${YELLOW}$_hostcode${NC}] $_hosturls" _helpprochostcodes="${_helpprochostcodes}:${_hostcode}:" fi done echo -e " [${YELLOW}direct=${NC}] any direct url or cdn url${NC}" echo -e " * Script supports multiple terminals/instances running." echo -e "" echo -e "${BLUE}Download Usage ----------------------------${NC}" echo -e "[${YELLOW}Process Downloads${NC}]: 1 terminal, process list" printf " %s urls.txt\\n" "$0" echo -e "" echo -e "[${YELLOW}Specific Host${NC}]: (1f, hex, kraken, pd, oshi, fh, dosya, upee, uhive, uflix)" printf " %s 1f urls.txt\\n" "$0" printf " %s oshi urls.txt\\n" "$0" echo -e "" echo -e "[${YELLOW}Multi Spawn${NC}]: Launch X terminals to process downloads (OS dependent)" printf " %s multi [2-8] urls.txt\\n" "$0" echo -e "" echo -e "[${YELLOW}Multi Spawn, Specific Host${NC}]: Launch X terminals to process specified host (OS dependent)" printf " %s multi hex [2-8] urls.txt\\n" "$0" printf " %s multi pd [2-8] urls.txt\\n" "$0" echo -e "" echo -e "[${YELLOW}Multi Auto Spawn${NC}]: Launch 1 terminal per host with downloads (OS dependent)" printf " %s multi auto urls.txt\\n" "$0" echo -e "" echo -e "[${YELLOW}Multi Auto Spawn [# terms]${NC}]: Launch 1 terminal per host with downloads [x total] (OS dependent)" printf " %s multi auto [2-8] urls.txt\\n" "$0" echo -e "" echo -e "${BLUE}Other Download Commands --------------------${NC}" echo -e "[${YELLOW}Status${NC}]: Displays the status of urls in the download urls.txt" printf " %s status urls.txt\\n" "$0" echo -e "[${YELLOW}Reset Urls${NC}]: Resets lines with #RETRY# comment in the download urls.txt" echo -e " - Only works when AutoCommentOnCompletion=true" printf " %s reset urls.txt\\n" "$0" echo -e "" echo -e "${BLUE}Upload Usage ----------------------------${NC}" echo -e "[${YELLOW}Process Uploads${NC}]: Uploads each file & HostCode line in uploads.txt" printf " %s upload uploads.txt\\n" "$0" echo -e "[${YELLOW}MAD Uploader UI${NC}]: Uploads files in the ./uploads folder via user prompts" printf " %s upload\\n" "$0" echo -e "" echo -e "${BLUE}Other Upload Commands --------------------${NC}" echo -e "[${YELLOW}Upload Status${NC}]: Displays the status of file uploads in the uploads.txt" printf " %s upload status uploads.txt\\n" "$0" echo -e "[${YELLOW}Reset Urls${NC}]: Resets lines with #RETRY# comment in the uploads.txt" echo -e " - Only works when AutoCommentOnCompletion=true" printf " %s upload reset uploads.txt\\n" "$0" echo -e "" echo -e "${BLUE}Other MAD Commands --------------------${NC}" echo -e "[${YELLOW}Install curl_impersonate${NC}]: Downloads the latest binary for curl_impersonate from github repo (3 choices)" printf " %s install_curl_impersonate\\n" "$0" echo -e "[${YELLOW}ClipboardMonitor${NC}]: Monitors clipboard for urls of supported download hosts and adds to urls.txt" printf " %s clipmon urls.txt\\n" "$0" echo -e "" echo -e "${BLUE}Information -----------------------${NC}" echo -e "[${YELLOW}Configurables${NC}]: Set optional params in the global section (top of script)" echo -e " - AutoReloadOnFilelistTxtChanges: Reloads urls.txt if modified to get additions" echo -e " - AutoCommentOnCompletion: Comments completed lines and adds status at end" echo -e " #OK#, #RETRY#, #REMOVED#, #FAILED#, etc -- Filename / Reason" echo -e " - ClearScreenOnAutoReload: Clears terminal session whenever the urls.txt is reloaded" echo -e " - AutoRepairBadPartials: Automatically repairs partial downloads with bad data to allow resuming" echo -e " through bad nodes" echo -e "" echo -e "${BLUE}Special Commands allowed in urls.txt ---${NC}" echo -e "[${YELLOW}MoveToFolder${NC}]: Sets the folder to move downloads to (blank resets to leave in downloads)" echo -e " - folder=My folder name (2020)" echo -e "[${YELLOW}FilenameOverride${NC}]: Sets the specified filename to save the download as per url" echo -e " (Add \"|filename.ext\" to end of url, no double quotes)" echo -e " - http://oshi.at/abcd/origAABB.rar|My specified file.part1.rar" echo -e "[${YELLOW}DirectUrl${NC}]: Will attempt to download directly from the url or cdn link" echo -e " - direct=http://pomf2.lain.la/f/abcd00zz.7z" echo -e " - If the direct url doesn't end in the filename, it is highly recommended to override it with |filename.ext" echo -e ' - ie. direct=http://somehost.onion/abcD|filename.part1.rar' echo -e "" exit 0 fi if [[ "$arg1" == "upload" ]] || [[ "$arg1" == "uploads" ]]; then if [ "$arg2" == "status" ] && [ -f "$arg3" ]; then madStatusUploads "$arg3" exit 0 fi if [ "$arg2" == "reset" ] && [ -f "$arg3" ]; then madResetUploads "$arg3" exit 0 fi if [ ! -z "$arg2" ]; then InputFile="$arg2" if [ ! -f "${InputFile}" ] && [ -f "${WorkDir}/${InputFile}" ]; then InputFile="${WorkDir}/${InputFile}" fi if [ ! -f "${InputFile}" ]; then echo -e "Unable to read file $InputFile [1]" exit 1 fi CleanInputFile $InputFile qChkLineCount=0 if [ -f "${InputFile}" ] ; then qChkLineCount=$(grep -Evi '^#|^$|#OK#|#FAIL#|#RETRY#)' "${InputFile}" | wc -l | awk '{ print $1 }') if ((qChkLineCount <= 0)); then if DoneProcessingAllUploads "$InputFile" ; then if [ "${AutoShowMadStatus}" == "true" ] ; then echo -e "${RED}❤${GREEN}Done! ${YELLOW}Me0W!${NC} :D" madStatusUploads "$InputFile" fi exit 0 fi exit 0 fi LoadMadUploadHosts MadUploadFromFileTxt "$InputFile" exit 0 else echo -e "Unable to read file $InputFile [2]" exit 1 fi else LoadMadUploadHosts MadUploadFilesInUploadsFolder exit 0 fi echo -e "Nothing to process.." exit 0 elif [[ "$arg1" == "reset" ]] && [[ ! -z "$arg2" ]]; then madReset "$arg2" elif [[ "$arg1" == "status" ]] && [[ ! -z "$arg2" ]]; then madStatus "$arg2" elif [[ "$arg1" == "clipmon" ]] && [[ ! -z "$arg2" ]]; then clipboard_monitor "$arg2" elif [[ "$arg1" == "install_curl_impersonate" ]]; then install_curl_impersonate exit 0 elif [[ "$arg1" == "hosts" ]]; then madHostDetails exit 0 elif [[ "$arg1" == "plugins" ]]; then madPluginDetails exit 0 fi if grep -Eqi "^(http|direct=http)" <<< "$1" ; then UrlOnly=true fi echo -e "${BLD}" echo -e "${PINK}:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:" echo -e ":${NC} ${GREEN}M${NC}ulti-host ${GREEN}A${NC}uto ${GREEN}D${NC}ownloader - ${YELLOW}v${ScriptVersion} ${BLUE}(by kittykat) ${PINK}:" echo -e ":-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:${NC}\\n" if [ "$didSourceMadConfig" == "true" ]; then echo -e "[${GREEN}LOAD${NC}] ${BLUE}mad.config${NC}" echo -e "" fi LoadMadDownloadHosts LoadMadUploadHosts LoadMadPlugins OnLoad $0 $@ if ((torPort >= 9050 && torPort <= 9150)); then printf "TorIp: ${GREEN}$TorIp${NC}, Tor listening on port ${GREEN}$torPort${NC}, " else printf "TorIp: ${RED}$TorIp${NC}, ${RED}Tor port not found !!${NC}.\\n" printf "Ensure Tor is setup and listening on a port between 9050 and 9150. Exiting...\\n" exit 1 fi if [ "${UseTorCurlImpersonate}" == "true" ]; then printf "client: ${GREEN}Tor${NC} + ${BLUE}curl_impersonate${NC}\\n" else printf "client: ${GREEN}Tor${NC} + ${GREEN}curl${NC}\\n" fi echo -e "ConnectTimeout: ${GREEN}$ConnectTimeout${NC}, CircuitRetries: ${GREEN}$CircuitRetries${NC}, UrlRetries: ${GREEN}$MaxUrlRetries${NC}, DownloadRetries: ${GREEN}$MaxDownloadRetries${NC}" if [ "${LoopThroughFileUntilComplete}" == "true" ]; then printf "Loop: ${GREEN}${LoopThroughFileUntilComplete}${NC}, " else printf "Loop: ${GREY}${LoopThroughFileUntilComplete}${NC}, " fi if [ "${AutoReloadOnFilelistTxtChanges}" == "true" ]; then printf "AutoReload: ${GREEN}${AutoReloadOnFilelistTxtChanges}${NC}, " else printf "AutoReload: ${GREY}${AutoReloadOnFilelistTxtChanges}${NC}, " fi if [ "${AutoCommentOnCompletion}" == "true" ]; then printf "AutoComment: ${GREEN}${AutoCommentOnCompletion}${NC}, " else printf "AutoComment: ${GREY}${AutoCommentOnCompletion}${NC}, " fi if [ "${AutoRepairBadPartials}" == "true" ]; then printf "AutoRepairPartials: ${GREEN}${AutoRepairBadPartials}${NC}" else printf "AutoRepairPartials: ${GREY}${AutoRepairBadPartials}${NC}" fi printf "\\n" if [ "${RateMonitorEnabled}" == "true" ]; then printf "RateMonitor: ${GREEN}${RateMonitorEnabled}${NC}, " else printf "RateMonitor: ${GREY}${RateMonitorEnabled}${NC}, " fi if grep -Eq "pjscloud.sh" <<< "$LoadPlugins" && [ "$PJSCloud_pixeldrain" == "true" ]; then if [ "${ar_pgsKey[0]}" == 'aa-bbbbb-ccccc-ddddd-eeeee-fffff' ] || [ "${ar_pgsKey[0]}" == "" ] ; then printf " ${RED}[==>${NC} Setup ${BLUE}PJS apikey${NC} in pjscloud.sh ${RED}<==]${NC}" PJSCloud_pixeldrain=false printf "PDPump: ${GREY}${PJSCloud_pixeldrain}${NC}, " else printf "PDPump: ${GREEN}${PJSCloud_pixeldrain}${NC}, " fi else printf "PDPump: ${GREY}${PJSCloud_pixeldrain}${NC}, " fi if [ "${UsePixeldrainBypass}" == "true" ]; then printf "PDBypass: ${GREEN}${UsePixeldrainBypass}${NC}, " else printf "PDBypass: ${GREY}${UsePixeldrainBypass}${NC}, " fi if [ "${EnableFiledotProcessing}" == "true" ]; then GetRandomFiledotUser if [ "${ar_fdUP[0]}" == 'user1|pass1' ] || [ "${ar_fdUP[0]}" == "" ]; then printf " ${RED}[==>${NC} Setup ${BLUE}user${NC}/${BLUE}pass${NC} in script ${RED}<==]${NC}" EnableFiledotProcessing=false printf "Filedot: ${GREY}$EnableFiledotProcessing${NC}" else printf "Filedot: ${GREEN}$EnableFiledotProcessing${NC}" fi else printf "Filedot: ${GREY}$EnableFiledotProcessing${NC}" fi printf "\\n" if [ "${DebugAllEnabled}" == "true" ] || [ "${DebugPluginsEnabled}" == "true" ] ; then bDebugMsgPrintCnt=0 if [ "${DebugAllEnabled}" == "true" ]; then printf "DebugHosts: ${BLUE}${DebugAllEnabled}${NC}" bDebugMsgPrintCnt=$((bDebugMsgPrintCnt + 1)) fi if [ "${DebugPluginsEnabled}" == "true" ]; then if ((bDebugMsgPrintCnt > 0)) ; then printf ", " fi printf "DebugPlugins: ${BLUE}${DebugPluginsEnabled}${NC}" fi printf "\\n" fi if [ ! "$UrlOnly" == "true" ]; then qChkLineCount=0 if [ -f "${WorkDir}/$1" ] ; then qChkLineCount=$(grep -Ei '^(http|direct=http)' "${WorkDir}/$1" | wc -l | awk '{ print $1 }') if ((qChkLineCount <= 0)); then if DoneProcessingAllUrls "$1" ; then if [ "${AutoShowMadStatus}" == "true" ] ; then echo -e "${RED}❤${GREEN}Done! ${YELLOW}Me0W!${NC} :D" madStatus "$1" fi exit 0 fi fi elif [ -f "${WorkDir}/$2" ] ; then qChkLineCount=$(grep -Ei '^(http|direct=http)' "${WorkDir}/$2" | wc -l | awk '{ print $1 }') if ((qChkLineCount <= 0)); then if DoneProcessingAllUrls "$2" ; then if [ "${AutoShowMadStatus}" == "true" ] ; then echo -e "${RED}❤${GREEN}Done! ${YELLOW}Me0W!${NC} :D" madStatus "$2" fi exit 0 fi fi elif [ -f "${WorkDir}/$3" ] ; then qChkLineCount=$(grep -Ei '^(http|direct=http)' "${WorkDir}/$3" | wc -l | awk '{ print $1 }') if ((qChkLineCount <= 0)); then if DoneProcessingAllUrls "$3" ; then if [ "${AutoShowMadStatus}" == "true" ] ; then echo -e "${RED}❤${GREEN}Done! ${YELLOW}Me0W!${NC} :D" madStatus "$3" fi exit 0 fi fi fi fi hostOnlyOrUrl="" multiCount=0 if [ "$UrlOnly" == "true" ]; then echo -e "" echo -e "${BLUE}:-:-:-: URL Only Mode :-:-:-:${NC}" line="$arg1" if [ ! -z "$arg2" ]; then filename_override="$arg2" fi if [[ $line =~ | ]] ; then line="${line%%\|*}" fi if [[ ${line} =~ ^# ]] ; then exit 0 elif [[ ${line} =~ ^direct=http ]]; then if ! PreProcessUrl "$line" ; then continue fi remote_url=${line/direct=/} if [[ ${remote_url} =~ ^http: ]] ; then remote_url=${remote_url/http:/https:} fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ] ; then echo -e "${YELLOW}Lock Exists (Skipping)${NC}: ./.flocks/${CLEANSTRING}" exit 0 fi printf "\nGetting ${YELLOW}direct${NC} file ${GREEN}1${NC}\\n" if [ ! "${filename_override}" == "" ] ; then printf "[${BLUE}FilenameOverride${NC}]: %s\\n" $filename_override fi printf "[${BLUE}DirectUrl${NC}]: %s\\n" "${remote_url}" mkdir -p "${WorkDir}/.flocks" touch "${WorkDir}/.flocks/${CLEANSTRING}" mkdir -p "${WorkDir}/.inflight" direct_DownloadFile "${remote_url}" "${fileCount}" elif [[ ${line} =~ ^http ]] ; then if ! PreProcessUrl "$line" ; then continue fi remote_url=${line} if [[ ${remote_url} =~ ^http: ]] ; then remote_url=${remote_url/http:/https:} fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ] ; then echo -e "${YELLOW}Lock Exists (Skipping)${NC}: ./.flocks/${CLEANSTRING}" exit 0 fi isHostMatchFound=false _hostcode="" _hostnick="" _hostfuncprefix="" _hosturls="" _hostdomainregex="" readarray -d '@' -t arrHostAndDomainRegexes <<< "${ListHostAndDomainRegexes}" for hline in "${arrHostAndDomainRegexes[@]}"; do if [[ "$hline" == "" ]] || [[ ! "$hline" == *":"* ]]; then continue fi chnk1="${hline%%:*}" _hostcode=$(echo $chnk1|cut -f2 -d '/') _hostnick=$(echo $chnk1|cut -f3 -d '/') _hostfuncprefix=$(echo $chnk1|cut -f4 -d '/') _hosturls=$(echo $chnk1|cut -f5 -d '/') _hostdomainregex="${hline#*\:}" if [[ $remote_url =~ $_hostdomainregex ]]; then isHostMatchFound=true printf "\nGetting ${YELLOW}$_hostnick${NC} file ${GREEN}1${NC}\\n" if [ ! "${filename_override}" == "" ] ; then printf "[${BLUE}FilenameOverride${NC}]: %s\\n" $filename_override fi printf "[DownloadUrl]: %s\\n" "${remote_url}" mkdir -p "${WorkDir}/.flocks" touch "${WorkDir}/.flocks/${CLEANSTRING}" mkdir -p "${WorkDir}/.inflight" ${_hostfuncprefix}_DownloadFile "${remote_url}" "${fileCount}" break fi done if [ "$isHostMatchFound" == "false" ]; then printf "${RED}Invalid url (bad format or unsupported host [UO]):${NC} \\n%s\\n" $remote_url badUrlDownload "${remote_url}" exit 0 fi else printf "Ignore garbage line.\\n" > /dev/null exit 0 fi exit 0 fi if [[ "$arg1" == "multi" ]] && [[ "$arg2" == "auto" ]] ; then useMultiCount=false multiCount=2 # Default ./mad.sh multi # urls.txt if [[ "$arg3" == "2" || "$arg3" == "3" || "$arg3" == "4" || \ "$arg3" == "5" || "$arg3" == "6" || "$arg3" == "7" || "$arg3" == "8" ]] ; then useMultiCount=true multiCount=$((arg3)) InputFile="$4" else InputFile="$3" fi if [ ! -f "${InputFile}" ] && [ -f "${WorkDir}/${InputFile}" ]; then InputFile="${WorkDir}/${InputFile}" fi if [ ! -f "${InputFile}" ]; then printf "Unable to read file %s! [1]\\n" "$1" exit 1 else isHostMatchFound=false _hostcode="" _hostnick="" _hostfuncprefix="" _hosturls="" _hostdomainregex="" readarray -d '@' -t arrHostAndDomainRegexes <<< "${ListHostAndDomainRegexes}" for hline in "${arrHostAndDomainRegexes[@]}"; do if [[ "$hline" == "" ]] || [[ ! "$hline" == *":"* ]]; then continue fi chnk1="${hline%%:*}" _hostcode=$(echo $chnk1|cut -f2 -d '/') _hostnick=$(echo $chnk1|cut -f3 -d '/') _hostfuncprefix=$(echo $chnk1|cut -f4 -d '/') _hosturls=$(echo $chnk1|cut -f5 -d '/') _hostdomainregex="${hline#*\:}" lineCount=0 lineCount=$(grep -Ei $_hostdomainregex "${InputFile}" | wc -l | awk '{ print $1 }') if ((lineCount > 0)) ; then if [ "$useMultiCount" == "false" ] || ( [ $useMultiCount == "true" ] && ((multiCount > 0)) ) ; then printf "%s has ${GREEN}%d ${YELLOW}$_hostnick${NC} files to download.\\n" "${InputFile}" $lineCount LaunchTerminal "$_hostcode" ${InputFile} multiCount=$((multiCount - 1)) fi fi done if [ $useMultiCount == "true" ] && ((multiCount > 0)) ; then printf "Spawning ${GREEN}%d ${YELLOW}allhosts${NC} terminals.\\n" $multiCount for ((k=1; k<=$multiCount; k++)); do LaunchTerminal ${InputFile} done fi fi exit 0 elif [[ "$arg1" == "multi" ]] && \ [[ "$arg3" == "2" || "$arg3" == "3" || "$arg3" == "4" || \ "$arg3" == "5" || "$arg3" == "6" || "$arg3" == "7" || "$arg3" == "8" ]] ; then multiCount=2 # Default ./mad.sh multi # urls.txt if [[ "$ListHostAndDomainRegexes" == *"$arg2"* ]]; then InputFile="$arg4" multiCount=$arg3 if [ ! -f "${InputFile}" ] && [ -f "${WorkDir}/${InputFile}" ]; then InputFile="${WorkDir}/${InputFile}" fi if [ ! -f "${InputFile}" ]; then printf "Unable to read file %s! [2]\\n" "$1" exit 1 else isHostMatchFound=false _hostcode="" _hostnick="" _hostfuncprefix="" _hostdomainregex="" readarray -d '@' -t arrHostAndDomainRegexes <<< "${ListHostAndDomainRegexes}" for hline in "${arrHostAndDomainRegexes[@]}"; do if [[ "$hline" == "" ]] || [[ ! "$hline" == *":"* ]]; then continue fi chnk1="${hline%%:*}" _hostcode=$(echo $chnk1|cut -f2 -d '/') _hostnick=$(echo $chnk1|cut -f3 -d '/') _hostfuncprefix=$(echo $chnk1|cut -f4 -d '/') _hostdomainregex="${hline#*\:}" if [ "$arg2" == "$_hostcode" ] ; then lineCount=$(grep -Ei $_hostdomainregex "${InputFile}" | wc -l | awk '{ print $1 }') printf "%s has ${GREEN}%d ${YELLOW}$_hostnick${NC} files to download.\\n" "${InputFile}" $lineCount for ((k=1; k<=$multiCount; k++)); do LaunchTerminal "$_hostcode" ${InputFile} done exit 0 fi done fi else InputFile="$arg3" if [[ "$arg1" == "multi" ]] && \ [[ "$arg2" == "2" || "$arg2" == "3" || "$arg2" == "4" || \ "$arg2" == "5" || "$arg2" == "6" || "$arg2" == "7" || "$arg2" == "8" ]] ; then multiCount=$arg2 fi if [ ! -f "${InputFile}" ] && [ -f "${WorkDir}/${InputFile}" ]; then InputFile="${WorkDir}/${InputFile}" fi if [ ! -f "${InputFile}" ]; then printf "Unable to read file %s! [3]\\n" "$1" exit 1 else isHostMatchFound=false _hostcode="" _hostnick="" _hostfuncprefix="" _hostdomainregex="" readarray -d '@' -t arrHostAndDomainRegexes <<< "${ListHostAndDomainRegexes}" lineCount=0 for hline in "${arrHostAndDomainRegexes[@]}"; do if [[ "$hline" == "" ]] || [[ ! "$hline" == *":"* ]]; then continue fi chnk1="${hline%%:*}" _hostcode=$(echo $chnk1|cut -f2 -d '/') _hostnick=$(echo $chnk1|cut -f3 -d '/') _hostfuncprefix=$(echo $chnk1|cut -f4 -d '/') _hostdomainregex="${hline#*\:}" if [ "$_hostfuncprefix" == "direct" ]; then lineCount2=$(grep -Ei '^direct=http' "${InputFile}" | wc -l | awk '{ print $1 }') lineCount=$((lineCount + lineCount2)) elif [ "$_hostcode" == "fdot" ] && [ "${EnableFiledotProcessing}" == "true" ]; then lineCount2=$(grep -Ei $_hostdomainregex "${InputFile}" | wc -l | awk '{ print $1 }') lineCount=$((lineCount + lineCount2)) else lineCount2=$(grep -Ei $_hostdomainregex "${InputFile}" | wc -l | awk '{ print $1 }') lineCount=$((lineCount + lineCount2)) fi done printf "%s has ${GREEN}%d${NC} files to download.\\n" "${InputFile}" $lineCount for ((k=1; k<=$multiCount; k++)); do LaunchTerminal "allhosts" ${InputFile} done exit 0 fi fi else isHostMatchFound=false _hostcode="" _hostnick="" _hostfuncprefix="" _hostdomainregex="" readarray -d '@' -t arrHostAndDomainRegexes <<< "${ListHostAndDomainRegexes}" lineCount=0 foundhostnick="" for hline in "${arrHostAndDomainRegexes[@]}"; do if [[ "$hline" == "" ]] || [[ ! "$hline" == *":"* ]]; then continue fi chnk1="${hline%%:*}" _hostcode=$(echo $chnk1|cut -f2 -d '/') _hostnick=$(echo $chnk1|cut -f3 -d '/') _hostfuncprefix=$(echo $chnk1|cut -f4 -d '/') _hostdomainregex="${hline#*\:}" if [ "$arg1" == "$_hostcode" ]; then hostOnlyOrUrl="$1" InputFile="$2" if [ ! -f "${InputFile}" ] && [ -f "${WorkDir}/${InputFile}" ]; then InputFile="${WorkDir}/${InputFile}" fi if [ ! -f "${InputFile}" ]; then printf "Unable to read file %s! [host]\\n" "$1" exit 1 else lineCount2=$(grep -Ei $_hostdomainregex "${InputFile}" | wc -l | awk '{ print $1 }') lineCount=$((lineCount + lineCount2)) foundhostnick="$_hostnick" fi isHostMatchFound=true fi done if [ "$isHostMatchFound" == "true" ]; then printf "%s has ${GREEN}%d ${YELLOW}$foundhostnick${NC} files to download.\\n" "${InputFile}" $lineCount else InputFile="$1" if [ ! -f "${InputFile}" ] && [ -f "${WorkDir}/${InputFile}" ]; then InputFile="${WorkDir}/${InputFile}" fi if [ ! -f "${InputFile}" ]; then printf "Unable to read file %s! [main]\\n" "$1" exit 1 fi lineCount=$(grep -Ei '^(http|direct=http)' "${InputFile}" | wc -l | awk '{ print $1 }') printf "%s has ${GREEN}%d${NC} files to download.\\n" "${InputFile}" $lineCount fi fi CleanInputFile $InputFile HashFilelistTxt=$( sha1sum ${InputFile} | awk '{print $1}' ) if [ "$VerboseLoading" == "true" ]; then printf "SHA1: %s\\n\\n" "${HashFilelistTxt}" fi mainLoopControl=true while ${mainLoopControl} do mainLoopControl="$LoopThroughFileUntilComplete" BeginProcessing "${InputFile}" qChkLineCount=$(grep -Ei '^(http|direct=http)' "${InputFile}" | wc -l | awk '{ print $1 }') if ((qChkLineCount <= 0)) ; then if DoneProcessingAllUrls "${InputFile}" ; then if [ "${AutoShowMadStatus}" == "true" ] ; then echo -e "${RED}❤${GREEN}Done! ${YELLOW}Me0W!${NC} :D" madStatus "${InputFile}" fi exit 0 else continue fi fi fileCount=1 lockCount=0 sed 's/^[[:space:]]*// ; s/[[:space:]]*$//' "${InputFile}" | while IFS= read -r line do filename_override="" line="${line//[$'\t\r\n']}" if [[ $line =~ \| ]] && grep -Eqi '^(http|direct=)' <<< "$line" && grep -Eqvi '^#' <<< "$line" ; then filename_override="${line##*\|}" line="${line%%\|*}" fi if [ -f "${InputFile}" ] ; then currentHashFilelistTxt=$( sha1sum "${InputFile}" | awk '{print $1}' ) else currentHashFilelistTxt=$( sha1sum ${InputFile} | awk '{print $1}' ) fi if [ -f "${WorkDir}/clear" ] ; then clear printf "[${PINK}ClearScreen${NC}] Clear file was found. (removing and clearing)\\n" if [ -f "${WorkDir}/clear" ] ; then rm -f "${WorkDir}/clear" fi elif ((qChkLineCount > 0)) && [ ! "${HashFilelistTxt}" == "${currentHashFilelistTxt}" ] && \ [ "${ClearScreenOnAutoReload}" == "true" ] && [ "${AutoReloadOnFilelistTxtChanges}" == "true" ] ; then clear printf "[${PINK}ClearScreen${NC}] Auto-clearing screen. (${InputFile} change detected)\\n" fi if [ -f "${WorkDir}/stop" ] ; then printf "\\n%s\\n" "--------------------------------------------" printf "[${PINK}Stop${NC}] stop file was found.\\n" printf "%s\\n" "--------------------------------------------" rm -f "${WorkDir}/stop" exit 1 fi if [ -f "${WorkDir}/restart" ] ; then printf "\\n%s\\n" "--------------------------------------------" printf "[${PINK}Restart${NC}] restart file was found.\\n" printf "%s\\n" "--------------------------------------------" rm -f "${WorkDir}/restart" if [ -f "${WorkDir}/reload" ] ; then rm -f "${WorkDir}/reload" fi ReloadScript "$@" exit 1 elif [ -f "${WorkDir}/reload" ] ; then printf "\\n%s\\n" "--------------------------------------------" printf "[${PINK}Reload${NC}] reload file was found.\\n" printf "%s\\n" "--------------------------------------------" rm -f "${WorkDir}/reload" if [ -f "${WorkDir}/restart" ] ; then rm -f "${WorkDir}/restart" fi ReloadScript "$@" exit 1 elif ((qChkLineCount > 0)) && [ "${AutoReloadOnFilelistTxtChanges}" == "true" ] && \ [ ! "${HashFilelistTxt}" == "${currentHashFilelistTxt}" ]; then printf "\\n%s\\n" "--------------------------------------------" printf "[${PINK}Reload${NC}] ${InputFile} was modified.\\n" printf "%s\\n" "--------------------------------------------" ReloadScript "$@" exit 1 fi if [[ $line =~ ^STOP! ]] ; then printf "\\n%s\\n" "----------------------------------------------------" printf "Stopping -- STOP! keyword encountered.\\n" printf "%s\\n" "----------------------------------------------------" exit 1 fi if grep -ie '^folder=' <<< "$line" > /dev/null ; then MoveToFolder=${line:7} #get everything starting at pos 7+ MoveToFolder=$(sanitize_file_or_folder_name "${MoveToFolder}") continue ## resume next line ## fi if grep -ie '^#[^ ].*=.\+' <<< "$line" > /dev/null ; then line=${line:1} key=${line%%=*} val=${line#*=} UrlsVars[$key]="$val" continue ## resume next line ## fi if [[ ${line} =~ ^(http|https|direct=http|direct=https):// ]] ; then if ! PreProcessUrl "$line" ; then continue fi fi if [ ! -z "${hostOnlyOrUrl}" ] ; then _hostcode="" _hostnick="" _hostfuncprefix="" _hostdomainregex="" readarray -d '@' -t arrHostAndDomainRegexes <<< "${ListHostAndDomainRegexes}" isHostMatchFound=false isSupportedHost=false isSkipOkay=false if [[ ${line} =~ ^# ]] ; then continue elif [[ $remote_url =~ ^direct=(http|https):// ]] ; then continue elif [[ ${line} =~ ^http ]] ; then remote_url=${line} if [[ ${remote_url} =~ ^http: ]] ; then remote_url=${remote_url/http:/https:} fi else printf "Ignore garbage line.\\n" > /dev/null continue fi for hline in "${arrHostAndDomainRegexes[@]}"; do if [[ "$hline" == "" ]] || [[ ! "$hline" == *":"* ]]; then continue fi chnk1="${hline%%:*}" _hostcode=$(echo $chnk1|cut -f2 -d '/') _hostnick=$(echo $chnk1|cut -f3 -d '/') _hostfuncprefix=$(echo $chnk1|cut -f4 -d '/') _hostdomainregex="${hline#*\:}" if [[ $remote_url =~ $_hostdomainregex ]] ; then isSupportedHost=true if [ "${hostOnlyOrUrl}" == "$_hostcode" ] ; then isHostMatchFound=true CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ] ; then if ((CatnapCount <= 0)) ; then echo -e "${YELLOW}Lock Exists (Skipping)${NC}: ./.flocks/${CLEANSTRING}" fi fileCount=$((fileCount + 1)) isSkipOkay=true break fi printf "\nGetting ${YELLOW}$_hostnick${NC} file ${GREEN}%d${NC} of ${GREEN}%d${NC} ${PINK}($_hostcode urls)${NC}\\n" $fileCount $lineCount if [ ! "${filename_override}" == "" ] ; then printf "[${BLUE}FilenameOverride${NC}]: %s\\n" $filename_override fi if [ ! "${MoveToFolder}" == "" ] ; then printf "[${BLUE}MoveToFolder${NC}]: %s\\n" $MoveToFolder fi printf "[DownloadUrl]: %s\\n" "${remote_url}" mkdir -p "${WorkDir}/.flocks" touch "${WorkDir}/.flocks/${CLEANSTRING}" mkdir -p "${WorkDir}/.inflight" ${_hostfuncprefix}_DownloadFile "${remote_url}" "${fileCount}" fileCount=$((fileCount + 1)) break fi fi done if [ "$isSkipOkay" == "true" ]; then continue elif [ "$isSupportedHost" == "false" ]; then printf "${RED}Invalid url (bad format or unsupported host [m1]):${NC} \\n%s\\n" $remote_url badUrlDownload "${remote_url}" continue fi else if [[ ${line} =~ ^# ]] ; then continue elif [[ ${line} =~ ^direct=http ]]; then remote_url=${line/direct=/} if [[ ${remote_url} =~ ^http: ]] ; then remote_url=${remote_url/http:/https:} fi CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ] ; then if ((CatnapCount <= 0)) ; then echo -e "${YELLOW}Lock Exists (Skipping)${NC}: ./.flocks/${CLEANSTRING}" fi fileCount=$((fileCount + 1)) continue fi printf "\nGetting ${YELLOW}direct${NC} file ${GREEN}%d${NC} of ${GREEN}%d${NC}\\n" $fileCount $lineCount if [ ! "${filename_override}" == "" ] ; then printf "[${BLUE}FilenameOverride${NC}]: %s\\n" $filename_override fi if [ ! "${MoveToFolder}" == "" ] ; then printf "[${BLUE}MoveToFolder${NC}]: %s\\n" $MoveToFolder fi printf "[${BLUE}DirectUrl${NC}]: %s\\n" "${remote_url}" mkdir -p "${WorkDir}/.flocks" touch "${WorkDir}/.flocks/${CLEANSTRING}" mkdir -p "${WorkDir}/.inflight" direct_DownloadFile "${remote_url}" "${fileCount}" fileCount=$((fileCount + 1)) elif [[ ${line} =~ ^http ]] ; then remote_url=${line} if [[ ${remote_url} =~ ^http: ]] ; then remote_url=${remote_url/http:/https:} fi else printf "Ignore garbage line.\\n" > /dev/null continue fi _hostcode="" _hostnick="" _hostfuncprefix="" _hostdomainregex="" readarray -d '@' -t arrHostAndDomainRegexes <<< "${ListHostAndDomainRegexes}" isHostMatchFound=false isSkipOkay=false for hline in "${arrHostAndDomainRegexes[@]}"; do if [[ "$hline" == "" ]] || [[ ! "$hline" == *":"* ]]; then continue fi chnk1="${hline%%:*}" _hostcode=$(echo $chnk1|cut -f2 -d '/') _hostnick=$(echo $chnk1|cut -f3 -d '/') _hostfuncprefix=$(echo $chnk1|cut -f4 -d '/') _hostdomainregex="${hline#*\:}" if [[ $remote_url =~ $_hostdomainregex ]] ; then isHostMatchFound=true CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} if [ -f "${WorkDir}/.flocks/${CLEANSTRING}" ] ; then if ((CatnapCount <= 0)) ; then echo -e "${YELLOW}Lock Exists (Skipping)${NC}: ./.flocks/${CLEANSTRING}" fi fileCount=$((fileCount + 1)) isSkipOkay=true break fi printf "\nGetting ${YELLOW}$_hostnick${NC} file ${GREEN}%d${NC} of ${GREEN}%d${NC}\\n" $fileCount $lineCount if [ ! "${filename_override}" == "" ] ; then printf "[${BLUE}FilenameOverride${NC}]: %s\\n" $filename_override fi if [ ! "${MoveToFolder}" == "" ] ; then printf "[${BLUE}MoveToFolder${NC}]: %s\\n" $MoveToFolder fi printf "[DownloadUrl]: %s\\n" "${remote_url}" mkdir -p "${WorkDir}/.flocks" touch "${WorkDir}/.flocks/${CLEANSTRING}" mkdir -p "${WorkDir}/.inflight" ${_hostfuncprefix}_DownloadFile "${remote_url}" "${fileCount}" fileCount=$((fileCount + 1)) break fi done if [ "$isSkipOkay" == "true" ]; then continue elif [ "$isHostMatchFound" == "false" ]; then printf "${RED}Invalid url (bad format or unsupported host [m*]):${NC} \\n%s\\n" $remote_url badUrlDownload "${remote_url}" continue fi fi done #loop through the file line by line qChkLineCount=$(grep -Ei '^(http|direct=http)' "${InputFile}" | wc -l | awk '{ print $1 }') if ((qChkLineCount > 0)) ; then if [ ! -z $hostOnlyOrUrl ] ; then echo -e "${NC}" echo -e "${YELLOW}Unprocessed / Skipped URL(s) Found:${NC}" echo -e "Most likely from a different host than ${YELLOW}$hostOnlyOrUrl${NC}, another terminal is downloading it, or a flock exists...${NC}" echo -e "Switching back to processing ${YELLOW}all host's${NC} urls...${NC}" ReloadScript "" exit 0 else if [ "$LoopThroughFileUntilComplete" == "false" ]; then if [ ! -z $hostOnlyOrUrl ] ; then echo -e "${NC}" echo -e "${YELLOW}Unprocessed / Skipped URL(s) Found:${NC}" echo -e "Most likely from a different host than ${YELLOW}$hostOnlyOrUrl${NC}, another terminal is downloading it, or a flock exists...${NC}" fi if DoneProcessingAllUrls "${InputFile}" ; then exit 0 else continue fi else CatnapCount=$((CatnapCount + 1)) if ((CatnapCount > 1)); then for ((a=1; a<=4; a++)); do printf "\033[1A\r" printf "\33[2K\r" done fi echo -e "${NC}" echo -e "${YELLOW}Unprocessed / Skipped URL(s) Found:${NC}" echo -e "Most likely another terminal is downloading it or a flock exists...${NC}" echo -e "${YELLOW}Catnapping${NC} for ${CatnapDuration} mins to allow ${YELLOW}$qChkLineCount${NC} download(s) to finish... ${YELLOW}zZzZzZ${NC} ${BLUE}x$CatnapCount${NC}" sleep ${CatnapDuration}m continue fi fi else if DoneProcessingAllUrls "${InputFile}" ; then exit 0 else continue fi fi done #loop until all urls processed IFS=$backupIFS