From 3049b3e305a6ec7b9086158349ab194779bfcbbd Mon Sep 17 00:00:00 2001 From: kittykat Date: Tue, 12 Nov 2024 16:59:58 +0000 Subject: [PATCH] # 2024.11.12 - [mad] Fix uploads.txt status marking (urls / messages) containg '&' chars # 2024.11.12 - [up_torup] Max upload filesize changed to 150MB --- documentation/!Changelog (Historical).txt | 23 +++++++++++++ documentation/README-upload_hosts.txt | 2 +- hosts/up_torup.sh | 4 +-- mad.sh | 42 ++++++----------------- 4 files changed, 36 insertions(+), 35 deletions(-) diff --git a/documentation/!Changelog (Historical).txt b/documentation/!Changelog (Historical).txt index 9b0253c..145a984 100755 --- a/documentation/!Changelog (Historical).txt +++ b/documentation/!Changelog (Historical).txt @@ -3,6 +3,29 @@ # # ---------- Initial release with MAD Uploader functionality ---------- +# 2024.10.14 - [pjscloud] Create pjscloud.sh plugin -- use PhantomJSCloud to get a response from a url +# * Change pixeldrain ViewPump to use pjscloud.sh plugin. (default enabled) +# * Testing usage on other javascript required response hosts (ie. daily, hexload) +# 2024.10.13 - [hexload] Add detection of new download2 type: Wait x seconds to create download link +# * Add detection of download2b type loop (still working on solution, exit loop for now) +# 2024.10.13 - [ocr_captcha] Add tessdata_type: NUMBERONLY, ALPHAONLY, ALPHANUMERIC, and not specified +# 2024.10.12 - [mad] Alternate with nohead function on direct download for hosts that do not allow head +# 2024.10.12 - [turboonion/up_turboonion] Add turbo.onion as an upload and download host +# 2024.10.12 - [tempfileme/up_tempfileme] Add tempfile.me as upload and download host +# 2024.10.11 - [tempsh/up_tempsh] Add temp.sh as upload and download host +# 2024.10.10 - [mad] Add cleanup of input file on processing (remove trailing whitespace) +# 2024.10.08 - [_audit] Add _audit.sh in release bundle which creates mad-audit-*.log for http/curl calls +# * ./_audit.sh +# 2024.10.07 - [mad] Add mad audit. Prints all lines that use http, curl, tor_curl in script files +# * Displays color hightlighted, file, line #, and matches in scripts. +# * ./mad.sh audit +# 2024.10.06 - [1fichier] Add new removal response "..file has been deleted following a notification .." +# 2024.10.06 - [mad] Update direct download no response processing to fail / retry +# 2024.10.05 - [dailyuploads] Fix head request, filename fallback, and add head retries +# 2024.10.05 - [dailyuploads] Add wrong captcha response +# 2024.10.04 - Add README-uploads.txt in ./documentation (cheat sheet of hosts, retention, max size) +# 2024.10.02 - [up_kouploader] Add kouploader.jp as upload host (240MB node) +# 2024.10.01 - [up_axfc] Add axfc.net as upload host # 2024.09.30 - [up_firestorage] Add firestorage.jp as upload host # 2024.09.29 - [free4e/up_free4e] Add free4e.com as download and upload host # 2024.09.29 - [harrault/up_harrault] Add harrault.fr as download and upload host diff --git a/documentation/README-upload_hosts.txt b/documentation/README-upload_hosts.txt index 4361992..590508b 100755 --- a/documentation/README-upload_hosts.txt +++ b/documentation/README-upload_hosts.txt @@ -21,10 +21,10 @@ Max Size . HostCode . Nickname . Notes * 2GB axfc axfc.net 90d+ inactive expiry - 1GB kraken krakenfiles.com 90d inactive expiry 1GB ansh anonsharing.com 6mo expiry - 1GB torp TorUp.onion 30d inactive expiry 300MB trbo turbo.onion ~40d expiry 250MB upev uploadev.org 90d inactive expiry * 240MB ko kouploader.jp 5mo expiry (240MB max) + 150MB torp TorUp.onion 30d inactive expiry 100MB bow bowfile.com 20d inactive expiry 100MB yolo yolobit ?? expiry 100MB nofile nofile.org ?? expiry diff --git a/hosts/up_torup.sh b/hosts/up_torup.sh index 6e58041..c47e0f6 100644 --- a/hosts/up_torup.sh +++ b/hosts/up_torup.sh @@ -3,7 +3,7 @@ #! Version: 2024.11.09 #! Desc: Add support for uploading files to ktgzpea2b76u7fgemiibp4a76onyybo4fw5gbsagtm6jrjzmgivppyyd.onion #! Info: Files are accessible at http://ktgzpea2b76u7fgemiibp4a76onyybo4fw5gbsagtm6jrjzmgivppyyd.onion/download/ -#! MaxSize: 1GB +#! MaxSize: 150MB #! Expire: 30d inactive expiry #! Usage: Copy this file into the ./${ScriptDir}/hosts/ folder #! @@ -44,7 +44,7 @@ torp_UploadFile() { fileAlreadyDone=false tor_identity="${RANDOM}" UploadTicket="${WorkDir}/.flocks/upload_${_hostCode}_${filepath//[^a-zA-Z0-9]/}" - MaxUploadSizeInBytes=1073741824 + MaxUploadSizeInBytes=157286400 fsize=$(GetFileSize "$filepath" "false") if ((fsize > MaxUploadSizeInBytes)); then m -f "${UploadTicket}" diff --git a/mad.sh b/mad.sh index 21a6436..fc609b9 100644 --- a/mad.sh +++ b/mad.sh @@ -34,6 +34,8 @@ ScriptVersion=2024.11.12 #================================================= # Recent Additions +# 2024.11.12 - [mad] Fix uploads.txt status marking (urls / messages) containg '&' chars +# 2024.11.12 - [up_torup] Max upload filesize changed to 150MB # 2024.11.12 - [up_uploadee] Add upload.ee as an upload host # 2024.11.11 - [up_offcat] Add Offshore.cat as upload host # 2024.11.11 - [mad] Add OffShore.cat Upload ApiKeys section to allow using Offshore.cat as upload host @@ -969,11 +971,7 @@ successUpload() { mkdir -p "${WorkDir}/data" echo -e "$dateStamp [OK] file: ${filename}, host: ${HostCode}, dl: ${downloadLink}, ticket: ${cTicket}, size: ${filesize}, path: ${filepath}" >> "${WorkDir}/data/uploads_completed.txt" if [ ! -z "$InputFile" ]; then - if [ "${AutoCommentOnCompletion}" == "true" ] ; then - sed -i -e "s>^${pLine}.*>#& #OK# $downloadLink>g" "${InputFile}" #processed line - else - sed -i -e "s>^$pLine.*>#& $downloadLink>g" "${InputFile}" #processed line - fi + sed -i -e "s>^${pLine}.*>#& #OK# ${downloadLink//&/\\&}>g" "${InputFile}" #processed line fi dateStamp=$(date '+%Y/%m/%d %H:%M:%S') mkdir -p "${WorkDir}/uploads" @@ -998,11 +996,7 @@ successUploadExists() { dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "[EXISTS] ${filename}, ${HostCode}, ${downloadLink}" >> "${WorkDir}/uploads/results.txt" if [ ! -z "$InputFile" ]; then - if [ "${AutoCommentOnCompletion}" == "true" ] ; then - sed -i -e "s>^${pLine}.*>#& #OK# (Upload exists) ${message}>g" "${InputFile}" #processed line - else - sed -i -e "s>^$pLine.*>#&>g" "${InputFile}" #processed line - fi + sed -i -e "s>^${pLine}.*>#& #OK# (Upload exists) ${message//&/\\&}>g" "${InputFile}" #processed line fi UploadTicket="${WorkDir}/.flocks/upload_${HostCode}_${filepath//[^a-zA-Z0-9]/}" if [ -f "${UploadTicket}" ]; then @@ -1023,11 +1017,7 @@ failedUpload() { dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "[FAIL] ${HostCode}, ${filename}, ${message}" >> "${WorkDir}/uploads/results.txt" if [ ! -z "$InputFile" ]; then - if [ "${AutoCommentOnCompletion}" == "true" ] ; then - sed -i -e "s>^${pLine}.*>#& #FAIL# ${message}>g" "${InputFile}" #processed line - else - sed -i -e "s>^$pLine.*>#&>g" "${InputFile}" #processed line - fi + sed -i -e "s>^${pLine}.*>#& #FAIL# ${message//&/\\&}>g" "${InputFile}" #processed line fi dateStamp=$(date '+%Y/%m/%d %H:%M:%S') mkdir -p "${WorkDir}/uploads" @@ -1051,11 +1041,7 @@ failedRetryUpload() { dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "[RETRY] ${HostCode}, ${filename}, ${message}" >> "${WorkDir}/uploads/results.txt" if [ ! -z "$InputFile" ]; then - if [ "${AutoCommentOnCompletion}" == "true" ] ; then - sed -i -e "s>^${pLine}.*>#& #RETRY# ${message}>g" "${InputFile}" #processed line - else - sed -i -e "s>^$pLine.*>#&>g" "${InputFile}" #processed line - fi + sed -i -e "s>^${pLine}.*>#& #RETRY# ${message//&/\\&}>g" "${InputFile}" #processed line fi dateStamp=$(date '+%Y/%m/%d %H:%M:%S') mkdir -p "${WorkDir}/uploads" @@ -1080,11 +1066,7 @@ skipFailedUpload() { dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "[SKIP/FAIL] ${HostCode}, ${filename}, ${message}" >> "${WorkDir}/uploads/results.txt" if [ ! -z "$InputFile" ]; then - if [ "${AutoCommentOnCompletion}" == "true" ] ; then - sed -i -e "s>^${pLine}.*>#& #FAIL# (Skip) ${message}>g" "${InputFile}" #processed line - else - sed -i -e "s>^$pLine.*>#&>g" "${InputFile}" #processed line - fi + sed -i -e "s>^${pLine}.*>#& #FAIL# (Skip) ${message//&/\\&}>g" "${InputFile}" #processed line fi dateStamp=$(date '+%Y/%m/%d %H:%M:%S') mkdir -p "${WorkDir}/uploads" @@ -1104,11 +1086,7 @@ uploadBadInputLine() { dateStamp=$(date '+%Y/%m/%d %H:%M:%S') echo -e "[BADLINE] ${pLine}${message}" >> "${WorkDir}/uploads/results.txt" if [ ! -z "$InputFile" ]; then - if [ "${AutoCommentOnCompletion}" == "true" ] ; then - sed -i -e "s>^${pLine}.*>#& #RETRY# (Bad Line)${message}>g" "${InputFile}" #processed line - else - sed -i -e "s>^$pLine.*>#&>g" "${InputFile}" #processed line - fi + sed -i -e "s>^${pLine}.*>#& #RETRY# (Bad Line)${message//&/\\&}>g" "${InputFile}" #processed line fi dateStamp=$(date '+%Y/%m/%d %H:%M:%S') mkdir -p "${WorkDir}/uploads" @@ -2731,9 +2709,9 @@ MadUploadFromFileTxt() { mkdir -p "${WorkDir}/uploads" clear echo -e "${BLD}" - echo -e "${PINK}:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:" + echo -e "${PINK}:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:" echo -e ":${NC} ${GREEN}MAD${PINK} Uploader${NC} :: Transfer files in uploads.txt${PINK}${BLD} :" - echo -e ":-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:${NC}\\n" + echo -e ":-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:-:${NC}\\n" OnLoad $0 $@ if ((torPort >= 9050 && torPort <= 9150)); then printf "TorIp: ${GREEN}$TorIp${NC}, Tor listening on port ${GREEN}$torPort${NC}, "