' | grep -oP ' /dev/null ; then if $is_password_protected; then echo -e "${RED}ERROR: Incorrect password${NC}\nSince this download required a password, you might have copied it incorrectly?" passwordProtectedDownload "${remote_url}" exitDownloadError=true return 2 else echo -e "${RED}ERROR: Could not find CDN URL${NC}" if [[ "${finalAttempt}" == "true" ]] ; then noCdnDownload "${remote_url}" fi return 1 fi fi echo -e "${GREEN}CDN URL has been found!${NC}" } #! #! ----------- (3) Fetch File / Download File Function --------------- # #! fich_GetFile() { local filecnt=$1 local num_attempt=$2 local finalAttempt=$3 local filename=$4 echo -e "\n${RED}❤${NC} Saving ${GREEN}${remote_url##*/?}${NC} to ${GREEN}${filename}${NC}" CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} for ((j=1; j<=4; j++)); do trap "rm -f ${WorkDir}/.flocks/${CLEANSTRING}; echo ""; tput cnorm; exit" 0 1 2 3 6 15 file_header=$(tor_curl_request --insecure -sSIL -e "${remote_url}" "${target_file_link}") file_size_bytes=$(grep -oPi '(?<=content-length: ).*' <<< "$file_header") file_size_bytes=${file_size_bytes//[$'\t\r\n']} download_inflight_path="${WorkDir}/.inflight/" completed_location="${WorkDir}/downloads/" file_path="${download_inflight_path}${filename}" if [[ "${DebugAllEnabled}" == "true" ]] ; then debugHtml "${remote_url##*/?}" "fich_savehead${num_attempt}_${j}" "target_file_link: ${target_file_link}"$'\n'"${file_header}" fi if [[ -z "$file_header" ]] || [[ -z "$file_size_bytes" ]]; then continue else break fi done if grep -Eqi "200 OK" <<< "${file_header}" > /dev/null ; then echo "We good" > /dev/null elif grep -Eqi "410 Gone" <<< "${file_header}" > /dev/null ; then echo -e "${RED}ERROR: Failed to retrieve file header (410 Gone).${NC}\nThis could be due to 1fichier experiencing a temporary issue." if [[ "${finalAttempt}" == "true" ]] ; then failedDownload "${remote_url}" "${filename}" "410 Gone" fi return 1 elif grep -Eqi "403 Forbidden" <<< "${file_header}" > /dev/null ; then echo -e "${RED}ERROR: Failed to retrieve file header (403 Forbidden).${NC}\nThis could be due to 1fichier experiencing a temporary issue." if [[ "${finalAttempt}" == "true" ]] ; then failedDownload "${remote_url}" "${filename}" "403 Forbidden" fi return 1 else echo -e "${RED}ERROR: Failed to retrieve file header (Unknown Head Response).${NC}\nThis could be due to 1fichier experiencing a temporary issue." if [[ "${finalAttempt}" == "true" ]] ; then failedDownload "${remote_url}" "${filename}" "Unknown Head Response" fi return 1 fi size_value=$(echo $size | cut -f1 -d' ') size_unit=$(echo $size | cut -f2 -d' ') case $size_unit in KB|Ko) size_bytes=$(echo "$size_value * 1000" | bc);; MB|Mo) size_bytes=$(echo "$size_value * 1000 * 1000" | bc);; GB|Go) size_bytes=$(echo "$size_value * 1000 * 1000 * 1000" | bc);; TB|To) size_bytes=$(echo "$size_value * 1000 * 1000 * 1000 * 1000" | bc);; *) size_bytes=$size_value;; esac percent_threshold=0.01 # 1% error threshold, should be fine percent_numerator=$(echo "$file_size_bytes - $size_bytes" | bc) percent_diff=$(echo "${percent_numerator/-/} / (($file_size_bytes + $size_bytes) / 2)" | bc -l) if (( $(echo "$percent_diff > $percent_threshold" |bc -l) )); then echo -e "${RED}ERROR: The difference between the advertised and retrieved file size is too big${NC}\nThis is most likely due to someone else taking the slot or some other error along the way." echo -e "AdvertisedSize: $size ($size_bytes), Size: $file_size_bytes, Diff: $percent_diff" if [[ "${finalAttempt}" == "true" ]] ; then droppedSizeBadDownload "${remote_url}" "${filename}" "${file_size_bytes}" "${size}" fi return 1 fi flockDownload="${WorkDir}/.flocks/${filename//[^a-zA-Z0-9\.\_\-]/}.flock" if CheckFileSize "${remote_url}" ${file_size_bytes} ; then return 1 fi if CheckDownloadExists "$remote_url" "$MoveToFolder" "$filecnt" "$filename" "$file_path" "$completed_location" ; then return 1 fi echo "${remote_url//[^a-zA-Z0-9]/}" > $flockDownload touch "${WorkDir}/.flocks/${remote_url//[^a-zA-Z0-9]/}" tor_identity="${RANDOM}" CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} trap "rm -f ${WorkDir}/.flocks/${CLEANSTRING}; rm -f $flockDownload; echo ""; tput cnorm; exit" 0 1 2 3 6 15 tor_curl_request --insecure -e "${remote_url}" "${target_file_link}" -C - -o "${file_path}" rm -f "$flockDownload"; received_file_size=0 if [[ -f "$file_path" ]] ; then received_file_size=$(stat --format="%s" "$file_path" | tr -d '[:space:]') fi if [[ "${received_file_size}" -ne "${file_size_bytes}" ]]; then echo -e "${RED}ERROR: Size mismatch after downloading${NC}\nPerhaps you or 1fichier lost connection for a while?" if [[ "${finalAttempt}" == "true" ]] ; then droppedSizeBadDownload "${remote_url}" "${filename}" "${received_file_size}" fi return 1 fi ProcessCompletedDownload "$remote_url" "$MoveToFolder" "$filecnt" "$filename" "$file_size_bytes" "$completed_location" "$file_path" } #! #! --------------- Host Extra Functions ------------------- # #! fich_FindEmptySlot() { mkdir -p "${WorkDir}/.temp" local lockfile="${WorkDir}/.temp/lockfile.lock" timer_start=$(date +%s) while [[ -e $lockfile ]]; do CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} trap "rm -f ${WorkDir}/.flocks/${CLEANSTRING}; rm -f $lockfile; echo ""; tput cnorm; exit" 0 1 2 3 6 15 timer_duration=$(($(date +%s) - timer_start)) echo -e "Lock file in place, time elapsed: $timer_duration seconds" sleep 1 printf "\033[F" tput el done touch $lockfile echo -e "\nSearching for a usable exit node" mkdir -p "${WorkDir}/.temp" rm -f "${WorkDir}/.temp/fich_adz_parameter" sleep 0.2 instances="10" timer_start=$(date +%s) CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} trap 'rm -f "${WorkDir}/.flocks/${CLEANSTRING}"; rm -f $lockfile; tput el; echo -e "${RED}Aborted by user!${NC}"; for i in $(seq 1 $instances); do tput el; echo ""; done; rm -f "${WorkDir}/.temp/fich_cookie_jar"; rm -f "${WorkDir}/.temp/fich_adz_parameter"; rm -f "${WorkDir}/.temp/tor_identity"; echo -e "${RED}Aborted by user!${NC}"; tput cnorm; exit' 0 1 2 3 6 15 for instance_no in $(seq 1 $instances); do fich_FetchTorExitNode "${instance_no}" ${instances} & sleep 0.1 done wait CLEANSTRING=${remote_url//[^a-zA-Z0-9]/} trap "rm -f ${WorkDir}/.flocks/${CLEANSTRING}; rm -f $lockfile; rm -f "${WorkDir}/.temp/fich_cookie_jar"; rm -f "${WorkDir}/.temp/fich_adz_parameter"; rm -f "${WorkDir}/.temp/tor_identity"; echo ""; tput cnorm; exit" 0 1 2 3 6 15 if [[ $(stat -c '%s' "${WorkDir}/.temp/fich_adz_parameter") -lt 1 ]]; then echo -e "${RED}One of the instances ran out of attempts, try again later${NC}" return 1 fi timer_end=$(date +%s) timer_duration=$((timer_end - timer_start)) tput el echo -e "${GREEN}Search took ${timer_duration} seconds${NC}" rm -f $lockfile } fich_FetchTorExitNode() { local max_attempts=${CircuitRetries} local iter=0 instance_no="$1" time_out=5 instances="$2" fich_cookie_jar="" tor_identity="" fich_adz_parameter="" PAGE="" mkdir -p "${WorkDir}/.temp" while :; do if [[ "${iter}" -ge "${max_attempts}" ]]; then rm -f "$fich_cookie_jar" if [[ -s "${WorkDir}/.temp/fich_adz_parameter" ]]; then rm -f "$fich_cookie_jar" exit 1 fi echo "" > "${WorkDir}/.temp/fich_adz_parameter" for i in $(seq 1 "$instances"); do echo ""; done echo -e "\n${RED}Instance ${instance_no} ran out of attempts${NC}" exit 1 fi ((iter++)) if [[ -s "${WorkDir}/.temp/fich_adz_parameter" ]]; then tput el printf "Closing instance %s\r" "${instance_no}" rm -f "${fich_cookie_jar}" # new exit 1 fi for ((i=1; i<=instance_no; i++)) ; do printf "\\n" done printf "Instance %s \t| Attempt %s \r" "${instance_no}" "${iter}/${max_attempts}" for ((i=1; i<=instance_no; i++)) ; do printf "\033[F" done if [[ ! $fich_adz_parameter ]]; then rm -f "$fich_cookie_jar" fich_cookie_jar=$(mktemp "${WorkDir}/.temp/fich_cookies""${instance_no}"".XXXXXX") fi tor_identity="${RANDOM}" trap "rm -f ${fich_cookie_jar}; echo ""; tput cnorm; exit" 0 1 2 3 6 15 PAGE=$(tor_curl_request --insecure --max-time "$ConnectTimeout" -c "${fich_cookie_jar}" -s "${remote_url}") if [[ -z ${PAGE} ]]; then rm -f "${fich_cookie_jar}" continue fi if grep -Eqi 'Warning !|Attention !' <<< "${PAGE}"; then rm -f "${fich_cookie_jar}" continue else fich_adz_parameter=$(grep -oPi 'name="adz" value="\K[^"]+' <<< "${PAGE}") if [[ $fich_adz_parameter ]]; then if [[ -s "${WorkDir}/.temp/fich_adz_parameter" ]]; then rm -f "$fich_cookie_jar" exit 1 fi echo "$fich_adz_parameter" > "${WorkDir}/.temp/fich_adz_parameter" echo "$fich_cookie_jar" > "${WorkDir}/.temp/fich_cookie_jar" echo "$tor_identity" > "${WorkDir}/.temp/fich_tor_identity" rm -f "$fich_cookie_jar" for i in $(seq 1 "$instances"); do echo ""; done echo "" tput el echo -e "${GREEN}Slot found by instance ${instance_no}${NC}" break fi fi done tput cnorm }