diff --git a/.github/scripts/test-file-manager.sh b/.github/scripts/test-file-manager.sh new file mode 100755 index 0000000000..7e4ef196dd --- /dev/null +++ b/.github/scripts/test-file-manager.sh @@ -0,0 +1,1297 @@ +#!/bin/bash +# shellcheck disable=SC2317 +# ##################################################################### # +# Integration test for file_manager +# +# Copyright 2026 Marc Gutt, Gutt.IT +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# Steps: +# - enables file_manager debug mode via /var/tmp/file.manager.debug +# - stops file_manager so the next start picks up the debug trigger +# - writes JSON for each action to /var/tmp/file.manager.active (as Control.php would do it) +# - waits for file_manager worker to process it and reads nchan output from debug file +# - verifies output +# - removes debug trigger and debug files on cleanup +# ##################################################################### # + +# settings +fm_job_json_file=/var/tmp/file.manager.active +fm_exitcode_file=/var/tmp/file.manager.exitcode +fm_stdout_file=/var/tmp/file.manager.status +fm_error_file=/var/tmp/file.manager.error +fm_file="/usr/local/emhttp/webGui/nchan/file_manager" +fm_debug_nchan_file="/var/tmp/file.manager.nchan.debug" +ssh_user=root +ssh_host=$1 +test_path=/mnt/disk1/fm_test +src_path="$test_path/src" +arc_path="$src_path/archives" +files_path="$src_path/files" +dst_path="$test_path/dst" +special_chars_name=$'utf8_файл\nspecial&chars\*file\$' +job_timeout=${TIMEOUT:-200} +IFS=' ' read -ra archive_multi_formats <<< "${MULTI:-tar.bz2 tar.lz4 tar.gz tar.xz tar.zst zip}" +IFS=' ' read -ra archive_single_formats <<< "${SINGLE:-bz2 gz lz4 xz zst}" +script_args=("$@") + +if [[ ! $ssh_host || $ssh_host == "-h" || $ssh_host == "--help" ]]; then + echo "Usage: $(basename -- "$0") host [filter...]" + echo "" + echo "Filters (any combination, space-separated):" + echo " compress, cmp, 16 compress tests (action 16)" + echo " extract, 17 extract tests (action 17)" + echo " arc compress and extract tests (actions 16, 17)" + echo " rename, rn, 2, 7 rename tests (actions 2, 7)" + echo " move, mv, 4, 9 move tests (actions 4, 9)" + echo " copy, cp, 3, 8 copy tests (actions 3, 8)" + echo " chmod, 12 chmod tests (action 12)" + echo " chown, 11 chown tests (action 11)" + echo " search, find, 15 search tests (action 15)" + echo " create, mk, 0 create folder tests (action 0)" + echo " delete, del, 1, 6 delete tests (actions 1, 6)" + echo "" + echo "ENV overrides:" + echo " TIMEOUT=N seconds to wait for a job to complete (default: 200)" + echo " MULTI='...' space-separated multi-file archive formats" + echo " default: tar.bz2 tar.lz4 tar.gz tar.xz tar.zst zip" + echo " SINGLE='...' space-separated single-file archive formats" + echo " default: bz2 gz lz4 xz zst" + echo "" + echo "Examples:" + echo " $(basename -- "$0") Tower # run all tests" + echo " $(basename -- "$0") myserver arc mv # compress/extract and rename/move" + echo " $(basename -- "$0") myserver 16 # compress and extract by action ID" + echo " MULTI='zip tar.gz' $(basename -- "$0") myserver arc" + exit 0 +fi + +# deploy this script to and run on remote host +if [[ $ssh_host && $0 != "/tmp/test-file-manager.sh" ]]; then + scp "$0" "$ssh_user@$ssh_host:/tmp/test-file-manager.sh" || { echo "Error: Failed to copy to remote host" 1>&2; exit 1; } + # prepend ENV overrides so they are available on the remote host + local_env="" + [[ ${TIMEOUT:-} ]] && local_env+="TIMEOUT=$(printf '%q' "$TIMEOUT") " + [[ ${MULTI:-} ]] && local_env+="MULTI=$(printf '%q' "$MULTI") " + [[ ${SINGLE:-} ]] && local_env+="SINGLE=$(printf '%q' "$SINGLE") " + # shellcheck disable=SC2086 + ssh -t "$ssh_user@$ssh_host" "${local_env}bash /tmp/test-file-manager.sh ${script_args[*]}" || { echo "Error: Failed to run test on remote host" 1>&2; exit 1; } + exit +fi + +# functions + +on_exit() { + local rc=$? + echo -e "\n=== exit ===" + stop_file_manager + if [[ $rc -eq 0 ]]; then + clean_up_created_files + else + echo " skipping cleanup (tests failed - files left for inspection in $dst_path)" + fi + exit $rc +} +on_signal() { + echo -e "\n=== signal ===" + stop_file_manager + echo " skipping cleanup (interrupted - files left for inspection in $dst_path)" + exit 1 +} +trap on_exit EXIT +trap on_signal INT TERM + +# check condition and print pass/fail +pass=0 +fail=0 +check() { + local label=$1 + local cond=$2 + if [[ $cond -eq 0 ]]; then + echo "[PASS] $label" + pass=$((pass + 1)) + else + echo "[FAIL] $label (did not happen)" + fail=$((fail + 1)) + fi +} + +remove_job_files() { + local timestamp + timestamp=$(date +%Y%m%d%H%M%S) + for f in "$fm_job_json_file" "$fm_exitcode_file" "$fm_stdout_file" "$fm_error_file"; do + if [[ -f $f ]]; then + # output first 10 and last 10 lines of each file for debugging before moving + echo " removing $f (first and last 10 lines):" + # return only unique lines if the file contains less then 20 lines to avoid duplicates in output + if [[ $(wc -l < "$f") -le 20 ]]; then + sed 's/^/ /' "$f" + else + { head -n 10 "$f"; echo "..."; tail -n 10 "$f"; } | sed 's/^/ /' + fi + # backup file with timestamp suffix + mv -v "$f" "/tmp/$(basename -- "$f")-$timestamp.bak" | sed 's/^/ /' + fi + done +} + +stop_file_manager() { + echo " stop file_manager if running..." + if pgrep -f "php.*$fm_file" >/dev/null; then + pkill -f "php.*$fm_file" + echo " stopped file_manager" + sleep 0.1 + if pgrep -f "php.*$fm_file" >/dev/null; then + echo "Error: failed to stop file_manager!" 1>&2 + exit 1 + fi + remove_job_files + else + echo " file_manager is not running" + fi +} + +# start file_manager in background +run_file_manager() { + echo " ensure file_manager is running..." + if ! pgrep -f "php.*$fm_file" >/dev/null; then + echo " starting file_manager..." + if [[ ! -f "$fm_file" ]]; then + echo "Error: file_manager executable not found" 1>&2 + exit 1 + fi + /usr/bin/php -q "$fm_file" & disown + sleep 0.1 + if ! pgrep -f "php.*$fm_file" >/dev/null; then + echo "Error: failed to start file_manager" 1>&2 + exit 1 + fi + else + echo " file_manager is already running" + fi +} + +# remove all files created by file manager actions +clean_up_created_files() { + echo -e "\n=== Clean up files created through file manager actions ===" + if [[ -d "$dst_path" ]]; then + rm -rf "$dst_path" || { echo "Error: failed to clean up $dst_path" 1>&2; exit 1; } + echo " cleaned up $dst_path" + else + echo " no files to clean up in $dst_path" + fi + # remove debug trigger and all debug files + rm -f /var/tmp/file.manager.debug /var/tmp/file.manager.*.debug +} + +# file manager is considered busy if any of the job files exist (job JSON, exit code, stdout, etc.) +is_action_in_progress() { + [[ -f $fm_job_json_file || -f $fm_exitcode_file || -f $fm_stdout_file || -f $fm_error_file ]] +} + +# cancel currently active job (if any) by writing cancel action JSON (file_manager deletes $fm_job_json_file) +cancel_action() { + echo '{"action":99}' > "$fm_job_json_file" + sleep 2 + if is_action_in_progress; then + echo "Error: could not cancel file_manager (job files still exist)!" 1>&2 + fail=$((fail + 1)) + return 2 + fi +} + +# write JSON data to file_manager's action file and wait for completion +# args: action source target overwrite [format archive_name] +# returns 0 on success (exit code 0), 1 on failure or timeout +run_action() { + local action=$1 source=$2 target=$3 overwrite=$4 + local format=${5:-} archive_name=${6:-} + local json rc + + # build action JSON + # title is unused by file_manager (not read in any switch case); format/archive_name are harmless empty strings for action 17 + local title + case $action in + 16) title="Compress" ;; + 17) title="Extract" ;; + *) title="" ;; + esac + local jq_args=( + --argjson action "$action" + --arg title "$title" + --arg source "$source" + --arg target "$target" + --argjson overwrite "$overwrite" + --arg format "$format" + --arg archive_name "$archive_name" + ) + # shellcheck disable=SC2016 + if ! json=$(jq -n "${jq_args[@]}" '{"action":$action,"title":$title,"source":$source,"target":$target,"H":"","sparse":"","overwrite":$overwrite,"zfs":"","format":$format,"archive_name":$archive_name}'); then + echo "Error: jq failed" 1>&2 + exit 1 + fi + echo "JSON: $json" | sed 's/^/ /' + + # ensure file_manager is running + run_file_manager + + # abort if another job is running + if is_action_in_progress; then + echo "Error: can not start new action as file_manager is busy (job files still exist)!" 1>&2 + fail=$((fail + 1)) + return 2 + fi + + echo " run action" + + printf "" >"$fm_debug_nchan_file" + + # write JSON to job file to trigger file_manager action + echo "$json" >"$fm_job_json_file" + + # follow nchan output in foreground; break on "done":1 or "done":2 closes stdin -> tail exits via SIGPIPE + # timeout acts as safety net in case file_manager hangs + local start=$SECONDS + timeout "$job_timeout" tail -n +1 -f "$fm_debug_nchan_file" | while IFS= read -r line; do + echo " nchan: $line" + [[ $line == *'"done":1'* || $line == *'"done":2'* ]] && break + done + local rc=${PIPESTATUS[0]} + + if [[ $rc -eq 124 ]]; then + echo "Error: timeout after ${job_timeout}s" + cancel_action + return 2 + fi + + # rc -1 = FM_EXITCODE_FILE not written for this op type (multi-file extract) => N/A, not failure + # rc 0 = explicit success + # rc >0 = explicit failure + rc=-1 + if [[ -f $fm_exitcode_file ]]; then + rc=$(cat "$fm_exitcode_file") + elif [[ -f $fm_exitcode_file.debug ]]; then + rc=$(cat "$fm_exitcode_file.debug") + fi + # treat non-empty stderr as failure + local stderr + stderr=$(cat "$fm_error_file" 2>/dev/null) + [[ ! $stderr ]] && stderr=$(cat "$fm_error_file.debug" 2>/dev/null) + if [[ $stderr ]]; then + echo " stderr: $stderr" + rc=99 + fi + # treat any nchan publish with a non-empty .error as failure + local nchan_err + nchan_err=$(jq -r 'select((.error // "") != "") | .error' "$fm_debug_nchan_file" | head -1) + if [[ $nchan_err ]]; then + echo " nchan error: $nchan_err" + rc=99 + fi + echo " exit code: $rc, waited: $((SECONDS - start))s" + [[ $rc -eq 0 || $rc -eq -1 ]] +} + +# returns 0 if test section should run: no filter set, or any keyword matches a filter arg +# numeric keywords use word-boundary matching to avoid partial matches (e.g. 1 != 16) +should_run() { + [[ ${#script_args[@]} -le 1 ]] && return 0 + # normalize filter: replace any non-keyword chars (not alphanumeric, dot, dash) with spaces + # allows both space- and comma-separated filters e.g. "arc,search,mv" or "arc search mv" + local filter + filter=$(printf '%s ' "${script_args[@]:1}" | tr -cs '[:alnum:].-' ' ') + local keyword + for keyword in "$@"; do + [[ $filter =~ (^|[[:space:]])$keyword([[:space:]]|$) ]] && return 0 + done + return 1 +} + +# Generic per-line nchan assertion: empty array publishes must be suppressed by file_manager +# args: line +# returns 1 if line should be skipped (empty publish), 0 if line should be processed +check_nchan_line() { + local line=$1 + if [[ $line == '[]' || $line == '{}' ]]; then + echo " [FAIL] nchan: received empty publish '$line' - should be suppressed" + fail=$((fail + 1)) + return 1 + fi + return 0 +} + +# test compress action and overwrite behavior +# args: format output_file input_file [overwrite_test] +test_compress() { + local format=$1 output_file=$2 input_file=$3 + local overwrite_test=${4:-} + local rc sz cond + local archive_name target + + should_run compress arc cmp 16 || return 0 + + echo -e "\n=== compress $input_file in $output_file ${overwrite_test:+with overwrite test} ===" + + archive_name=$(basename -- "$output_file") + target=$(dirname -- "$output_file") + + # cleanup any leftovers from previous runs + if [[ -f "$output_file" ]]; then + rm -f "$output_file" || { echo "Error: failed to remove $output_file" 1>&2; exit 1; } + echo " removed leftover archive $output_file" + fi + + run_action 16 "$input_file" "$target" 1 "$format" "$archive_name" + rc=$? + # check nchan output: no empty publishes, empty text at most once, % non-decreasing, text[0] = "Compressing..." + local empty_text_count=0 last_pct=-1 text0 text1 err_msg pct text_len speed_unit + while IFS= read -r line; do + [[ $line ]] || continue + check_nchan_line "$line" || continue + text0=$(echo "$line" | jq -r '.status.text[0] // empty') + text1=$(echo "$line" | jq -r '.status.text[1] // empty') + err_msg=$(echo "$line" | jq -r '.error // empty') + [[ $err_msg ]] && echo " nchan error: $err_msg" + if [[ ! $text0 ]]; then + text_len=$(echo "$line" | jq -r '.status.text | length') + if [[ $text_len -gt 0 ]]; then + echo " [FAIL] nchan compress $format: non-empty text array but text[0] missing" + fail=$((fail + 1)) + else + empty_text_count=$(( empty_text_count + 1 )) + fi + continue + fi + [[ $text0 == "Done" ]] && continue + if [[ $text1 =~ Completed:\ ([0-9]+)% ]]; then + pct=${BASH_REMATCH[1]} + if [[ $pct -lt $last_pct ]]; then + echo " [FAIL] nchan compress $format: % decreased (${last_pct}% -> ${pct}%)" + fail=$((fail + 1)) + fi + last_pct=$pct + if [[ $text0 != "Compressing"* ]]; then + echo " [FAIL] nchan compress $format: text[0] does not start with 'Compressing': '$text0'" + fail=$((fail + 1)) + fi + fi + if [[ $text1 =~ Speed:\ ([0-9.]+)(B|KB|MB|GB|TB)/s ]]; then + speed_unit=${BASH_REMATCH[2]} + if [[ $speed_unit == B || $speed_unit == KB ]]; then + echo " [FAIL] nchan compress $format: speed below 1 MB/s: ${BASH_REMATCH[1]}${speed_unit}/s" + fail=$((fail + 1)) + fi + fi + done <"$fm_debug_nchan_file" + cond=1; [[ $empty_text_count -le 1 ]] && cond=0 + check "nchan compress $format: empty status must appear at most once ($empty_text_count)" $cond + check "compress $format: action run must succeed" "$rc" + + rc=1 && [[ -f $output_file ]] && rc=0 + check "compress $format: archive creation must succeed" "$rc" + + # verify no .tmp files are left in the destination directory (cleanup logic should remove them) + rc=0 + for f in "$output_file"*.tmp; do + [[ -e $f ]] && { rc=1; break; } + done + check "compress $format: .tmp cleanup must succeed" "$rc" + + sz=$(stat -c%s "$output_file" 2>/dev/null || echo 0) + rc=1 && [[ $sz -gt 0 ]] && rc=0 + check "compress $format: archive size (${sz}B) must be greater than 0" "$rc" + + # overwrite sub-test: existing archive must be refused when overwrite=0 + [[ ! $overwrite_test ]] && return 0 + ! run_action 16 "$input_file" "$target" 0 "$format" "$archive_name" && rc=0 || rc=1 + check "compress $format overwrite=0: action run must fail" "$rc" + rc=1 && [[ $(stat -c%s "$output_file" 2>/dev/null || echo 0) -eq $sz ]] && rc=0 + check "compress $format overwrite=0: archive must be unchanged" "$rc" +} + +# test extract action and overwrite behavior +# args: format archive dest [overwrite_test] [source_dir] +test_extract() { + local format=$1 archive=$2 dest=$3 + local overwrite_test=${4:-} + local source_dir=${5:-$files_path} + + should_run extract arc 17 || return 0 + + echo -e "\n=== extract $archive in $dest ${overwrite_test:+with overwrite test} ===" + + if [[ ! -f $archive ]]; then + echo "Error: source archive $archive does not exist!" 1>&2 + return 1 + fi + + # cleanup any leftovers from previous runs + if [[ -d "$dest" ]]; then + rm -rf "${dest:?}" || { echo "Error: failed to remove $dest" 1>&2; exit 1; } + echo " removed leftover $dest" + fi + mkdir "$dest" || { echo "Error: failed to create destination directory $dest" 1>&2; exit 1; } + + run_action 17 "$archive" "$dest" 0; rc=$? + + # check nchan output: no empty publishes, empty text at most once, % non-decreasing, text[0] = "Extracting..." + local empty_text_count=0 last_pct=-1 text0 text1 err_msg pct text_len speed_unit + while IFS= read -r line; do + [[ $line ]] || continue + check_nchan_line "$line" || continue + text0=$(echo "$line" | jq -r '.status.text[0] // empty') + text1=$(echo "$line" | jq -r '.status.text[1] // empty') + err_msg=$(echo "$line" | jq -r '.error // empty') + [[ $err_msg ]] && echo " nchan error: $err_msg" + if [[ ! $text0 ]]; then + text_len=$(echo "$line" | jq -r '.status.text | length') + if [[ $text_len -gt 0 ]]; then + echo " [FAIL] nchan extract $format: non-empty text array but text[0] missing" + fail=$((fail + 1)) + else + empty_text_count=$((empty_text_count + 1)) + fi + continue + fi + [[ $text0 == "Done" ]] && continue + if [[ $text1 =~ Completed:\ ([0-9]+)% ]]; then + pct=${BASH_REMATCH[1]} + if [[ $pct -lt $last_pct ]]; then + echo " [FAIL] nchan extract $format: % decreased (${last_pct}% -> ${pct}%)" + fail=$((fail + 1)) + fi + last_pct=$pct + if [[ $text0 != "Extracting"* ]]; then + echo " [FAIL] nchan extract $format: text[0] does not start with 'Extracting': '$text0'" + fail=$((fail + 1)) + fi + fi + if [[ $text1 =~ Speed:\ ([0-9.]+)(B|KB|MB|GB|TB)/s ]]; then + speed_unit=${BASH_REMATCH[2]} + if [[ $speed_unit == B || $speed_unit == KB ]]; then + echo " [FAIL] nchan extract $format: speed below 1 MB/s: ${BASH_REMATCH[1]}${speed_unit}/s" + fail=$((fail + 1)) + fi + fi + done <"$fm_debug_nchan_file" + check "nchan extract $format: empty status must appear at most once ($empty_text_count)" $(( empty_text_count > 1 ? 1 : 0 )) + check "extract $format: action run must succeed" "$rc" + + local extracted_file_count extracted_file + extracted_file_count=$(find "$dest" -type f -printf '.' | wc -c) + if [[ $extracted_file_count -eq 1 ]]; then + extracted_file=$(find "$dest" -type f) + # single-file formats: file_manager does not restore mtime - compare type and size only + local single_fmt='%P %y %s\n' + diff <(find_metadata "$source_dir/$(basename -- "$extracted_file")" "$single_fmt") <(find_metadata "$extracted_file" "$single_fmt") | sed 's/^/ /' + rc=${PIPESTATUS[0]} + else + extracted_subdir=$(find "$dest" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' -quit) + if [[ $format == zip ]]; then + # zip: DOS time (2s resolution), no hardlinks + # %P first so sort is by filename - timestamps differ before sed zeroing, sort must be stable + # zero last 2 digits of epoch before decimal (100s bucket >> 2s DOS error) + local zip_fmt='%P %y %#m %T@ %s %l\n' + local zip_sed='s/[0-9][0-9]\.[0-9]+/00/' + diff <(find_metadata "$source_dir" "$zip_fmt" | sed -E "$zip_sed") <(find_metadata "$dest/$extracted_subdir" "$zip_fmt" | sed -E "$zip_sed") | sed 's/^/ /' + rc=${PIPESTATUS[0]} + else + diff <(find_metadata "$source_dir") <(find_metadata "$dest/$extracted_subdir") | sed 's/^/ /' + rc=${PIPESTATUS[0]} + fi + fi + check "extract $format: metadata must match" "$rc" + + # overwrite sub-test: reuse already-extracted destination as baseline + [[ ! $overwrite_test ]] && return 0 + local dst_file cond + dst_file=$(find "$dest" -type f -name "*.txt" -print -quit) + echo "MODIFIED" >"$dst_file" + + # single file extractions fail with overwrite=0, except of zip + if [[ $extracted_file_count -eq 1 && $format != "zip" ]]; then + ! run_action 17 "$archive" "$dest" 0 && rc=0 || rc=1 + check "extract $format overwrite=0: action must fail" "$rc" + # multi-file extractions succeed with overwrite=0 (existing files are skipped silently) + else + run_action 17 "$archive" "$dest" 0 && rc=0 || rc=1 + check "extract $format overwrite=0: action must succeed" "$rc" + fi + + # existing file must be unchanged in any case + rc=1 && [[ $(cat "$dst_file") == "MODIFIED" ]] && rc=0 + check "extract $format overwrite=0: file must be unchanged" "$rc" + + run_action 17 "$archive" "$dest" 1; rc=$? + check "extract $format overwrite=1: action run must succeed" "$rc" + rc=1 && [[ $(cat "$dst_file") != "MODIFIED" ]] && rc=0 + check "extract $format overwrite=1: file must be overwritten" "$rc" +} + +# test create folder action (action 0) +# args: label parent_dir folder_name +test_create_folder() { + local label=$1 parent=$2 name=$3 + local rc + + should_run create mk 0 || return 0 + + echo -e "\n=== create folder: $label ===" + + # cleanup any leftovers + if [[ -d "$parent/$name" ]]; then + rm -rf "${parent:?}/$name" || { echo "Error: failed to remove $parent/$name" 1>&2; exit 1; } + echo " removed leftover $parent/$name" + fi + + run_action 0 "$parent" "$name" 0; rc=$? + + # check nchan output: no empty publishes, text[0] must be "Creating..." or "Done" + local empty_text_count=0 text0 text_len + while IFS= read -r line; do + [[ $line ]] || continue + check_nchan_line "$line" || continue + text0=$(echo "$line" | jq -r '.status.text[0] // empty') + if [[ ! $text0 ]]; then + text_len=$(echo "$line" | jq -r '.status.text | length') + if [[ $text_len -gt 0 ]]; then + echo " [FAIL] nchan create folder $label: non-empty text array but text[0] missing" + fail=$((fail + 1)) + else + empty_text_count=$((empty_text_count + 1)) + fi + continue + fi + if [[ $text0 != "Creating"* && $text0 != "Done" ]]; then + echo " [FAIL] nchan create folder $label: unexpected text[0]: '$text0'" + fail=$((fail + 1)) + fi + done <"$fm_debug_nchan_file" + check "nchan create folder $label: empty status must appear at most once ($empty_text_count)" $(( empty_text_count > 1 ? 1 : 0 )) + check "create folder $label: action run must succeed" "$rc" + + rc=1 && [[ -d "$parent/$name" ]] && rc=0 + check "create folder $label: folder must exist" "$rc" +} + +# test delete action - internal implementation +# args: label path action (1=delete folder, 6=delete file) +test_delete() { + local label=$1 path=$2 action=$3 + local rc + + should_run delete del 1 6 || return 0 + + if [[ $action != 1 && $action != 6 ]]; then + echo "Error: test_delete: invalid action $action (must be 1 or 6)" 1>&2 + fail=$((fail + 1)) + return 1 + fi + + echo -e "\n=== delete $label (action $action) ===" + + if [[ ! -e "$path" ]]; then + echo "Error: path to delete does not exist: $path" 1>&2 + fail=$((fail + 1)) + return 1 + fi + + run_action "$action" "$path" "" 0; rc=$? + + # check nchan output: no empty publishes, at most 1 empty text (initial stale-clear) + local empty_text_count=0 text0 text_len + while IFS= read -r line; do + [[ $line ]] || continue + check_nchan_line "$line" || continue + text0=$(echo "$line" | jq -r '.status.text[0] // empty') + if [[ ! $text0 ]]; then + text_len=$(echo "$line" | jq -r '.status.text | length') + if [[ $text_len -gt 0 ]]; then + echo " [FAIL] nchan delete $label: non-empty text array but text[0] missing" + fail=$((fail + 1)) + else + empty_text_count=$((empty_text_count + 1)) + fi + fi + done <"$fm_debug_nchan_file" + check "nchan delete $label: empty status must appear at most once ($empty_text_count)" $(( empty_text_count > 1 ? 1 : 0 )) + check "delete $label: action run must succeed" "$rc" + + rc=1 && [[ ! -e "$path" ]] && rc=0 + check "delete $label: path must no longer exist" "$rc" +} + +# wrappers with fixed action IDs +test_delete_file() { test_delete "$1" "$2" 6; } +test_delete_folder() { test_delete "$1" "$2" 1; } + +# test rename action - internal implementation +# args: label path new_name action (2=rename folder, 7=rename file) +test_rename() { + local label=$1 path=$2 new_name=$3 action=$4 + local rc new_path pre + + should_run rename rn 2 7 || return 0 + + if [[ $action != 2 && $action != 7 ]]; then + echo "Error: test_rename: invalid action $action (must be 2 or 7)" 1>&2 + fail=$((fail + 1)) + return 1 + fi + + echo -e "\n=== rename $label (action $action) ===" + + if [[ ! -e "$path" ]]; then + echo "Error: path to rename does not exist: $path" 1>&2 + fail=$((fail + 1)) + return 1 + fi + + new_path=$(dirname -- "$path")/$new_name + + # cleanup any leftover renamed target from previous runs + if [[ -e "$new_path" ]]; then + rm -rf "${new_path:?}" || { echo "Error: failed to remove $new_path" 1>&2; exit 1; } + echo " removed leftover $new_path" + fi + + # fingerprint before rename to verify integrity at destination (source won't exist after) + pre=$(find_metadata "$path") + + run_action "$action" "$path" "$new_name" 0; rc=$? + + # check nchan output: text[0] must be "Renaming..." or "Done" + local empty_text_count=0 text0 text_len + while IFS= read -r line; do + [[ $line ]] || continue + check_nchan_line "$line" || continue + text0=$(echo "$line" | jq -r '.status.text[0] // empty') + if [[ ! $text0 ]]; then + text_len=$(echo "$line" | jq -r '.status.text | length') + if [[ $text_len -gt 0 ]]; then + echo " [FAIL] nchan rename $label: non-empty text array but text[0] missing" + fail=$((fail + 1)) + else + empty_text_count=$((empty_text_count + 1)) + fi + continue + fi + if [[ $text0 != "Renaming"* && $text0 != "Done" ]]; then + echo " [FAIL] nchan rename $label: unexpected text[0]: '$text0'" + fail=$((fail + 1)) + fi + done <"$fm_debug_nchan_file" + check "nchan rename $label: empty status must appear at most once ($empty_text_count)" $(( empty_text_count > 1 ? 1 : 0 )) + check "rename $label: action run must succeed" "$rc" + + rc=1 && [[ ! -e "$path" ]] && rc=0 + check "rename $label: source path must no longer exist" "$rc" + + rc=1 && [[ -e "$new_path" ]] && rc=0 + check "rename $label: target path must exist" "$rc" + + diff <(echo "$pre") <(find_metadata "$new_path") | sed 's/^/ /' + rc=${PIPESTATUS[0]} + check "rename $label: fingerprint must match" "$rc" +} + +# wrappers with fixed action IDs +test_rename_file() { test_rename "$1" "$2" "$3" 7; } +test_rename_folder() { test_rename "$1" "$2" "$3" 2; } + +# test chmod action (action 12) +# args: label path mode +test_chmod() { + local label=$1 path=$2 mode=$3 + local rc + + should_run chmod 12 || return 0 + + echo -e "\n=== chmod $label ===" + + if [[ ! -e "$path" ]]; then + echo "Error: path to chmod does not exist: $path" 1>&2 + fail=$((fail + 1)) + return 1 + fi + + run_action 12 "$path" "$mode" 0; rc=$? + + # check nchan output: text[0] must be "Updating..." or "Done" + local empty_text_count=0 text0 text_len + while IFS= read -r line; do + [[ $line ]] || continue + check_nchan_line "$line" || continue + text0=$(echo "$line" | jq -r '.status.text[0] // empty') + if [[ ! $text0 ]]; then + text_len=$(echo "$line" | jq -r '.status.text | length') + if [[ $text_len -gt 0 ]]; then + echo " [FAIL] nchan chmod $label: non-empty text array but text[0] missing" + fail=$((fail + 1)) + else + empty_text_count=$((empty_text_count + 1)) + fi + continue + fi + if [[ $text0 != "Updating"* && $text0 != "Done" ]]; then + echo " [FAIL] nchan chmod $label: unexpected text[0]: '$text0'" + fail=$((fail + 1)) + fi + done <"$fm_debug_nchan_file" + check "nchan chmod $label: empty status must appear at most once ($empty_text_count)" $(( empty_text_count > 1 ? 1 : 0 )) + check "chmod $label: action run must succeed" "$rc" + + local actual_mode + actual_mode=$(stat -c '%a' "$path") + rc=1 && [[ $actual_mode == "${mode#0}" || $actual_mode == "$mode" ]] && rc=0 + check "chmod $label: mode must be $mode (got $actual_mode)" "$rc" +} + +# build an archive from source; writes to .tmp first, then renames atomically +# args: format source dest_dir archive_name +build_archive() { + local fmt=$1 source=$2 dest_dir=$3 archive_name=$4 + local output="$dest_dir/$archive_name" + local tmp="$output.tmp" + local src_dir src_base rc + src_dir=$(dirname -- "$source") + src_base=$(basename -- "$source") + echo " building $output ..." + local tar_opts=(--sparse --xattrs "--xattrs-include=*.*" --acls) + case $fmt in + tar.gz) tar -czvf "$tmp" "${tar_opts[@]}" -C "$src_dir" "$src_base" ;; + tar.bz2) tar -cjvf "$tmp" "${tar_opts[@]}" -C "$src_dir" "$src_base" ;; + tar.xz) tar -cJvf "$tmp" "${tar_opts[@]}" -C "$src_dir" "$src_base" ;; + tar.zst) tar -cvf - "${tar_opts[@]}" -C "$src_dir" "$src_base" | zstd > "$tmp" ;; + tar.lz4) tar -cvf - "${tar_opts[@]}" -C "$src_dir" "$src_base" | lz4 > "$tmp" ;; + zip) (cd "$src_dir" && LANG=en_US.UTF-8 zip -ry "$tmp" "$src_base") ;; + gz) gzip -vc "$source" > "$tmp" ;; + bz2) bzip2 -vc "$source" > "$tmp" ;; + xz) xz -vc "$source" > "$tmp" ;; + zst) zstd -f "$source" -o "$tmp" ;; + lz4) lz4 -f "$source" "$tmp" ;; + *) echo "Error: build_archive: unknown format: $fmt" 1>&2; exit 1 ;; + esac + rc=$? + [[ $rc -ne 0 ]] && { echo "Error: failed to create archive $output" 1>&2; exit 1; } + mv "$tmp" "$output" || { echo "Error: failed to move $tmp to $output" 1>&2; exit 1; } +} + +# outputs sorted find metadata for a file or directory - used for diff-based integrity checks +# fields: relative path, type, perms, mtime, size, link count, symlink target +# %P is first so sort is stable by filename regardless of timestamp differences +# pass custom fmt as second arg to override fields (e.g. omit %n when hardlinks are not preserved by zip) +# additional args from $3 onward are passed to find as extra predicates (e.g. -not "(" -type d -empty ")") +find_metadata() { + local fmt=${2:-'%P %y %#m %T@ %s %n %l\n'} + find "$1" "${@:3}" -printf "$fmt" | sort +} + +# test chown action (action 11) +# args: label path owner +test_chown() { + local label=$1 path=$2 owner=$3 + local rc + + should_run chown 11 || return 0 + + echo -e "\n=== chown $label ===" + + if [[ ! -e "$path" ]]; then + echo "Error: path to chown does not exist: $path" 1>&2 + fail=$((fail + 1)) + return 1 + fi + + run_action 11 "$path" "$owner" 0; rc=$? + + local empty_text_count=0 text0 text_len + while IFS= read -r line; do + [[ $line ]] || continue + check_nchan_line "$line" || continue + text0=$(echo "$line" | jq -r '.status.text[0] // empty') + if [[ ! $text0 ]]; then + text_len=$(echo "$line" | jq -r '.status.text | length') + if [[ $text_len -gt 0 ]]; then + echo " [FAIL] nchan chown $label: non-empty text array but text[0] missing" + fail=$((fail + 1)) + else + empty_text_count=$((empty_text_count + 1)) + fi + continue + fi + if [[ $text0 != "Updating"* && $text0 != "Done" ]]; then + echo " [FAIL] nchan chown $label: unexpected text[0]: '$text0'" + fail=$((fail + 1)) + fi + done <"$fm_debug_nchan_file" + check "nchan chown $label: empty status must appear at most once ($empty_text_count)" $(( empty_text_count > 1 ? 1 : 0 )) + check "chown $label: action run must succeed" "$rc" + + local actual_owner + actual_owner=$(stat -c '%U:%G' "$path") + rc=1 && [[ $actual_owner == "$owner" ]] && rc=0 + check "chown $label: owner must be $owner (got $actual_owner)" "$rc" +} + +# test search action (action 15) +# args: label search_dir pattern +test_search() { + local label=$1 search_dir=$2 pattern=$3 + local rc + + should_run search find 15 || return 0 + + echo -e "\n=== search $label ===" + + if [[ ! -d "$search_dir" ]]; then + echo "Error: search dir does not exist: $search_dir" 1>&2 + fail=$((fail + 1)) + return 1 + fi + + # count expected results using -print0 so filenames with newlines are counted correctly + local expected_count + expected_count=$(find "$search_dir" -iname "$pattern" -print0 | tr -cd '\0' | wc -c) + + run_action 15 "$search_dir" "$pattern" 0; rc=$? + + local empty_text_count=0 text0 done_line="" + while IFS= read -r line; do + [[ $line ]] || continue + check_nchan_line "$line" || continue + # during search: text[0] = "Searching... N" + text0=$(echo "$line" | jq -r '.status.text[0] // empty') + if [[ $text0 ]]; then + if [[ $text0 != "Searching"* ]]; then + echo " [FAIL] nchan search $label: unexpected text[0]: '$text0'" + fail=$((fail + 1)) + fi + continue + fi + # on done: status has results field (array of {location, path} objects) + if echo "$line" | jq -e '.status.results != null' >/dev/null 2>&1; then + done_line=$line + continue + fi + empty_text_count=$((empty_text_count + 1)) + done <"$fm_debug_nchan_file" + + check "nchan search $label: empty status must appear at most once ($empty_text_count)" $(( empty_text_count > 1 ? 1 : 0 )) + check "search $label: action run must succeed" "$rc" + rc=1 && [[ $done_line ]] && rc=0 + check "search $label: results must be present in nchan output" "$rc" + + if [[ $done_line ]]; then + # check result count against local find (detects split filenames caused by newlines in names) + local actual_count + actual_count=$(echo "$done_line" | jq '.status.results | length') + rc=1 && [[ $actual_count -eq $expected_count ]] && rc=0 + check "search $label: result count must be $expected_count (got $actual_count)" "$rc" + + # check all paths are under search_dir (split filenames appear without directory prefix) + # use jq for the check to avoid false failures on paths that legitimately contain newlines + local all_valid + all_valid=$(echo "$done_line" | jq --arg dir "$search_dir/" '[.status.results[].path | startswith($dir)] | all') + rc=1 && [[ $all_valid == "true" ]] && rc=0 + check "search $label: all result paths must be under $search_dir" "$rc" + if [[ $rc -ne 0 ]]; then + echo "$done_line" | jq -r --arg dir "$search_dir/" '.status.results[] | select(.path | startswith($dir) | not) | .path | @json' + fi + fi +} + +# test copy action (3=copy folder, 8=copy file) +# args: label source dest_dir action +test_copy() { + local label=$1 source=$2 dest_dir=$3 action=$4 + local rc dest_item + + should_run copy cp 3 8 || return 0 + + echo -e "\n=== copy $label (action $action) ===" + + if [[ ! -e "$source" ]]; then + echo "Error: source does not exist: $source" 1>&2 + fail=$((fail + 1)) + return 1 + fi + + dest_item="$dest_dir/$(basename -- "$source")" + + # cleanup leftover from previous run + if [[ -e "$dest_item" ]]; then + rm -rf "${dest_item:?}" || { echo "Error: failed to remove $dest_item" 1>&2; exit 1; } + echo " removed leftover $dest_item" + fi + + run_action "$action" "$source" "$dest_dir" 0; rc=$? + + # check nchan output: text[0] must be "Copying..." or "Done" + local empty_text_count=0 text0 text_len + while IFS= read -r line; do + [[ $line ]] || continue + check_nchan_line "$line" || continue + text0=$(echo "$line" | jq -r '.status.text[0] // empty') + if [[ ! $text0 ]]; then + text_len=$(echo "$line" | jq -r '.status.text | length') + if [[ $text_len -gt 0 ]]; then + echo " [FAIL] nchan copy $label: non-empty text array but text[0] missing" + fail=$((fail + 1)) + else + empty_text_count=$((empty_text_count + 1)) + fi + continue + fi + if [[ $text0 != "Copying"* && $text0 != "Done" ]]; then + echo " [FAIL] nchan copy $label: unexpected text[0]: '$text0'" + fail=$((fail + 1)) + fi + done <"$fm_debug_nchan_file" + check "nchan copy $label: empty status must appear at most once ($empty_text_count)" $(( empty_text_count > 1 ? 1 : 0 )) + check "copy $label: action run must succeed" "$rc" + + rc=1 && [[ -e "$source" ]] && rc=0 + check "copy $label: source must still exist" "$rc" + + rc=1 && [[ -e "$dest_item" ]] && rc=0 + check "copy $label: destination must exist" "$rc" + + diff <(find_metadata "$source") <(find_metadata "$dest_item") | sed 's/^/ /' + rc=${PIPESTATUS[0]} + check "copy $label: fingerprint must match" "$rc" +} + +test_copy_file() { test_copy "$1" "$2" "$3" 8; } +test_copy_folder() { test_copy "$1" "$2" "$3" 3; } + +# test move action (4=move folder, 9=move file) +# args: label source dest_dir action [force_copy_delete] +test_move() { + local label=$1 source=$2 dest_dir=$3 action=$4 + local force_copy_delete=${5:-} + local rc dest_item pre + local force_file=/var/tmp/file.manager.force-copy-delete.debug + + should_run move mv 4 9 || return 0 + + if [[ $action != 4 && $action != 9 ]]; then + echo "Error: test_move: invalid action $action (must be 4 or 9)" 1>&2 + fail=$((fail + 1)) + return 1 + fi + + local path_label=$label + [[ $force_copy_delete ]] && path_label="$label (copy-delete)" + + echo -e "\n=== move $path_label (action $action) ===" + + if [[ ! -e "$source" ]]; then + echo "Error: source does not exist: $source" 1>&2 + fail=$((fail + 1)) + return 1 + fi + + dest_item="$dest_dir/$(basename -- "$source")" + + # cleanup leftover from previous run + if [[ -e "$dest_item" ]]; then + rm -rf "${dest_item:?}" || { echo "Error: failed to remove $dest_item" 1>&2; exit 1; } + echo " removed leftover $dest_item" + fi + + # ensure destination directory exists (rsync-rename requires target to be an existing dir) + [[ ! -d "$dest_dir" ]] && mkdir -p "$dest_dir" + + # fingerprint source before move to verify integrity at destination + pre=$(find_metadata "$source") + + # force rsync copy-delete path if requested (requires FM_DEBUG_MODE active) + # FM_DEBUG_FORCE_COPY_DELETE is a PHP define() evaluated at process start, so file_manager + # must be restarted after creating the force file to pick up the new value + if [[ $force_copy_delete ]]; then + touch "$force_file" + stop_file_manager + fi + + run_action "$action" "$source" "$dest_dir" 0; rc=$? + + [[ $force_copy_delete ]] && rm -f "$force_file" + + # check nchan output: text[0] must be "Moving..." or "Done" + local empty_text_count=0 text0 text_len + while IFS= read -r line; do + [[ $line ]] || continue + check_nchan_line "$line" || continue + text0=$(echo "$line" | jq -r '.status.text[0] // empty') + if [[ ! $text0 ]]; then + text_len=$(echo "$line" | jq -r '.status.text | length') + if [[ $text_len -gt 0 ]]; then + echo " [FAIL] nchan move $label: non-empty text array but text[0] missing" + fail=$((fail + 1)) + else + empty_text_count=$((empty_text_count + 1)) + fi + continue + fi + if [[ $text0 != "Moving"* && $text0 != "Done" ]]; then + echo " [FAIL] nchan move $label: unexpected text[0]: '$text0'" + fail=$((fail + 1)) + fi + done <"$fm_debug_nchan_file" + check "nchan move $path_label: empty status must appear at most once ($empty_text_count)" $(( empty_text_count > 1 ? 1 : 0 )) + check "move $path_label: action run must succeed" "$rc" + + rc=1 && [[ ! -e "$source" ]] && rc=0 + [[ $rc -ne 0 ]] && echo " source still exists: $(ls -b -- "$source" 2>&1)" + check "move $path_label: source must no longer exist" "$rc" + + rc=1 && [[ -e "$dest_item" ]] && rc=0 + [[ $rc -ne 0 ]] && echo " dest missing: $dest_item ; dest_dir contents: $(ls -la -- "$dest_dir" 2>&1)" + check "move $path_label: destination must exist" "$rc" + + diff <(echo "$pre") <(find_metadata "$dest_item") | sed 's/^/ /' + rc=${PIPESTATUS[0]} + check "move $path_label: fingerprint must match after move" "$rc" + + # clean up debug cmd file if present + rm -f /var/tmp/fm_debug_cmd.txt +} + +test_move_file() { test_move "$1" "$2" "$3" 9 "${4:-}"; } +test_move_folder() { test_move "$1" "$2" "$3" 4 "${4:-}"; } + +# create source files; idempotent - safe to call multiple times +create_source_files() { + # create directories + [[ ! -d "$test_path" ]] && ! mkdir -m 777 "$test_path" && { echo "Error: failed to create $test_path" 1>&2; exit 1; } + [[ ! -d "$src_path" ]] && ! mkdir -m 777 "$src_path" && { echo "Error: failed to create $src_path" 1>&2; exit 1; } + [[ ! -d "$dst_path" ]] && ! mkdir -m 777 "$dst_path" && { echo "Error: failed to create $dst_path" 1>&2; exit 1; } + [[ ! -d "$files_path" ]] && ! mkdir -m 777 "$files_path" && { echo "Error: failed to create $files_path" 1>&2; exit 1; } + [[ ! -d "$arc_path" ]] && ! mkdir -m 777 "$arc_path" && { echo "Error: failed to create $arc_path" 1>&2; exit 1; } + [[ ! -d "$files_path/small_files" ]] && ! mkdir -m 777 "$files_path/small_files" && { echo "Error: failed to create $files_path/small_files" 1>&2; exit 1; } + + # random data (compressible only slightly) + [[ ! -f "$files_path/small_files/urandom10MB.bin" ]] && dd if=/dev/urandom bs=1M count=10 of="$files_path/small_files/urandom10MB.bin" + [[ ! -f "$files_path/urandom100MB.bin" ]] && dd if=/dev/urandom bs=1M count=100 of="$files_path/urandom100MB.bin" + [[ ! -f "$files_path/urandom1000MB.bin" ]] && dd if=/dev/urandom bs=1M count=1000 of="$files_path/urandom1000MB.bin" + + # zeros (highly compressible) + [[ ! -f "$files_path/small_files/zero10MB.bin" ]] && dd if=/dev/zero bs=1M count=10 of="$files_path/small_files/zero10MB.bin" + [[ ! -f "$files_path/zero100MB.bin" ]] && dd if=/dev/zero bs=1M count=100 of="$files_path/zero100MB.bin" + [[ ! -f "$files_path/zero1000MB.bin" ]] && dd if=/dev/zero bs=1M count=1000 of="$files_path/zero1000MB.bin" + + # create partially compressible file: random data with zero blocks interspersed + if [[ ! -f "$files_path/mix1000MB.bin" ]]; then + for _ in {1..50}; do + cat "$files_path/small_files/urandom10MB.bin" "$files_path/small_files/zero10MB.bin" + done >"$files_path/mix1000MB.bin" + fi + + # create huge amount of empty directories + [[ ! -d "$files_path/empty_dirs" ]] && ! mkdir "$files_path/empty_dirs" && { echo "Error: failed to create $files_path/empty_dirs" 1>&2; exit 1; } + for i in {1..1000}; do + [[ ! -d "$files_path/empty_dirs/$i" ]] && ! mkdir "$files_path/empty_dirs/$i" && { echo "Error: failed to create $files_path/empty_dirs/$i" 1>&2; exit 1; } + done + + # empty text file + [[ ! -f "$files_path/empty.txt" ]] && touch "$files_path/empty.txt" + + # tiny text file + [[ ! -f "$files_path/hello.txt" ]] && echo "hello world" >"$files_path/hello.txt" + + # tiny text file in subdirectory + [[ ! -f "$files_path/small_files/subfile.txt" ]] && echo "hello subfile" >"$files_path/small_files/subfile.txt" + + # create file with utf-8 chars in name + [[ ! -f "$files_path/utf8_файл.txt" ]] && echo "utf-8 filename" >"$files_path/utf8_файл.txt" + + # create file with newline and tabulator in name + [[ ! -f "$files_path/$'newline\ntab\tfile.txt'" ]] && echo "newline in filename" >"$files_path/$'newline\ntab\tfile.txt'" + + # create file with shell-related special chars in name + [[ ! -f "$files_path/shell.\${specific}.special&chars\$file|name.txt" ]] && echo "special chars in filename" >"$files_path/shell.\${specific}.special&chars\$file|name.txt" + + # create file with utf-8, newline and special chars in name + [[ ! -f "$files_path/$special_chars_name.txt" ]] && echo "utf-8, newline and special chars in filename" >"$files_path/$special_chars_name.txt" + + # create directory with content for delete test + [[ ! -d "$files_path/delete_dir" ]] && mkdir "$files_path/delete_dir" + [[ ! -d "$files_path/delete_dir/subdir" ]] && mkdir "$files_path/delete_dir/subdir" + [[ ! -f "$files_path/delete_dir/file.txt" ]] && echo "file in dir" >"$files_path/delete_dir/file.txt" + [[ ! -f "$files_path/delete_dir/subdir/file.txt" ]] && echo "file in subdir" >"$files_path/delete_dir/subdir/file.txt" + + # create file and directory for rename test (renamed targets are left in place; re-created by guards on next run) + [[ ! -f "$files_path/$special_chars_name-rename.txt" ]] && echo "file for rename test" >"$files_path/$special_chars_name-rename.txt" + [[ ! -d "$files_path/$special_chars_name-rename-dir" ]] && mkdir "$files_path/$special_chars_name-rename-dir" + [[ ! -f "$files_path/$special_chars_name-rename-dir/content.txt" ]] && echo "content in rename dir" >"$files_path/$special_chars_name-rename-dir/content.txt" + # create file for chmod test + [[ ! -f "$files_path/$special_chars_name-chmod.txt" ]] && echo "file for chmod test" >"$files_path/$special_chars_name-chmod.txt" + # create file for chown test + [[ ! -f "$files_path/$special_chars_name-chown.txt" ]] && echo "file for chown test" >"$files_path/$special_chars_name-chown.txt" + + # create files and directories for copy test + [[ ! -f "$files_path/$special_chars_name-copy.txt" ]] && echo "file for copy test" >"$files_path/$special_chars_name-copy.txt" + + # create files for move tests (small dedicated files for file-move tests; folder-move tests use $files_path directly) + [[ ! -f "$files_path/$special_chars_name-move-rr.txt" ]] && echo "file for move test (rsync-rename)" >"$files_path/$special_chars_name-move-rr.txt" + [[ ! -f "$files_path/$special_chars_name-move-cd.txt" ]] && echo "file for move test (copy-delete)" >"$files_path/$special_chars_name-move-cd.txt" + + # create hidden files + [[ ! -f "$files_path/.hiddenfile" ]] && echo "hidden file" >"$files_path/.hiddenfile" + [[ ! -d "$files_path/.hiddendir" ]] && mkdir "$files_path/.hiddendir" + [[ ! -f "$files_path/.hiddendir/.hiddenfile" ]] && echo "hidden file in hidden directory" >"$files_path/.hiddendir/.hiddenfile" + + # create file with .tmp extension to test that it doesn't interfere with a cleanup logic + [[ ! -f "$files_path/not_fm_related_temp_file.tmp" ]] && echo "this temporary file should not be deleted by file manager" >"$files_path/not_fm_related_temp_file.tmp" + + # create hidden file to test that it doesn't interfere with a cleanup logic + [[ ! -f "$files_path/.not_fm_related_hidden_temp_file.tmp" ]] && echo "this hidden file should not be deleted by file manager" >"$files_path/.not_fm_related_hidden_temp_file.tmp" + + # create file and hardlink it; verify both share the same inode (rsync breaks hardlinks - re-link if needed) + [[ ! -f "$files_path/hardlink1.bin" ]] && dd if=/dev/urandom bs=1M count=5 of="$files_path/hardlink1.bin" + if [[ -f "$files_path/hardlink2.bin" ]]; then + inode1=$(stat -c '%i' "$files_path/hardlink1.bin") + inode2=$(stat -c '%i' "$files_path/hardlink2.bin") + if [[ $inode1 != "$inode2" ]]; then + rm "$files_path/hardlink2.bin" + fi + fi + [[ ! -f "$files_path/hardlink2.bin" ]] && ln "$files_path/hardlink1.bin" "$files_path/hardlink2.bin" + + # create file and symlink targeting it + [[ ! -f "$files_path/symlink1.bin" ]] && dd if=/dev/urandom bs=1M count=5 of="$files_path/symlink1.bin" + [[ ! -L "$files_path/symlink2.bin" ]] && ln -s "symlink1.bin" "$files_path/symlink2.bin" + + # create broken symlink + [[ ! -L "$files_path/broken_symlink" ]] && ln -s "nonexistent_target.bin" "$files_path/broken_symlink" + + # normalize all mtimes to a fixed value so extract diffs are reproducible across runs + find "$files_path" -exec touch -h -t 202001010000 {} + + + # build archives for extract tests + for fmt in "${archive_multi_formats[@]}"; do + [[ ! -f "$arc_path/archive.$fmt" ]] && build_archive "$fmt" "$files_path" "$arc_path" "archive.$fmt" + [[ ! -f "$arc_path/archive.overwrite.$fmt" ]] && build_archive "$fmt" "$files_path/small_files" "$arc_path" "archive.overwrite.$fmt" + done + for fmt in "${archive_single_formats[@]}"; do + [[ ! -f "$arc_path/$special_chars_name.txt.$fmt" ]] && build_archive "$fmt" "$files_path/$special_chars_name.txt" "$arc_path" "$special_chars_name.txt.$fmt" + done + find "$arc_path" -exec touch -h -t 202001010000 {} + + + echo " test files have been created in $src_path" +} + +echo -e "\n=== Start file manager integration test ===" + +# verify file_manager has no active job before starting test +is_action_in_progress && { echo "Error: file_manager is busy with existing job files!" 1>&2; exit 1; } + +# enable file_manager debug mode: activates nchan debug logging + renames job files instead of deleting +touch /var/tmp/file.manager.debug +printf "" >"$fm_debug_nchan_file" + +# stop running file_manager so the next start picks up the debug trigger +stop_file_manager + +# =========================== +# create source files +# =========================== +echo " create test files..." +create_source_files + +# =========================== +# compress tests +# =========================== +for fmt in "${archive_multi_formats[@]}"; do + test_compress "$fmt" "$dst_path/archive.$fmt" "$files_path" +done +for fmt in "${archive_multi_formats[@]}"; do + test_compress "$fmt" "$dst_path/archive.overwrite.$fmt" "$files_path/small_files" 1 +done +for fmt in "${archive_single_formats[@]}"; do + test_compress "$fmt" "$dst_path/$special_chars_name.txt.$fmt" "$files_path/$special_chars_name.txt" 1 +done + +# =========================== +# extract tests +# =========================== +for fmt in "${archive_multi_formats[@]}"; do + test_extract "$fmt" "$arc_path/archive.$fmt" "$dst_path/extract_$fmt/" +done +for fmt in "${archive_multi_formats[@]}"; do + test_extract "$fmt" "$arc_path/archive.overwrite.$fmt" "$dst_path/extract_overwrite_$fmt/" 1 "$files_path/small_files" +done +for fmt in "${archive_single_formats[@]}"; do + test_extract "$fmt" "$arc_path/$special_chars_name.txt.$fmt" "$dst_path/extract_$fmt/" 1 +done + +# =========================== +# create folder tests +# =========================== +test_create_folder "special chars" "$dst_path" "$special_chars_name-dir" + +# =========================== +# delete tests +# =========================== +test_delete_file "special chars" "$files_path/$special_chars_name.txt" +test_delete_folder "with content" "$files_path/delete_dir" +create_source_files >/dev/null + +# =========================== +# rename tests +# =========================== +test_rename_file "special chars" "$files_path/$special_chars_name-rename.txt" "$special_chars_name-renamed.txt" +test_rename_folder "special chars dir" "$files_path/$special_chars_name-rename-dir" "$special_chars_name-renamed-dir" +create_source_files >/dev/null + +# =========================== +# chmod tests +# =========================== +test_chmod "special chars to 0755" "$files_path/$special_chars_name-chmod.txt" "0755" +test_chmod "special chars to 0644" "$files_path/$special_chars_name-chmod.txt" "0644" + +# =========================== +# chown tests +# =========================== +test_chown "special chars to nobody:users" "$files_path/$special_chars_name-chown.txt" "nobody:users" +test_chown "special chars to root:root" "$files_path/$special_chars_name-chown.txt" "root:root" + +# =========================== +# search tests +# =========================== +test_search "hello.txt in src" "$files_path" "hello.txt" +test_search "*.txt wildcard" "$files_path" "*.txt" +test_search "no match pattern" "$files_path" "no_such_file_xyz.bin" + +# =========================== +# copy tests +# =========================== +test_copy_file "special chars" "$files_path/$special_chars_name-copy.txt" "$dst_path" +test_copy_folder "files dir" "$files_path" "$dst_path" + +# =========================== +# move tests +# =========================== +test_move_file "special chars rsync-rename" "$files_path/$special_chars_name-move-rr.txt" "$dst_path" +test_move_file "special chars copy-delete" "$files_path/$special_chars_name-move-cd.txt" "$dst_path" 1 +test_move_folder "files dir rsync-rename" "$files_path" "$dst_path" +[[ ! -d "$files_path" && -d "$dst_path/files" ]] && mv "$dst_path/files" "$src_path/" 2>/dev/null +create_source_files >/dev/null # repair source if broken +test_move_folder "files dir copy-delete" "$files_path" "$dst_path" 1 +[[ ! -d "$files_path" && -d "$dst_path/files" ]] && mv "$dst_path/files" "$src_path/" 2>/dev/null +create_source_files >/dev/null # repair source if broken + +# =========================== +# summary +# =========================== +echo -e "\n=== summary: $pass passed, $fail failed ===" +[[ $fail -eq 0 ]] diff --git a/emhttp/plugins/dynamix/Browse.page b/emhttp/plugins/dynamix/Browse.page index f52893e8b6..ae768fb5f0 100644 --- a/emhttp/plugins/dynamix/Browse.page +++ b/emhttp/plugins/dynamix/Browse.page @@ -38,6 +38,22 @@ $isshare = $root == 'mnt' && (!$main || !$next || ($main == 'rootshare' && !$res $editor = '/boot/config/editor.cfg'; if (!file_exists($editor)) file_put_contents($editor, implode("\n",['','txt','js','php','page','plg','xml','old','bak','log','css'])); + +// Read configured docker paths for compress format auto-detection. +// Strip the /mnt// prefix so the subpath can match on any disk. +$docker_cfg = @parse_ini_file('/boot/config/docker.cfg') ?: []; +$dfm_docker_subpaths = []; +foreach (['DOCKER_APP_CONFIG_PATH', 'DOCKER_IMAGE_FILE'] as $key) { + $val = rtrim(trim($docker_cfg[$key] ?? ''), '/'); + if ($val && preg_match('#^/mnt/[^/]+/(.+)$#', $val, $m)) { + $dfm_docker_subpaths[] = $m[1]; + } +} +// Fallback to defaults when docker is not configured +if (empty($dfm_docker_subpaths)) { + $dfm_docker_subpaths = ['appdata', 'docker']; +} +$dfm_docker_subpaths_js = json_encode($dfm_docker_subpaths); ?> "> "> @@ -47,6 +63,8 @@ if (!file_exists($editor)) file_put_contents($editor, implode("\n",['','txt','js !--> + +
+_(Source)_: +: + +_(Archive format)_: +: + +_(Archive name)_: +: + + +: + + + + + + +: _(save to)_ ... + +_(Target folder)_: +: +
+ +
+_(Archive)_: +: + + +: + + + + + +: _(extract to)_ ... + +_(Target folder)_: +: +
diff --git a/emhttp/plugins/dynamix/nchan/file_manager b/emhttp/plugins/dynamix/nchan/file_manager index 97c82b97a0..f22bfa7a52 100755 --- a/emhttp/plugins/dynamix/nchan/file_manager +++ b/emhttp/plugins/dynamix/nchan/file_manager @@ -2,6 +2,7 @@ /dev/null", $new_lines); + foreach ($new_lines as $line) { + $line = trim($line); + $parts = explode(' ', $line, 2); + if (count($parts) !== 2) continue; + $elapsed = floatval($parts[0]); + $bytes = floatval($parts[1]); + if ($elapsed <= 0 || $bytes < 0) continue; + $progress_events[] = [$elapsed, $bytes]; + if (count($progress_events) > 10) array_shift($progress_events); + } + $last_line_number += count($new_lines); + + if (empty($progress_events)) return []; + + $last_event = $progress_events[count($progress_events) - 1]; + $processed_bytes = $last_event[1]; + if ($processed_bytes <= 0) return []; + + // Speed from rolling window using pv timestamps + $speed_bytes = 0; + if (count($progress_events) >= 2) { + $first = $progress_events[0]; + $last = $progress_events[count($progress_events) - 1]; + $dt = $last[0] - $first[0]; + $db = $last[1] - $first[1]; + if ($dt > 0 && $db > 0) $speed_bytes = $db / $dt; + } + $speed = $speed_bytes > 0 ? bytes_to_size($speed_bytes) . '/s' : ''; + + $progress_parts = []; + if ($total_bytes > 0) { + $percent = min(100, intval($processed_bytes / $total_bytes * 100)); + $eta = 'N/A'; + if ($speed_bytes > 0) { + $remaining = max(0.0, $total_bytes - $processed_bytes); + $eta_s = $remaining / $speed_bytes; + if ($last_eta_seconds !== null) { + $eta_s = $last_eta_seconds * 0.7 + $eta_s * 0.3; + } + $last_eta_seconds = max(0.0, $eta_s); + $eta = seconds_to_time(max(0, intval($eta_s))); + } + $progress_parts[] = _('Completed') . ': ' . $percent . '%'; + $progress_parts[] = _('Total') . ': ' . bytes_to_size($total_bytes); + if ($speed) $progress_parts[] = _('Speed') . ': ' . $speed; + $progress_parts[] = _('ETA') . ': ' . $eta; + } else { + $progress_parts[] = bytes_to_size($processed_bytes) . ' ' . _('transferred'); + if ($speed) $progress_parts[] = _('Speed') . ': ' . $speed; + } + $text[1] = implode(', ', $progress_parts); + + return $text; +} + /** * Parse rsync progress output and track transfer statistics. * @@ -205,21 +332,21 @@ function parse_rsync_progress($status, $action_label, $reset = false) { // This causes ~2% error per percent point. We average multiple measurements at different percents. if ($total_size === null || count($total_calculations) < RSYNC_TOTAL_SIZE_SAMPLES) { $percent_val = intval(str_replace('%', '', $percent)); - + // Track if this is a "running transfer" line (no xfr# info) $is_running_line = !isset($parts[4]); - + // Calculate total when we have minimum progress on a running transfer line // and the percent changed since last calculation if ($is_running_line && $percent_val >= RSYNC_MIN_PROGRESS_PERCENT && $last_calc_percent !== $percent_val) { // Convert transferred size to bytes $transferred_bytes = size_to_bytes($transferred); - + // Calculate total from transferred and percent (rsync truncates, so add 0.5% for better accuracy) $calculated_total = $transferred_bytes * 100 / ($percent_val + 0.5); $total_calculations[] = $calculated_total; $last_calc_percent = $percent_val; - + // Once we have enough measurements, average them and lock the value if (count($total_calculations) >= RSYNC_TOTAL_SIZE_SAMPLES) { $total_size = array_sum($total_calculations) / count($total_calculations); @@ -242,14 +369,14 @@ function parse_rsync_progress($status, $action_label, $reset = false) { $progress_parts[] = _('Completed') . ": " . $percent; $progress_parts[] = _('Speed') . ": " . $speed; $progress_parts[] = _('ETA') . ": " . $time; - + // Always show Total (either calculated or N/A) if ($total_size !== null) { $progress_parts[] = _('Total') . ": ~" . bytes_to_size($total_size); } else { $progress_parts[] = _('Total') . ": N/A"; } - + $text[1] = implode(", ", $progress_parts); } } @@ -299,7 +426,8 @@ function cat($file) { global $null; $results = []; $set = []; - $rows = file($file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + // use null-separated read to handle filenames with newlines + $rows = array_filter(explode("\0", file_get_contents($file) ?: ''), fn($s) => $s !== ''); if (count($rows) > 0) { natcasesort($rows); $user = array_filter($rows, function($path){return preg_match('/^\/mnt\/user0?\//', $path);}); @@ -335,7 +463,12 @@ function quoted($name) {return is_array($name) ? implode(' ',array_map('escape', function source($name) {return is_array($name) ? implode(' ',array_map('escapeshellarg', $name)) : escapeshellarg($name);} function rsync_escape_component($s) { - // escape: *, ?, [, ], \ + // escape rsync pattern special chars: *, ?, [, ], and also \ + // note: \ must be escaped too - a literal filename like "foo\*bar.txt" needs + // pattern \\\* (escaped-\ + escaped-*), so rsync matches it as a literal + // backslash followed by a literal asterisk. + // without escaping \, the pattern \* would be treated as an escaped wildcard (literal *) + // but the original backslash would be silently consumed return preg_replace('/([*?\[\]\\\\])/', '\\\\$1', $s); } @@ -362,19 +495,27 @@ if (!file_exists($empty_dir)) { // initialize $delete_empty_dirs state: null = not a move operation (yet), true = rsync copy-delete phase, false = cleanup phase (done) $delete_empty_dirs = null; +$pid = false; +$last_published_status = null; +$last_published_time = 0; // infinite loop to monitor and execute file operations // Note: exec() uses /bin/sh which is symlinked to bash in unraid and a requirement for process substitution syntax >(...) while (true) { - unset($action, $source, $target, $H, $sparse, $exist, $zfs); - if (!isset($pid)) $pid = false; + // Reset all per-iteration variables. $tar_flags/$cmd must be here to prevent stale + // values from a previous tar iteration overriding the zip code path. + unset($action, $source, $target, $H, $sparse, $overwrite, $zfs, $format, $archive_name, $target_tmp, $tar_flags, $single_compressor, $pv_compressor, $pv_single_compress, $cmd); - // read job parameters from JSON file: $action, $title, $source, $target, $H, $sparse, $exist, $zfs (set by emhttp/plugins/dynamix/include/Control.php) + // read job parameters from JSON file: $action, $title, $source, $target, $H, $sparse, $overwrite, $zfs (set by emhttp/plugins/dynamix/include/Control.php) if (file_exists($active)) { $json = file_get_contents($active); $data = json_decode($json, true); if (is_array($data)) { extract($data); + // Preserve compress-specific parameters before plugin config extract + $compress_format = $format ?? null; + $compress_archive_name = $archive_name ?? null; + if (($action ?? 0) == 16) fm_debug_log('DEBUG after extract(data): archive_name=' . ($archive_name ?? 'NULL') . ', compress_archive_name=' . ($compress_archive_name ?? 'NULL')); } elseif ($json !== false && trim($json) !== '') { // Log JSON parse failure for debugging (non-empty file that failed to parse) exec('logger -t file_manager "Warning: Failed to parse active job JSON: ' . escapeshellarg(substr($json, 0, 100)) . '"'); @@ -390,20 +531,37 @@ while (true) { if (isset($action)) { // check for language changes extract(parse_plugin_cfg('dynamix', true)); + // Restore compress parameters after plugin config extract + if (isset($compress_format)) $format = $compress_format; + if (isset($compress_archive_name)) $archive_name = $compress_archive_name; + if (($action ?? 0) == 16) fm_debug_log('DEBUG after plugin_cfg+restore: archive_name=' . ($archive_name ?? 'NULL') . ', compress_archive_name=' . ($compress_archive_name ?? 'NULL')); if ($display['locale'] != $locale_init) { $locale_init = $display['locale']; update_translation($locale_init); } $source = explode("\r", $source); + + // When no process is running yet, clear any stale nchan state from the previous operation. + // nchan buffers the last message for 30s; without this a new client would briefly see + // the previous operation's final status. + if (!$pid) { + $last_published_status = null; + $last_published_time = 0; + // Clears stale progress from the nchan buffer. Uses the same {status:{...}} envelope + // as normal replies so the JS handler processes it uniformly (empty text = no output). + publish('filemanager', json_encode(['status' => ['action' => $action, 'text' => []]])); + if (FM_DEBUG_MODE) file_put_contents(FM_DEBUG_NCHAN_FILE, json_encode(['status' => ['action' => $action, 'text' => []]]) . "\n", FILE_APPEND); + } + switch ($action) { case 0: // create folder // return status of running action if (!empty($pid)) { - $reply['status'] = json_encode([ + $reply['status'] = [ 'action' => $action, 'text' => [_('Creating').'...'] - ]); + ]; // start action } else { @@ -417,10 +575,10 @@ while (true) { // return status of running action if (!empty($pid)) { - $reply['status'] = json_encode([ + $reply['status'] = [ 'action' => $action, 'text' => [mb_strimhalf(exec("tail -1 $status"), 70, '...')] - ]); + ]; // start action } else { @@ -432,15 +590,22 @@ while (true) { // return status of running action if (!empty($pid)) { - $reply['status'] = json_encode([ + $reply['status'] = [ 'action' => $action, 'text' => [_('Renaming').'...'] - ]); + ]; // start action } else { $path = dirname($source[0]); - exec("mv -f ".quoted($source)." ".quoted("$path/$target")." 1>$null 2>$error & echo \$!", $pid); + $target_path = "$path/$target"; + + // Check if target already exists + if (file_exists($target_path)) { + $reply['error'] = _('Target already exists'); + } else { + exec("mv ".quoted($source)." ".quoted($target_path)." 1>$null 2>$error & echo \$!", $pid); + } } break; case 3: // copy folder @@ -448,10 +613,10 @@ while (true) { // return status of running action if (!empty($pid)) { - $reply['status'] = json_encode([ + $reply['status'] = [ 'action' => $action, 'text' => parse_rsync_progress($status, _('Copying')) - ]); + ]; // start action } else { @@ -459,7 +624,8 @@ while (true) { $target = validname($target, false); if ($target) { $mkpath = isdir($target) ? '--mkpath' : ''; - $cmd = "rsync -ahPIX$H $sparse $exist $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 ".quoted($source)." ".escapeshellarg($target)." > >(stdbuf -o0 tr '\\r' '\\n' >$status) 2>$error & echo \$!"; + $ignore_existing = $overwrite ? '' : '--ignore-existing'; + $cmd = "rsync -ahPIX$H $sparse $ignore_existing $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 ".quoted($source)." ".escapeshellarg($target)." > >(stdbuf -o0 tr '\\r' '\\n' >$status) 2>$error & echo \$!"; // note: rsync defaults to "inc_recurse mode" which causes creating the whole directory structure at target before transfering files. // This is a problem because those target directories get temporary permissions of "root:root" and "chmod 0700", which // breaks access for all unraid users through /mnt/user/sharename. We avoid this behavior by using "--no-inc-recursive". @@ -484,23 +650,25 @@ while (true) { // cleanup empty directories: simple status if ($delete_empty_dirs === false) { - $reply['status'] = json_encode([ + $reply['status'] = [ 'action' => $action, 'text' => [mb_strimhalf(exec("tail -1 $status"), 70, '...')] - ]); + ]; // moving: progress } else { - $reply['status'] = json_encode([ + $reply['status'] = [ 'action' => $action, 'text' => parse_rsync_progress($status, _('Moving')) - ]); + ]; } // start action } else { parse_rsync_progress(null, null, true); // Reset static variables + fm_debug_log('move/start: action=' . $action . ' target_raw=' . addcslashes($target ?? 'NULL', "\0..\37\\")); $target = validname($target, false); + fm_debug_log('move/start: target_valid=' . addcslashes($target ?: 'EMPTY', "\0..\37\\")); if ($target) { $mkpath = isdir($target) ? '--mkpath' : ''; @@ -555,7 +723,7 @@ while (true) { } // selected source files and directories must not exist on target when "Overwrite existing files" is not set - if (!empty($exist)) { // would add "--ignore-existing" to rsync + if (!$overwrite) { // skip overwrite check when overwrite is enabled $target_item = rtrim($target, '/') . '/' . basename($valid_source_path); if (file_exists($target_item)) { $use_rsync_rename = false; @@ -583,6 +751,10 @@ while (true) { // - missing directories are created in --backup-dir (like using --mkpath) // - rsync prefixes the moved files with "deleting " in the output, which we strip with sed, to not confuse the user // - rsync --backup deletes empty directories instead of moving them to --backup-dir (https://github.com/RsyncProject/rsync/issues/842), so we copy empty directories first + // - we can not use "mv" here as it cannot move files/dirs into a target directory that already + // contains files with the same name + if (FM_DEBUG_FORCE_COPY_DELETE) $use_rsync_rename = false; + fm_debug_log('move/start: use_rsync_rename=' . ($use_rsync_rename ? 'true' : 'false')); if ($use_rsync_rename) { $parent_dir = dirname(validname($source[0])); $parent_dir_escaped = escapeshellarg($parent_dir); @@ -599,26 +771,42 @@ while (true) { } } $source_relative_joined = implode(' ', $source_relative); - - // Execute both rsync commands in a single bash block so they share the same PID and output stream - // First: copy only empty directories to target, then: move everything with rsync rename trick + + // Execute all steps in a single bash block so they share the same PID and output stream: + // (0) record dir mtimes, (1) copy empty dirs, (2) rsync rename trick, (3) restore dir mtimes + // note: rsync --backup-dir creates directories with mkdir (new mtime) instead of preserving the original, + // so we record dir mtimes before the move and restore them afterwards $cmd = <</dev/null + done } > >(stdbuf -o0 tr '\\r' '\\n' | sed 's/^deleting //' >$status) 2>$error & echo \$! BASH; + if (FM_DEBUG_MODE) @file_put_contents('/var/tmp/fm_debug_cmd.txt', 'rsync-rename: ' . addcslashes($cmd, "\0..\37\\") . "\n---\n", FILE_APPEND); + fm_debug_log('move/cmd(rr): ' . substr(addcslashes($cmd, "\0..\37\\"), 0, 300)); + fm_debug_log('move/cmd(rr)2: ' . substr(addcslashes($cmd, "\0..\37\\"), 300, 300)); + fm_debug_log('move/includes: ' . addcslashes($rsync_includes, "\0..\37\\")); exec($cmd, $pid); // use rsync copy-delete } else { $delete_empty_dirs = true; // cleanup needed: rsync --remove-source-files leaves empty directories - $cmd = "rsync -ahPIX$H $sparse $exist $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." > >(stdbuf -o0 tr '\\r' '\\n' >$status) 2>$error & echo \$!"; + $ignore_existing = $overwrite ? '' : '--ignore-existing'; + $cmd = "rsync -ahPIX$H $sparse $ignore_existing $mkpath --no-inc-recursive --out-format=%f --info=flist0,misc0,stats0,name1,progress2 --remove-source-files ".quoted($source)." ".escapeshellarg($target)." > >(stdbuf -o0 tr '\\r' '\\n' >$status) 2>$error & echo \$!"; // note: rsync defaults to "inc_recurse mode" which causes creating the whole directory structure at target before transfering files. // This is a problem because those target directories get temporary permissions of "root:root" and "chmod 0700", which // breaks access for all unraid users through /mnt/user/sharename. We avoid this behavior by using "--no-inc-recursive". + if (FM_DEBUG_MODE) @file_put_contents('/var/tmp/fm_debug_cmd.txt', 'copy-delete: ' . addcslashes($cmd, "\0..\37\\") . "\n---\n", FILE_APPEND); + fm_debug_log('move/cmd(cd): ' . substr(addcslashes($cmd, "\0..\37\\"), 0, 300)); exec($cmd, $pid); } @@ -632,10 +820,10 @@ while (true) { // return status of running action if (!empty($pid)) { $file_line = exec("tail -2 $status | grep -Pom1 \"^.+ of '\\K[^']+\""); - $reply['status'] = json_encode([ + $reply['status'] = [ 'action' => $action, 'text' => [_('Updating').'... '.mb_strimhalf($file_line, 70, '...')] - ]); + ]; // start action } else { @@ -647,10 +835,10 @@ while (true) { // return status of running action if (!empty($pid)) { $file_line = exec("tail -2 $status | grep -Pom1 \"^.+ of '\\K[^']+\""); - $reply['status'] = json_encode([ + $reply['status'] = [ 'action' => $action, 'text' => [_('Updating').'... '.mb_strimhalf($file_line, 70, '...')] - ]); + ]; // start action } else { @@ -661,27 +849,514 @@ while (true) { // return status of running action if (!empty($pid)) { - $count = exec("wc -l $status | grep -Pom1 '^[0-9]+'"); - $reply['status'] = json_encode([ + // count null terminators to handle filenames with newlines correctly + $count = substr_count(file_get_contents($status) ?: '', "\0"); + $reply['status'] = [ 'action' => $action, 'text' => [_('Searching').'... '.$count] - ]); + ]; // start action } else { - exec("find ".source($source)." -iname ".escapeshellarg($target)." 1>$status 2>$null & echo \$!", $pid); + // use -print0 so filenames with newlines are not split across lines + exec("find ".source($source)." -iname ".escapeshellarg($target)." -print0 1>$status 2>$null & echo \$!", $pid); } break; - case 99: // kill running background process - if (!empty($pid)) exec("kill $pid"); - delete_file($active, $pid_file, $status, $error); - unset($pid); - $delete_empty_dirs = null; + case 16: // compress + + // return status of running action + if (!empty($pid)) { + fm_debug_log('Case 16 status check: pid=' . $pid); + $format = $format ?? 'zip'; + $archive_name = basename($archive_name ?? 'archive.zip'); + $archive_path = rtrim($target, '/').'/'.$archive_name; + + fm_debug_log('Status check: format=' . $format . ', status_file=' . $status . ', exists=' . (file_exists($status) ? 'yes' : 'no')); + + $text = parse_pv_progress($status, (int)($pv_total_bytes ?? 0), $fname_file ?? null, _('Compressing')); + + if (!empty($text)) { + $reply['status'] = [ + 'action' => $action, + 'text' => $text + ]; + } + + // start action + } else { + if (FM_DEBUG_MODE) { + fm_debug_log('Case 16 start: pid is empty, starting new job'); + @unlink('/tmp/status-debug-compress.txt'); + if (file_exists($active)) copy($active, '/tmp/active-debug-compress.txt'); + fm_debug_log('Case 16 start: archive_name=' . ($archive_name ?? 'NULL') . ', compress_archive_name=' . ($compress_archive_name ?? 'NULL') . ', format=' . ($format ?? 'NULL') . ', target=' . ($target ?? 'NULL')); + } + parse_pv_progress(null, 0, null, null, true); + $dot_mb = COMPRESS_DOT_SIZE_MB; + $format = $format ?? 'zip'; + $archive_name = basename($archive_name ?? 'archive.zip'); + if ($archive_name === '' || $archive_name === '.' || $archive_name === '..') { + $reply['error'] = _('Invalid archive name'); + break; + } + $target = validname($target, false); + if (!$target) { + $reply['error'] = _('Invalid target path'); + break; + } + $archive_path = rtrim($target, '/').'/'.$archive_name; + // Set archive permissions: docker paths get root:root 600, + // all other paths (shares) get nobody:users 666 (matches Unraid share standard) + $docker_cfg = @parse_ini_file('/boot/config/docker.cfg') ?: []; + $docker_subpaths = []; + foreach (['DOCKER_APP_CONFIG_PATH', 'DOCKER_IMAGE_FILE'] as $key) { + $val = rtrim(trim($docker_cfg[$key] ?? ''), '/'); + // Strip /mnt// prefix so the subpath matches on any disk/pool + if ($val && preg_match('#^/mnt/[^/]+/(.+)$#', $val, $m)) { + $docker_subpaths[] = $m[1]; + } + } + if (empty($docker_subpaths)) { + $docker_subpaths = ['appdata', 'docker']; + } + $is_docker_path = false; + if (preg_match('#^/mnt/[^/]+/(.+)$#', $archive_path, $m)) { + $archive_subpath = $m[1]; + foreach ($docker_subpaths as $dp) { + if ($archive_subpath === $dp || str_starts_with($archive_subpath, $dp.'/')) { + $is_docker_path = true; + break; + } + } + } + $archive_chown = $is_docker_path ? 'root:root' : 'nobody:users'; + $archive_chmod = $is_docker_path ? '600' : '666'; + // Create target directory with proper permissions if user typed a new path + $target_trimmed = rtrim($target, '/'); + if (!is_dir($target_trimmed)) { + $new_root = $target_trimmed; + $check = $target_trimmed; + while ($check !== '/' && !is_dir($check)) { + $new_root = $check; + $check = dirname($check); + } + exec("mkdir -pm0".($is_docker_path ? '755' : '777')." ".quoted($target_trimmed)); + if (!$is_docker_path) exec("chown -R nobody:users ".quoted($new_root)); + } + // unique tmp path in same dir; loop to avoid collision with existing files + do { + $target_tmp = rtrim($target, '/').'/.'.$archive_name.'_'.uniqid('', true).'.tmp'; + } while (file_exists($target_tmp)); + // persist fields in active JSON so case 99 (cancel) and completion block can use them + if (file_exists($active)) { + $active_data = json_decode(file_get_contents($active), true); + // Only update if decoding succeeded; a failed decode would produce an array without + // 'source', causing mode:'read' to return JSON that makes the JS dialog lose its sources. + if (is_array($active_data)) { + $active_data['target_tmp'] = $target_tmp; + $active_data['archive_path'] = $archive_path; + $active_data['archive_chown'] = $archive_chown; + $active_data['archive_chmod'] = $archive_chmod; + file_put_contents($active, json_encode($active_data)); + } + } + + fm_debug_log('Start compress: format=' . $format . ', archive=' . $archive_path); + + // Check if target archive already exists (skip check when overwrite is allowed) + if (!$overwrite && file_exists($archive_path)) { + $reply['error'] = _('Archive already exists'); + break; + } + + // Use parent directory of first source as working directory. + // All selected files are always from the same directory in the file manager view. + $source_path = validname(rtrim($source[0], '/')); + if (!$source_path) { + $reply['error'] = _('Invalid source path'); + break; + } + $source_parent = dirname($source_path); + $source_basenames = []; + $source_valid = true; + foreach ($source as $s) { + $valid = validname(rtrim($s, '/')); + if (!$valid || dirname($valid) !== $source_parent) { + $source_valid = false; + break; + } + $source_basenames[] = basename($valid); + } + if (!$source_valid) { + $reply['error'] = _('Invalid source path'); + break; + } + + // Build compression command based on format + $escaped_basenames = implode(' ', array_map('escapeshellarg', $source_basenames)); + // Pre-escape all user-controlled paths used in heredocs so shell metacharacters + // in filenames (quotes, $, backticks) cannot escape the shell string context. + $esc_source_parent = escapeshellarg($source_parent); + $esc_target_tmp = escapeshellarg($target_tmp); + + switch ($format) { + case 'zip': + $du_sources = implode(' ', array_map(function($bn) use ($source_parent) { + return escapeshellarg($source_parent . '/' . $bn); + }, $source_basenames)); + $du_out = []; + exec("du -sb $du_sources 2>/dev/null | awk '{s+=\$1}END{print s+0}'", $du_out); + $pv_total_bytes = (int)($du_out[0] ?? 0); + if (file_exists($active)) { + $active_data = json_decode(file_get_contents($active), true); + if (is_array($active_data)) { + $active_data['fname_file'] = FM_FNAME_FILE; + $active_data['pv_total_bytes'] = $pv_total_bytes; + file_put_contents($active, json_encode($active_data)); + } + } + $zip_args = [ + '-r', // recursion into directories + '-y', // store symlinks as symlinks + '-du', // displays file sizes + '-dd', // display dot per dot size + '-ds', $dot_mb . 'm', // chunk size per dot + $esc_target_tmp, + '--', + $escaped_basenames, + ]; + $pv_zip_args = [ + 'zip', + $pv_total_bytes, + $dot_mb, + FM_FNAME_FILE, + ]; + // pv-zip reads zip stdout, writes FM_FNAME_FILE + emits pv-compatible elapsed/bytes to stderr + // LANG=en_US.UTF-8 required: Unraid has POSIX locale by default, causing zip + // to mangle non-ASCII filenames as #Uxxxx escape sequences in the archive. + $cmd = "{ cd $esc_source_parent || exit 1; LANG=en_US.UTF-8 zip " . implode(' ', $zip_args) . " | " . PV_ZIP_SCRIPT . " " . implode(' ', $pv_zip_args) . " 2>$status; echo \$?>" . FM_EXITCODE_FILE . "; } >/dev/null 2>$error & echo \$!"; + break; + case 'tar': + $pv_compressor = 'cat'; // no compression, pipe through as-is + break; + case 'tar.gz': + $pv_compressor = 'gzip'; + break; + case 'tar.bz2': + $pv_compressor = 'bzip2'; + break; + case 'tar.xz': + $pv_compressor = 'xz'; + break; + case 'tar.zst': + $pv_compressor = 'zstd'; + break; + case 'tar.lz4': + $pv_compressor = 'lz4'; + break; + case 'gz': + $pv_single_compress = 'gzip'; + break; + case 'xz': + $pv_single_compress = 'xz'; + break; + case 'bz2': + $pv_single_compress = 'bzip2'; + break; + case 'zst': + $pv_single_compress = 'zstd'; + break; + case 'lz4': + $pv_single_compress = 'lz4'; + break; + default: + $reply['error'] = sprintf(_('Unsupported archive format: %s'), $format); + break 2; // break out of switch($format) and switch($action) + } + + if (isset($pv_compressor)) { + $du_sources = implode(' ', array_map(function($bn) use ($source_parent) { + return escapeshellarg($source_parent . '/' . $bn); + }, $source_basenames)); + $du_out = []; + exec("du -sb $du_sources 2>/dev/null | awk '{s+=\$1}END{print s+0}'", $du_out); + $pv_total_bytes = (int)($du_out[0] ?? 0); + fm_debug_log('pv compress tar.zst: pv_total_bytes=' . $pv_total_bytes); + if (file_exists($active)) { + $active_data = json_decode(file_get_contents($active), true); + if (is_array($active_data)) { + $active_data['fname_file'] = FM_FNAME_FILE; + $active_data['pv_total_bytes'] = $pv_total_bytes; + file_put_contents($active, json_encode($active_data)); + } + } + $pv_args = [ + '--interval 1', // report progress every 1 second + '--numeric', // output machine-readable lines (elapsed bytes) + '--timer', // include elapsed time in numeric output + '--bytes', // include byte count in numeric output + "--size $pv_total_bytes", // expected total for ETA + ]; + $tar_args = [ + '--create', // create new archive + '--file=-', // write to stdout + '--no-unquote', // do not unquote filenames read from -T + '--xattrs', // preserve extended attributes + "--xattrs-include='*.*'", // include all namespaces: user.*, security.*, trusted.*, etc. + '--acls', // preserve POSIX ACLs + '--sparse', // handle sparse files efficiently (VM images, databases) + '--verbose', // write filenames to stderr (captured to fname_file) + "-- $escaped_basenames", + ]; + $cmd = "{ cd $esc_source_parent || exit 1; tar " . implode(' ', $tar_args) . " 2>" . FM_FNAME_FILE . " | pv " . implode(' ', $pv_args) . " 2>$status | $pv_compressor > $esc_target_tmp; echo \$?>" . FM_EXITCODE_FILE . "; } >/dev/null 2>$error & echo \$!"; + } + + if (isset($pv_single_compress)) { + $esc_basename = escapeshellarg($source_basenames[0]); + $source_fsize = (int)(filesize($source_path) ?: 0); + $pv_args = [ + '--interval 1', // report progress every 1 second + '--numeric', // output machine-readable lines (elapsed bytes) + '--timer', // include elapsed time in numeric output + '--bytes', // include byte count in numeric output + "--size $source_fsize", // expected total for %/ETA + ]; + $pv_total_bytes = $source_fsize; + if (file_exists($active)) { + $active_data = json_decode(file_get_contents($active), true); + if (is_array($active_data)) { + $active_data['pv_total_bytes'] = $pv_total_bytes; + file_put_contents($active, json_encode($active_data)); + } + } + $cmd = "{ cd $esc_source_parent || exit 1; pv " . implode(' ', $pv_args) . " -- $esc_basename 2>$status | $pv_single_compress > $esc_target_tmp; echo \$?>" . FM_EXITCODE_FILE . "; } >/dev/null 2>$error & echo \$!"; + } + + if (FM_DEBUG_MODE) { + fm_debug_log('Executing command (first 300 chars): ' . substr($cmd ?? '', 0, 300)); + file_put_contents('/tmp/cmd-debug-compress.txt', $cmd ?? '(null)'); + } + exec($cmd, $pid); + if (FM_DEBUG_MODE && !empty($pid[0])) { + $ps_out = []; + exec('ps -p ' . (int)$pid[0] . ' -o pid,cmd --no-headers 2>/dev/null', $ps_out); + fm_debug_log('Process cmd: ' . implode(' ', $ps_out)); + } + } + break; + case 17: // extract + + // return status of running action + if (!empty($pid)) { + $text = parse_pv_progress($status, (int)($pv_total_bytes ?? 0), $fname_file ?? null, _('Extracting')); + if (!empty($text)) { + $reply['status'] = [ + 'action' => $action, + 'text' => $text + ]; + } + + // start action + } else { + $archive = validname($source[0]); + if (!$archive) { + $reply['error'] = _('Invalid source path'); + break; + } + $dest = rtrim($target, '/'); + $dest = validname($dest, false); + if (!$dest) { + $reply['error'] = _('Invalid target path'); + break; + } + // Pre-escape user-controlled paths for safe interpolation + $esc_archive = escapeshellarg($archive); + $esc_dest = escapeshellarg($dest); + + fm_debug_log('extract start: archive=' . $archive . ', dest=' . $dest); + fm_debug_log('extract start: quoted_archive=' . quoted($archive) . ', quoted_dest=' . quoted($dest)); + + // Create dest with proper permissions if it doesn't exist. + // Docker paths get root:root 755, all other shares get nobody:users 777. + if (!is_dir($dest)) { + $docker_cfg = @parse_ini_file('/boot/config/docker.cfg') ?: []; + $docker_subpaths = []; + foreach (['DOCKER_APP_CONFIG_PATH', 'DOCKER_IMAGE_FILE'] as $key) { + $val = rtrim(trim($docker_cfg[$key] ?? ''), '/'); + if ($val && preg_match('#^/mnt/[^/]+/(.+)$#', $val, $m)) { + $docker_subpaths[] = $m[1]; + } + } + if (empty($docker_subpaths)) $docker_subpaths = ['appdata', 'docker']; + $is_docker_dest = false; + if (preg_match('#^/mnt/[^/]+/(.+)$#', $dest, $m)) { + foreach ($docker_subpaths as $dp) { + if ($m[1] === $dp || str_starts_with($m[1], $dp.'/')) { + $is_docker_dest = true; + break; + } + } + } + $new_root = $dest; + $check = $dest; + while ($check !== '/' && !is_dir($check)) { + $new_root = $check; + $check = dirname($check); + } + exec("mkdir -pm0".($is_docker_dest ? '755' : '777')." ".quoted($dest)); + if (!$is_docker_dest) exec("chown -R nobody:users ".quoted($new_root)); + } + + // Reset progress parser state for new extract operation (shares static vars with compress) + parse_pv_progress(null, 0, null, null, true); + + // Set shell command for zip + if (preg_match('/\.zip$/i', $archive)) { + $unzip_overwrite_flag = $overwrite ? '-o' : '-n'; + // -^ preserves special characters (e.g. newlines) + // note: stat-poller in the progress script requires this, too + // LANG=en_US.UTF-8 required: With POSIX locale, unzip mangles + // non-ASCII filenames as #Uxxxx escape sequences on extraction. + // Also set in the progress script's 'unzip -l' call, so both sides use the same locale. + // Total uncompressed size for progress: reflects actual bytes written to disk. + // filesize($archive) would cause ETA explosion for highly compressible files + // (e.g. all-zeros: 4GB -> 600KB -> speed ~0, ETA -> hours). + $unzip_total_out = []; + exec("LANG=en_US.UTF-8 unzip -lv " . $esc_archive . " 2>/dev/null | awk 'NF>=8 && /^[[:space:]]*[0-9]/{s+=\$1}END{print s+0}'", $unzip_total_out); + $pv_total_bytes = (int)($unzip_total_out[0] ?? 0) ?: (int)(filesize($archive) ?: 0); + if (file_exists($active)) { + $active_data = json_decode(file_get_contents($active), true); + if (is_array($active_data)) { + $active_data['fname_file'] = FM_FNAME_FILE; + $active_data['pv_total_bytes'] = $pv_total_bytes; + file_put_contents($active, json_encode($active_data)); + } + } + $pv_zip_args = [ + 'unzip', + $pv_total_bytes, + $esc_archive, + FM_FNAME_FILE, + ]; + $cmd = "{ cd $esc_dest || exit 1; LANG=en_US.UTF-8 unzip -^ $unzip_overwrite_flag $esc_archive | " . PV_ZIP_SCRIPT . " " . implode(' ', $pv_zip_args) . " 2>$status; } >/dev/null 2>$error & echo \$!"; + + // Single-file decompressors: output filename = archive name without last extension + } elseif (preg_match('/\.(gz|bz2|xz|zst|lz4)$/i', $archive, $m) && !preg_match('/\.tar\.[^.]+$/i', $archive)) { + $ext = strtolower($m[1]); + $decompressor = [ + 'gz' => 'gunzip -c', + 'bz2' => 'bunzip2 -c', + 'xz' => 'unxz -c', + 'zst' => 'unzstd -c', + 'lz4' => 'unlz4 -c', + ][$ext]; + $out_base = basename($archive, '.' . $ext); + $out_path = rtrim($dest, '/') . '/' . $out_base; + // overwrite check: if dest file exists and overwrite not set, abort + if (!$overwrite && file_exists($out_path)) { + $reply['error'] = _('File already exists'); + break; + } + // write to hidden tmp file first; rename to final name on success + $target_tmp = rtrim($dest, '/') . '/.' . $out_base . '_' . uniqid('', true) . '.tmp'; + $esc_target_tmp = escapeshellarg($target_tmp); + $archive_path = $out_path; + $archive_chown = $is_docker_dest ? 'root:root' : 'nobody:users'; + $archive_chmod = $is_docker_dest ? '600' : '666'; + $pv_total_bytes = filesize($archive) ?: 0; + if (file_exists($active)) { + $active_data = json_decode(file_get_contents($active), true); + if (is_array($active_data)) { + $active_data['target_tmp'] = $target_tmp; + $active_data['archive_path'] = $archive_path; + $active_data['archive_chown'] = $archive_chown; + $active_data['archive_chmod'] = $archive_chmod; + $active_data['pv_total_bytes'] = $pv_total_bytes; + file_put_contents($active, json_encode($active_data)); + } + } + $pv_args = [ + '--interval 1', // report progress every 1 second + '--numeric', // output machine-readable lines (elapsed bytes) + '--timer', // include elapsed time in numeric output + '--bytes', // include byte count in numeric output + "--size $pv_total_bytes", // expected total for %/ETA + '--', // end of options + $esc_archive, + ]; + $cmd = "{ cd $esc_dest || exit 1; pv " . implode(' ', $pv_args) . " 2>$status | $decompressor > $esc_target_tmp; echo \$?>" . FM_EXITCODE_FILE . "; } >/dev/null 2>$error & echo \$!"; + + // Set shell command for tar + } else { + + // Detect decompressor from archive extension + if (preg_match('/\.(tar\.gz|tgz)$/i', $archive)) { + $tar_decompress = '--use-compress-program=gzip'; + } elseif (preg_match('/\.(tar\.bz2|tbz2)$/i', $archive)) { + $tar_decompress = '--use-compress-program=bzip2'; + } elseif (preg_match('/\.(tar\.xz|txz)$/i', $archive)) { + $tar_decompress = '--use-compress-program=xz'; + } elseif (preg_match('/\.(tar\.zst)$/i', $archive)) { + $tar_decompress = '--use-compress-program=zstd'; + } elseif (preg_match('/\.(tar\.lz4)$/i', $archive)) { + $tar_decompress = '--use-compress-program=lz4'; + } else { + $tar_decompress = ''; // plain tar, no decompressor + } + + $pv_total_bytes = filesize($archive) ?: 0; + fm_debug_log('pv extract tar: pv_total_bytes=' . $pv_total_bytes . ', decompress=' . ($tar_decompress ?: 'none')); + if (file_exists($active)) { + $active_data = json_decode(file_get_contents($active), true); + if (is_array($active_data)) { + $active_data['fname_file'] = FM_FNAME_FILE; + $active_data['pv_total_bytes'] = $pv_total_bytes; + file_put_contents($active, json_encode($active_data)); + } + } + $keep_existing_files = $overwrite ? '' : '--skip-old-files --warning=no-existing-file'; + $pv_args = [ + '--interval 1', // report progress every 1 second + '--numeric', // output machine-readable lines (elapsed bytes) + '--timer', // include elapsed time in numeric output + '--bytes', // include byte count in numeric output + "--size $pv_total_bytes", // expected total for ETA + '--', // end of options + $esc_archive, + ]; + $tar_args = array_filter([ + '--extract', // extract files from archive + $tar_decompress, // decompressor (empty string for plain tar) + '--file=-', // read from stdin (pv output) + '--no-unquote', // do not unquote filenames read from -T + '--xattrs', // restore extended attributes + "--xattrs-include='*.*'", // include all namespaces: user.*, security.*, trusted.*, etc. + '--acls', // restore POSIX ACLs + '--sparse', // restore sparse files (VM images, databases) + '--numeric-owner', // restore exact uid/gid numbers, not name-based lookup + '--verbose', // write filenames to stdout (captured to fname_file) + $keep_existing_files, + ]); + $cmd = "{ cd $esc_dest || exit 1; pv " . implode(' ', $pv_args) . " 2>$status | tar " . implode(' ', $tar_args) . " >" . FM_FNAME_FILE . "; } >/dev/null 2>$error & echo \$!"; + } + + fm_debug_log('extract cmd=' . substr($cmd, 0, 200)); + exec($cmd, $pid); + fm_debug_log('extract pid=' . json_encode($pid)); + } break; - default: + case 99: // cancel + if (!empty($pid)) exec("kill $pid"); + // clean up compress tmp file if present (loaded from active JSON via extract) + if (!empty($target_tmp)) { + @unlink($target_tmp); + } + if (isset($fname_file)) @unlink($fname_file); + fm_delete_file($active, $pid_file, $status, $error, FM_EXITCODE_FILE); continue 2; } + fm_debug_log('After switch: action=' . $action . ', pid before pid_exists=' . json_encode($pid ?? null)); $pid = pid_exists($pid??0); + fm_debug_log('After pid_exists: pid=' . json_encode($pid)); // Store PID to survive file_manager restarts if ($pid !== false) { @@ -695,16 +1370,16 @@ while (true) { $pid = pid_exists($pid); } else { if ($action != 15) { - $reply['status'] = json_encode([ + $reply['status'] = [ 'action' => $action, 'text' => [_('Done')] - ]); + ]; $reply['done'] = 1; } else { - $reply['status'] = json_encode([ + $reply['status'] = [ 'action' => $action, 'results' => cat($status) - ]); + ]; $reply['done'] = 2; } if ($zfs) { @@ -715,19 +1390,94 @@ while (true) { foreach ($datasets as $dataset) if (exec("ls --indicator-style=none /mnt/$dataset|wc -l")==0) exec("zfs destroy $dataset 2>/dev/null"); } } + + // For compress/extract-single-file: move tmp to final name + set permissions, or clean up on failure + if (in_array($action, [16, 17]) && isset($target_tmp)) { + $exitcode = file_exists(FM_EXITCODE_FILE) ? (int)trim(file_get_contents(FM_EXITCODE_FILE)) : -1; + if ($exitcode === 0 && isset($archive_path, $archive_chown, $archive_chmod)) { + rename($target_tmp, $archive_path); + exec('chown ' . escapeshellarg($archive_chown) . ' ' . escapeshellarg($archive_path)); + exec('chmod ' . escapeshellarg($archive_chmod) . ' ' . escapeshellarg($archive_path)); + } else { + @unlink($target_tmp); + } + } + + // zip: restore symlink mtimes + // note: unzip does not call lutimes() for symlinks (https://sourceforge.net/p/infozip/bugs/19/) + if ($action == 17 && !empty($source[0]) && preg_match('/\.zip$/i', $source[0])) { + $zip_arc = validname($source[0]); + $zip_dst = validname(rtrim($target, '/'), false); + if ($zip_arc && $zip_dst) { + $za = new ZipArchive(); + if ($za->open($zip_arc) === true) { + $base = realpath($zip_dst); + if ($base !== false) { + for ($i = 0; $i < $za->numFiles; $i++) { + // identify symlinks via Unix external attributes (S_IFLNK = 0xA000 in high 16 bits) + if (!$za->getExternalAttributesIndex($i, $opsys, $attr)) continue; + if ($opsys !== ZipArchive::OPSYS_UNIX || ($attr >> 16 & 0xF000) !== 0xA000) continue; + $zstat = $za->statIndex($i); + if (!$zstat || empty($zstat['mtime'])) continue; + $name = rtrim($zstat['name'], '/'); + if (!$name) continue; + $dir_real = realpath(dirname($base . '/' . $name)); + if ($dir_real === false || !str_starts_with($dir_real . '/', $base . '/')) continue; + exec('touch -h -m -d @' . (int)$zstat['mtime'] . ' ' . escapeshellarg($dir_real . '/' . basename($name))); + } + } + $za->close(); + } + fm_debug_log('zip extract: symlink mtimes restored'); + } + } + + if (isset($fname_file)) @unlink($fname_file); + if (file_exists($error)) $reply['error'] = trim(file_get_contents($error)); - delete_file($active, $pid_file, $status, $error); + + // Debug: Copy status file before deletion (compress and extract) + if (FM_DEBUG_MODE && ($action == 16 || $action == 17) && file_exists($status)) { + $debug_copy = $action == 16 ? '/tmp/status-debug-compress.txt' : '/tmp/status-debug-extract.txt'; + copy($status, $debug_copy); + fm_debug_log('Status file copied to ' . $debug_copy . ' before deletion'); + } + + fm_delete_file($active, $pid_file, $status, $error, FM_EXITCODE_FILE); unset($pid); $delete_empty_dirs = null; } } } if (time() - $timer) { - // update every second + // time() has 1-second resolution: fires once per second regardless of usleep() interval. + // Signals Browse.page whether an operation is active (1) or finished (0) so it can + // trigger a directory refresh after completion. publish('filemonitor', file_exists($active) ? 1 : 0); $timer = time(); } - publish('filemanager', json_encode($reply)); - usleep(250000); + $now = time(); + // Don't publish identical status (avoids redundant nchan traffic for rsync/zip). + // But republish every 20s to keep nchan buffer alive (nchan_message_timeout in /etc/rc.d/rc.nginx is 30s). + if (isset($reply['status']) && !isset($reply['done'])) { + if ($reply['status'] === $last_published_status && ($now - $last_published_time) < 20) { + unset($reply['status']); + } else { + $last_published_status = $reply['status']; + $last_published_time = $now; + } + } elseif (!isset($reply['status']) && !isset($reply['done']) && isset($action) && $last_published_status !== null && ($now - $last_published_time) >= 20) { + // No new status but process still running: republish last to keep nchan buffer alive + $reply['status'] = $last_published_status; + $last_published_time = $now; + } + // publish new status or error, or if action just completed (done=1 or 2) + if (!empty($reply)) { + publish('filemanager', json_encode($reply)); + if (FM_DEBUG_MODE) file_put_contents(FM_DEBUG_NCHAN_FILE, json_encode($reply) . "\n", FILE_APPEND); + } + // Slow down polling while any background process is running: sub-second updates + // are pointless since the user can't react faster and pv/rsync write once per second anyway. + usleep(isset($pid) ? 1000000 : 250000); } ?> diff --git a/emhttp/plugins/dynamix/scripts/pvzip b/emhttp/plugins/dynamix/scripts/pvzip new file mode 100755 index 0000000000..e5c5829d84 --- /dev/null +++ b/emhttp/plugins/dynamix/scripts/pvzip @@ -0,0 +1,202 @@ +#!/bin/bash +# ##################################################################### # +# pvzip - zip/unzip wrapper that emits pv-compatible progress to stderr +# +# Copyright 2026 Marc Gutt, Gutt.IT +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# Usage (compress): pvzip zip [fname_file] +# Usage (extract): pvzip unzip [fname_file] +# +# Reads zip/unzip stdout from stdin, writes current filename to fname_file, +# emits "elapsed bytes\n" lines to stderr every ~1 second (same format as pv). +# The PHP parse_pv_progress() function reads these lines from the $status file. +# +# zip compress: total_bytes = du -sb of source files; dot_mb = bytes per dot +# (from zip -ds arg). bytes_done = committed file sizes + dot count +# for the file currently being compressed. +# unzip extract: total_bytes = total uncompressed size (computed from unzip -lv manifest); +# bytes_done = accumulated uncompressed file sizes + cur stat() of current file. + +mode=$1 +total_bytes=$2 +# zip: dot_mb (MB per dot); unzip: path to archive +arg3=$3 +# optional: path to file where current filename is written +fname_file=${4:-} + +if [[ ! $mode || ! $total_bytes || ! $arg3 ]]; then + printf 'Usage: pv-zip zip [fname_file]\n' >&2 + printf ' pv-zip unzip [fname_file]\n' >&2 + exit 1 +fi + +start_time=$EPOCHREALTIME +bytes_done=0 +last_emit=0 + +emit_progress() { + local now=$EPOCHREALTIME + local elapsed + elapsed=$(awk -v now="$now" -v start="$start_time" 'BEGIN{printf "%.4f", now - start}') + printf '%s %s\n' "$elapsed" "$bytes_done" >&2 + last_emit=$now +} + +maybe_emit() { + # integer-second comparison: strip fractional part, no awk needed + local now_s=${EPOCHREALTIME%.*} + local last_s=${last_emit%.*} + (( now_s > last_s )) && emit_progress +} + +if [[ $mode == zip ]]; then + # zip stdout (with -du -dd -ds Xm): + # " adding: path/to/file (1.2G) .......... (deflated 45%)\n" + # The \n only arrives when the file is fully compressed. Read char-by-char: + # - line mode: accumulate until filename+size match, then enter dot_mode + # - dot_mode: each '.' = dot_mb MB of source data; '\n' = file done + # + # bytes_done = committed (sum of fsizes of completed files) + dot progress of current file. + dot_bytes=$(( arg3 * 1024 * 1024 )) + committed=0 + dot_progress=0 + char_buf="" + dot_mode= + while IFS= read -r -n1 char || [[ $char ]]; do + if [[ ! $dot_mode ]]; then + char_buf+="$char" + if [[ $char_buf =~ (adding|updating):[[:space:]]+(.+)[[:space:]]+\(([0-9.]+[KkMmGgTt]?)\)[[:space:]] ]]; then + fname="${BASH_REMATCH[2]}" + fname="${fname%"${fname##*[! ]}"}" + src_size="${BASH_REMATCH[3]}" + fsize=$(awk -v s="$src_size" 'BEGIN{ + n=s+0; u=substr(s,length(s)) + if (u~/[Kk]/) print int(n*1024) + else if (u~/[Mm]/) print int(n*1048576) + else if (u~/[Gg]/) print int(n*1073741824) + else if (u~/[Tt]/) print int(n*1099511627776) + else print int(n) + }') + dot_progress=0 + [[ $fname_file ]] && printf '%s' "$fname" > "$fname_file" + dot_mode=1 + elif [[ ! $char ]]; then + char_buf="" + fi + else + if [[ $char == '.' ]]; then + dot_progress=$(( dot_progress + dot_bytes )) + bytes_done=$(( committed + dot_progress )) + maybe_emit + elif [[ ! $char ]]; then + # file done: commit its actual size, not the dot approximation + committed=$(( committed + fsize )) + bytes_done=$committed + dot_progress=0 + dot_mode= + char_buf="" + fi + fi + done + bytes_done=$total_bytes + emit_progress + +elif [[ $mode == unzip ]]; then + archive_path=$arg3 + + # Build manifest from unzip -lv: fname -> compressed size + uncompressed size. + # unzip -lv columns: Length Method Size Cmpr Date Time CRC-32 Name + # bytes_done uses uncompressed sizes: reflects actual bytes written to disk. + # Using compressed sizes causes ETA explosion for highly compressible files + # (e.g. all-zeros: 4GB -> 600KB, ratio 1:7000 -> speed ~0, ETA -> hours). + declare -A fname_to_uncompressed + while IFS= read -r line; do + [[ $line =~ ^(-+|Archive:) ]] && continue + read -r fsize_unc _ _ _ _ _ _ fname <<< "$line" + if [[ $fname && $fsize_unc =~ ^[0-9]+$ ]]; then + fname_to_uncompressed["$fname"]="$fsize_unc" + fi + done < <(LANG=en_US.UTF-8 unzip -lv "$archive_path" 2>/dev/null) + + # Override total_bytes with total uncompressed size + total_uncompressed=0 + for fname in "${!fname_to_uncompressed[@]}"; do + total_uncompressed=$(( total_uncompressed + fname_to_uncompressed[$fname] )) + done + [[ $total_uncompressed -gt 0 ]] && total_bytes=$total_uncompressed + + committed=0 + polling_fpath="" + polling_uncompressed=0 + + char_buf="" + already_handled= + stall_buf=__unset__ + while true; do + IFS= read -r -n 1 -t 0.1 char + rc=$? + if [[ $rc -eq 0 ]]; then + if [[ ! $char ]]; then + # \n: file write complete + if [[ $already_handled ]]; then + # commit uncompressed size of the just-finished file + committed=$(( committed + polling_uncompressed )) + bytes_done=$committed + elif [[ $char_buf =~ ^[[:space:]]+(inflating|extracting|creating):[[:space:]]+(.+) ]]; then + # fast file: \n arrived before the 100ms timeout (file written almost instantly) + candidate="${BASH_REMATCH[2]}" + candidate="${candidate%"${candidate##*[! ]}"}" + func="${fname_to_uncompressed[$candidate]:-0}" + committed=$(( committed + func )) + bytes_done=$committed + [[ $fname_file ]] && printf '%s' "$candidate" > "$fname_file" + fi + maybe_emit + char_buf="" + already_handled= + polling_fpath="" + polling_uncompressed=0 + continue + fi + char_buf+="$char" + continue + fi + # rc 1..128: EOF + [[ $rc -le 128 ]] && break + # rc > 128: 100ms timeout + if [[ $already_handled && $polling_fpath ]]; then + # stat-poll the output file: cur_size is the actual uncompressed bytes written so far + cur_size=$(stat -c%s -- "$polling_fpath" 2>/dev/null || echo 0) + bytes_done=$(( committed + cur_size )) + maybe_emit + continue + fi + if ! [[ $char_buf =~ ^[[:space:]]+(inflating|extracting|creating):[[:space:]]+(.+) ]]; then + [[ $char_buf == "$stall_buf" ]] && break + stall_buf=$char_buf + continue + fi + stall_buf=__unset__ + candidate="${BASH_REMATCH[2]}" + candidate="${candidate%"${candidate##*[! ]}"}" + polling_fpath="$candidate" + polling_uncompressed="${fname_to_uncompressed[$candidate]:-0}" + [[ $fname_file ]] && printf '%s' "$candidate" > "$fname_file" + already_handled=1 + # first stat poll immediately + cur_size=$(stat -c%s -- "$polling_fpath" 2>/dev/null || echo 0) + bytes_done=$(( committed + cur_size )) + maybe_emit + done + bytes_done=$total_bytes + emit_progress +fi diff --git a/emhttp/plugins/dynamix/sheets/BrowseButton.css b/emhttp/plugins/dynamix/sheets/BrowseButton.css index f9ce63c1e0..26a8e73f5d 100644 --- a/emhttp/plugins/dynamix/sheets/BrowseButton.css +++ b/emhttp/plugins/dynamix/sheets/BrowseButton.css @@ -51,7 +51,7 @@ span.dfm_speed { width: 140px; } input#dfm_sparse, -input#dfm_exist { +input#dfm_overwrite { margin-left: 0; } /* select.dfm {