apache-tuner

#!/bin/bash
# ==============================================================================
# Apache Smart Tuner v1.20.0 for cPanel / WHM & generic Apache
# Author: Ryan MacDonald <ryan@rfxn.com>
# License: GPL-3.0-or-later
#
# - Tiered by RAM & cores: LOW, LOW-MID, MID, MID-HIGH, HIGH
# - PHP-aware: conservative CPU caps on MID/HIGH
# - Tiered MaxRequestWorkers caps (prefork and threaded):
#     * LOW:       128
#     * LOW-MID:   256
#     * MID:       1024
#     * MID-HIGH:  2048
#     * HIGH:      4096
# - Floors:
#     * Prefork:   ServerLimit / MaxRequestWorkers >= 128
#     * Event/worker: MaxRequestWorkers >= 128 (threads)
# - Doubles effective concurrency on LOW and LOW-MID tiers (within caps)
# - Applies RAM, CPU, and tier-based global caps to MaxRequestWorkers
# - Shows current vs proposed values (current => proposed)
# - cPanel-aware:
#     * prefers pre_virtualhost_global.conf if present for both apply and "Current" values
#     * otherwise uses pre_main_global.conf (for apply)
#     * runs /scripts/rebuildhttpdconf after changes
# - Safe apply with backup and rollback
# - --locate: list only conf files defining MPM / concurrency settings
# - --apply: removes legacy prefork/worker/event/mpm_* IfModule blocks in target include,
#            then writes a single Smart Tuner block for the active MPM
# ==============================================================================

set -u   # keep "undefined var" safety; avoid -e/pipefail to prevent silent exits

VERSION_MAJOR=1
VERSION_MINOR=20
VERSION_PATCH=0
VERSION="${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}"
VERSION_NAME="Apache Smart Tuner v${VERSION}"

LOG_FILE="/var/log/apache-smart-tuner.log"
BUDGET_OVERRIDE=""
BUDGET_SOURCE="tier"
ERROR_LOG_LOOKBACK_HOURS=24

SPINNER_PID=""
SPINNER_MESSAGE=""

MODE="analyze"
MODE_SET=0
OUTPUT_FORMAT="text"
RELOAD_AFTER_APPLY=1
MPM_OVERRIDE=""
EXPORT_PATH=""
MPM_SOURCE="unknown"
APACHE_BINARIES=()

# Error log review defaults
APACHE_ERROR_LOG=""
ERROR_LOG_SAMPLE_LINES=0
LOG_REVIEW_STATUS="not_run"
LOG_REVIEW_MESSAGE=""
SCOREBOARD_HITS=0
SERVER_LIMIT_HITS=0
RESTART_TOTAL=0
RESTART_DAY_COUNT=0
RESTART_MAX_PER_DAY=0
LOG_SAMPLE_CONTENT=""

# Scoreboard safety defaults
SERVER_LIMIT_BUFFER_PCT="0.05"
MIN_SERVER_LIMIT_BUFFER=8

# Common error patterns to highlight during log review
declare -A COMMON_ERROR_PATTERNS=(
    [segfaults]="segfault|segmentation fault|child process .*dumped core"
    [timeouts]="script timed out|mod_fcgid: read data timeout|mod_fcgid: can't apply process slot"
    [denied]="client denied by server configuration|AH01630|access to .* denied"
)

# Friendly labels for log review output
declare -A COMMON_ERROR_LABELS=(
    [segfaults]="Process crashes/segfaults"
    [timeouts]="Slow CGI/PHP timeouts"
    [denied]="Client denials"
)

LOG_REVIEW_KEYS=(segfaults timeouts denied)

# Initialize per-run counts
declare -A COMMON_ERROR_COUNTS=()

set_mode() {
    local NEW_MODE="$1"

    if [[ "$MODE_SET" -eq 1 && "$MODE" != "$NEW_MODE" ]]; then
        echo "ERROR: Modes --$MODE and --$NEW_MODE cannot be combined."
        exit 1
    fi

    MODE="$NEW_MODE"
    MODE_SET=1
}

usage() {
    cat <<EOF
Usage: $0 [--analyze] [--locate] [--apply] [--version] [--json] [--batch] [--export <path>] [--no-reload] [--mpm <type>] [--budget <0.x>] [--log-file <path|none>]

  --analyze           (default) Print recommended Apache MPM config based on current RAM/cores
  --locate            Show which Apache config files currently define MPM-related directives
  --apply             Safely write/update recommended config into a cPanel include (or generic conf.d)
  --json              Emit analyze/apply output as JSON for automation (mutually exclusive with --locate)
  --batch             Emit analyze/apply output as a single-line bash-friendly key=value string (mutually exclusive with --locate)
  --export <path>     Write ONLY the recommended Smart Tuner block to <path> (no config edits/reloads)
  --no-reload         Skip Apache reload/restart after successful --apply (writes config only)
  --mpm <type>        Force MPM assumption (prefork|worker|event) when binary probing is unavailable
  --budget <0.x>      Override tier-based Apache RAM budget percentage (0.05-0.95)
  --log-file <path>   Write logs to the given file (default: /var/log/apache-smart-tuner.log)
                      Use "none" to disable filesystem logging
  --version           Display the current Apache Smart Tuner version and exit

Notes:
  - --apply must be run as root
  - On cPanel systems, the config will be written to the standard include locations
  - Backup files are created with .bk-YYYYmmddHHMMSS suffix
EOF
}

print_version() {
    echo "$VERSION_NAME"
}

parse_cli_arguments() {
    OPTIND=1

    local opt
    while getopts ":h-:" opt; do
        case "$opt" in
            h)
                usage
                exit 0
                ;;
            -)
                case "$OPTARG" in
                    analyze)
                        set_mode "analyze"
                        OUTPUT_FORMAT="text"
                        RELOAD_AFTER_APPLY=1
                        ;;
                    locate)
                        set_mode "locate"
                        ;;
                    apply)
                        set_mode "apply"
                        ;;
                    json)
                        OUTPUT_FORMAT="json"
                        ;;
                    batch)
                        OUTPUT_FORMAT="batch"
                        ;;
                    no-reload)
                        RELOAD_AFTER_APPLY=0
                        ;;
                    mpm)
                        local next_mpm="${!OPTIND:-}"
                        if [[ -z "$next_mpm" || "$next_mpm" == -* ]]; then
                            echo "ERROR: --mpm requires an argument (prefork|worker|event)"
                            exit 1
                        fi
                        case "$next_mpm" in
                            prefork|worker|event)
                                MPM_OVERRIDE="$next_mpm"
                                ;;
                            *)
                                echo "ERROR: Unsupported MPM type for --mpm: $next_mpm"
                                exit 1
                                ;;
                        esac
                        OPTIND=$((OPTIND + 1))
                        ;;
                    mpm=*)
                        local next_mpm="${OPTARG#*=}"
                        if [[ -z "$next_mpm" ]]; then
                            echo "ERROR: --mpm requires an argument (prefork|worker|event)"
                            exit 1
                        fi
                        case "$next_mpm" in
                            prefork|worker|event)
                                MPM_OVERRIDE="$next_mpm"
                                ;;
                            *)
                                echo "ERROR: Unsupported MPM type for --mpm: $next_mpm"
                                exit 1
                                ;;
                        esac
                        ;;
                    export)
                        local export_path="${!OPTIND:-}"
                        if [[ -z "$export_path" || "$export_path" == -* ]]; then
                            echo "ERROR: --export requires a file path argument"
                            exit 1
                        fi
                        EXPORT_PATH="$export_path"
                        OPTIND=$((OPTIND + 1))
                        ;;
                    export=*)
                        local export_path="${OPTARG#*=}"
                        if [[ -z "$export_path" ]]; then
                            echo "ERROR: --export requires a file path argument"
                            exit 1
                        fi
                        EXPORT_PATH="$export_path"
                        ;;
                    budget)
                        local budget_value="${!OPTIND:-}"
                        if [[ -z "$budget_value" || "$budget_value" == -* ]]; then
                            echo "ERROR: --budget requires a percentage in decimal form (e.g., 0.40)"
                            exit 1
                        fi
                        if ! validate_budget_pct "$budget_value"; then
                            echo "ERROR: --budget must be between 0.05 and 0.95 (e.g., 0.35)"
                            exit 1
                        fi
                        BUDGET_OVERRIDE="$budget_value"
                        OPTIND=$((OPTIND + 1))
                        ;;
                    budget=*)
                        local budget_value="${OPTARG#*=}"
                        if [[ -z "$budget_value" ]]; then
                            echo "ERROR: --budget requires a percentage in decimal form (e.g., 0.40)"
                            exit 1
                        fi
                        if ! validate_budget_pct "$budget_value"; then
                            echo "ERROR: --budget must be between 0.05 and 0.95 (e.g., 0.35)"
                            exit 1
                        fi
                        BUDGET_OVERRIDE="$budget_value"
                        ;;
                    log-file)
                        local log_path="${!OPTIND:-}"
                        if [[ -z "$log_path" || "$log_path" == -* ]]; then
                            echo "ERROR: --log-file requires a path or 'none'"
                            exit 1
                        fi
                        if [[ "$log_path" == "none" ]]; then
                            LOG_FILE=""
                        else
                            LOG_FILE="$log_path"
                        fi
                        OPTIND=$((OPTIND + 1))
                        ;;
                    log-file=*)
                        local log_path="${OPTARG#*=}"
                        if [[ -z "$log_path" ]]; then
                            echo "ERROR: --log-file requires a path or 'none'"
                            exit 1
                        fi
                        if [[ "$log_path" == "none" ]]; then
                            LOG_FILE=""
                        else
                            LOG_FILE="$log_path"
                        fi
                        ;;
                    version)
                        print_version
                        exit 0
                        ;;
                    help)
                        usage
                        exit 0
                        ;;
                    *)
                        echo "Unknown argument: --${OPTARG%%=*}"
                        usage
                        exit 1
                        ;;
                esac
                ;;
            \?)
                echo "Unknown argument: -$OPTARG"
                usage
                exit 1
                ;;
        esac
    done

    shift $((OPTIND - 1))

    if [[ $# -gt 0 ]]; then
        echo "Unknown argument: $1"
        usage
        exit 1
    fi
}

# ----------------- helpers -----------------

need_cmd() {
    local CMD="$1"
    if ! command -v "$CMD" >/dev/null 2>&1; then
        echo "ERROR: Required command '$CMD' not found in PATH."
        exit 1
    fi
}

discover_apache_binaries() {
    APACHE_BINARIES=()

    for bin in apachectl apache2ctl httpd; do
        if command -v "$bin" >/dev/null 2>&1; then
            APACHE_BINARIES+=("$bin")
        fi
    done
}

log_message() {
    local LEVEL="$1"
    local MESSAGE="$2"
    local TS
    TS=$(date +"%Y-%m-%d %H:%M:%S")
    local FORMATTED="[$TS] [$LEVEL] $MESSAGE"

    if command -v logger >/dev/null 2>&1; then
        logger -t apache-smart-tuner "$MESSAGE"
    fi

    if [[ -n "$LOG_FILE" ]]; then
        echo "$FORMATTED" >> "$LOG_FILE" 2>/dev/null || true
    fi
}

print_if_text() {
    [[ "$OUTPUT_FORMAT" == "text" ]] || return
    echo "$@"
}

start_progress_indicator() {
    [[ "$OUTPUT_FORMAT" == "json" || "$OUTPUT_FORMAT" == "batch" ]] && return
    [[ -t 1 ]] || return

    local MESSAGE="$1"
    SPINNER_MESSAGE="$MESSAGE"
    local FRAMES=('|' '/' '-' '\\')
    local i=0

    (
        while true; do
            printf "\r%s %s" "$SPINNER_MESSAGE" "${FRAMES[$((i % 4))]}"
            sleep 0.2
            ((i++))
        done
    ) &

    SPINNER_PID=$!
}

stop_progress_indicator() {
    [[ "$OUTPUT_FORMAT" == "json" || "$OUTPUT_FORMAT" == "batch" ]] && return
    [[ -t 1 ]] || return

    if [[ -n "${SPINNER_PID:-}" ]]; then
        kill "$SPINNER_PID" >/dev/null 2>&1 || true
        wait "$SPINNER_PID" 2>/dev/null || true
        SPINNER_PID=""
        printf "\r%s ... done\n" "$SPINNER_MESSAGE"
        SPINNER_MESSAGE=""
    fi
}

preflight_validate_options() {
    if [[ "$MODE" == "locate" && ( "$OUTPUT_FORMAT" == "json" || "$OUTPUT_FORMAT" == "batch" ) ]]; then
        echo "ERROR: --$OUTPUT_FORMAT is not supported with --locate mode."
        exit 1
    fi

    if [[ "$MODE" == "locate" && -n "$EXPORT_PATH" ]]; then
        echo "ERROR: --export cannot be combined with --locate mode."
        exit 1
    fi
}

validate_budget_pct() {
    local PCT="$1"

    if [[ ! "$PCT" =~ ^0\.[0-9]+$ ]]; then
        return 1
    fi

    awk "BEGIN { exit !($PCT >= 0.05 && $PCT <= 0.95) }"
}

round_down_to_multiple_of_8() {
    local VALUE=$1
    local MULTIPLE=8

    if [[ -z "$VALUE" || "$VALUE" -le 0 ]]; then
        echo 1
        return
    fi

    if [[ "$VALUE" -eq 1 || "$VALUE" -eq 2 ]]; then
        echo "$VALUE"
        return
    fi

    local REMAINDER=$(( VALUE % MULTIPLE ))
    local NEW_VALUE=$(( VALUE - REMAINDER ))

    if [[ "$NEW_VALUE" -lt 1 ]]; then
        echo "$MULTIPLE"
    else
        echo "$NEW_VALUE"
    fi
}

round_up_to_multiple_of_8() {
    local VALUE=$1
    local MULTIPLE=8

    if [[ -z "$VALUE" || "$VALUE" -le 0 ]]; then
        echo 0
        return
    fi

    local REMAINDER=$(( VALUE % MULTIPLE ))
    if [[ "$REMAINDER" -eq 0 ]]; then
        echo "$VALUE"
    else
        echo $(( VALUE + MULTIPLE - REMAINDER ))
    fi
}

calculate_server_limit_with_buffer() {
    local BASE_VALUE=$1

    local RAW_BUFFER
    RAW_BUFFER=$(awk -v target="$BASE_VALUE" -v pct="$SERVER_LIMIT_BUFFER_PCT" 'BEGIN { printf "%.0f", (target * pct) }')

    if [[ "$RAW_BUFFER" -lt "$MIN_SERVER_LIMIT_BUFFER" ]]; then
        RAW_BUFFER=$MIN_SERVER_LIMIT_BUFFER
    fi

    local BUFFER=$(round_up_to_multiple_of_8 "$RAW_BUFFER")
    echo $(( BASE_VALUE + BUFFER ))
}

# ----------------- Apache layout / env detection -----------------

detect_apache_layout() {
    APACHE_ROOT=""
    HTTPD_CONF=""
    INCLUDES_DIR=""
    PREFORK_INCLUDE=""

    if [[ -f /etc/apache2/conf/httpd.conf ]]; then
        APACHE_ROOT="/etc/apache2"
        HTTPD_CONF="/etc/apache2/conf/httpd.conf"
        INCLUDES_DIR="/etc/apache2/conf.d/includes"
        PREFORK_INCLUDE="$APACHE_ROOT/conf.d/includes/pre_virtualhost_global.conf"
        [[ ! -f "$PREFORK_INCLUDE" && -f "$APACHE_ROOT/conf/includes/pre_virtualhost_global.conf" ]] && PREFORK_INCLUDE="$APACHE_ROOT/conf/includes/pre_virtualhost_global.conf"
    elif [[ -f /usr/local/apache/conf/httpd.conf ]]; then
        APACHE_ROOT="/usr/local/apache"
        HTTPD_CONF="/usr/local/apache/conf/httpd.conf"
        INCLUDES_DIR="/usr/local/apache/conf/includes"
        PREFORK_INCLUDE="$APACHE_ROOT/conf/includes/pre_virtualhost_global.conf"
    elif [[ -f /etc/httpd/conf/httpd.conf ]]; then
        APACHE_ROOT="/etc/httpd"
        HTTPD_CONF="/etc/httpd/conf/httpd.conf"
        INCLUDES_DIR="/etc/httpd/conf.d"
        PREFORK_INCLUDE="$APACHE_ROOT/conf.d/pre_virtualhost_global.conf"
    fi

    if [[ -z "$APACHE_ROOT" || -z "$HTTPD_CONF" ]]; then
        echo "ERROR: Could not detect Apache config layout."
        exit 1
    fi
}

detect_cpanel() {
    if [[ -d /usr/local/cpanel || -d /etc/cpanel ]]; then
        IS_CPANEL=1
    else
        IS_CPANEL=0
    fi
}

detect_mpm() {
    if [[ -n "$MPM_OVERRIDE" ]]; then
        MPM_TYPE="$MPM_OVERRIDE"
        MPM_SOURCE="override"
        return
    fi

    local VOUT=""
    [[ ${#APACHE_BINARIES[@]} -eq 0 ]] && discover_apache_binaries

    for bin in "${APACHE_BINARIES[@]}"; do
        VOUT=$("$bin" -V 2>/dev/null || true)
        [[ -n "$VOUT" ]] && break
    done

    RAW_MPM=$(echo "$VOUT" | grep -i "Server MPM" | awk '{print $3}')
    MPM_TYPE=${RAW_MPM,,}
    [[ -n "$MPM_TYPE" ]] && MPM_SOURCE="binary -V"

    if [[ -z "$MPM_TYPE" ]]; then
        local MOUT=""
        for bin in "${APACHE_BINARIES[@]}"; do
            MOUT=$("$bin" -M 2>/dev/null || true)
            [[ -n "$MOUT" ]] && break
        done

        if [[ -n "$MOUT" ]]; then
            RAW_MPM=$(echo "$MOUT" | awk '/mpm_(prefork|worker|event)_module/ {print $1; exit}')
            if [[ -n "$RAW_MPM" ]]; then
                MPM_TYPE=$(echo "$RAW_MPM" | sed -E 's/.*mpm_([a-z]+)_module/\1/i')
                MPM_TYPE=${MPM_TYPE,,}
                MPM_SOURCE="module-list"
            fi
        fi
    fi

    if [[ -z "$MPM_TYPE" ]]; then
        if [[ -z "${APACHE_ROOT:-}" || -z "${HTTPD_CONF:-}" ]]; then
            detect_apache_layout
        fi
        MPM_TYPE=$(infer_mpm_from_config)
        [[ -n "$MPM_TYPE" ]] && MPM_SOURCE="config"
    fi

    if [[ -z "$MPM_TYPE" ]]; then
        echo "ERROR: Unable to detect Apache MPM type."
        exit 1
    fi
    case "$MPM_TYPE" in
        prefork|worker|event) ;;
        *)
            echo "ERROR: Unsupported or unknown MPM type detected: $MPM_TYPE"
            exit 1
            ;;
    esac
}

detect_resources() {
    need_cmd free
    need_cmd nproc
    need_cmd ps
    need_cmd bc
    need_cmd awk
    need_cmd grep
    need_cmd sed
    need_cmd tail

    TOTAL_MEM=$(free -m | awk '/^Mem:/{print $2}')
    CORES=$(nproc)

    if [[ -z "$TOTAL_MEM" || -z "$CORES" ]]; then
        echo "ERROR: Unable to detect RAM or CPU core count."
        exit 1
    fi
}

detect_apache_running() {
    detect_apache_process_name

    APACHE_PROCS=$(ps -C "$APACHE_PROC_NAME" -o rss= 2>/dev/null | wc -l || echo 0)
    APACHE_RUNNING=0
    [[ "$APACHE_PROCS" -gt 0 ]] && APACHE_RUNNING=1
}

detect_configtest_cmd() {
    [[ ${#APACHE_BINARIES[@]} -eq 0 ]] && discover_apache_binaries

    local FALLBACK_HTTPD=""

    for cmd in "${APACHE_BINARIES[@]}"; do
        case "$cmd" in
            apachectl|apache2ctl)
                APACHE_CONFIGTEST_CMD="$cmd"
                return
                ;;
            httpd)
                FALLBACK_HTTPD="$cmd"
                ;;
        esac
    done

    APACHE_CONFIGTEST_CMD="$FALLBACK_HTTPD"
}

# Pin down which httpd process name to monitor before sampling memory.
detect_apache_process_name() {
    for candidate in httpd apache2; do
        if ps -C "$candidate" -o pid= >/dev/null 2>&1; then
            APACHE_PROC_NAME="$candidate"
            return
        fi
    done

    APACHE_PROC_NAME="httpd"
}

# Gather layout, platform, and MPM facts ahead of analysis or apply actions.
bootstrap_environment() {
    detect_apache_layout
    detect_cpanel
    detect_mpm
    detect_resources
    detect_apache_running
}

reset_log_review() {
    APACHE_ERROR_LOG=""
    ERROR_LOG_SAMPLE_LINES=0
    LOG_REVIEW_STATUS="not_run"
    LOG_REVIEW_MESSAGE=""
    LOG_SAMPLE_CONTENT=""
    SCOREBOARD_HITS=0
    SERVER_LIMIT_HITS=0
    RESTART_TOTAL=0
    RESTART_DAY_COUNT=0
    RESTART_MAX_PER_DAY=0

    for key in "${!COMMON_ERROR_PATTERNS[@]}"; do
        COMMON_ERROR_COUNTS[$key]=0
    done
}

extract_error_log_from_config() {
    [[ -z "$HTTPD_CONF" || ! -f "$HTTPD_CONF" ]] && return

    local raw=""
    raw=$(awk '/^[[:space:]]*ErrorLog[[:space:]]+/ {print $2; exit}' "$HTTPD_CONF")
    raw=${raw//\"/}

    [[ -z "$raw" || "$raw" == "|"* ]] && return

    if [[ "$raw" != /* ]]; then
        echo "${APACHE_ROOT%/}/$raw"
    else
        echo "$raw"
    fi
}

detect_error_log_path() {
    local candidates=()

    local cfg_log
    cfg_log=$(extract_error_log_from_config)
    [[ -n "$cfg_log" ]] && candidates+=("$cfg_log")

    [[ -n "$APACHE_ROOT" ]] && candidates+=("$APACHE_ROOT/logs/error_log" "$APACHE_ROOT/logs/error.log")
    candidates+=("/usr/local/apache/logs/error_log" "/etc/httpd/logs/error_log" "/var/log/httpd/error_log" "/var/log/apache2/error.log" "/etc/apache2/logs/error_log")

    for path in "${candidates[@]}"; do
        if [[ -f "$path" ]]; then
            APACHE_ERROR_LOG="$path"
            return
        fi
    done
}

load_error_log_sample() {
    LOG_SAMPLE_CONTENT=""
    ERROR_LOG_SAMPLE_LINES=0

    if [[ -z "$APACHE_ERROR_LOG" ]]; then
        return
    fi

    if [[ ! -r "$APACHE_ERROR_LOG" ]]; then
        LOG_REVIEW_STATUS="unreadable"
        LOG_REVIEW_MESSAGE="Apache error log is not readable ($APACHE_ERROR_LOG)"
        return
    fi

    local cutoff_epoch
    cutoff_epoch=$(date -d "${ERROR_LOG_LOOKBACK_HOURS} hours ago" +%s 2>/dev/null || true)
    if [[ -z "$cutoff_epoch" ]]; then
        cutoff_epoch=$(python - <<'PY'
import time
import sys
from datetime import datetime, timedelta

cutoff = datetime.now() - timedelta(hours=int(sys.argv[1]))
print(int(cutoff.timestamp()))
PY
        "$ERROR_LOG_LOOKBACK_HOURS")
    fi

    LOG_SAMPLE_CONTENT=$(perl -MTime::Piece -e '
        use strict;
        use warnings;

        my ($path, $cutoff) = @ARGV;
        my $keep = 0;
        my @collected;

        open my $fh, "<", $path or exit 0;

        while (my $line = <$fh>) {
            if ($line =~ /\[[A-Za-z]{3}\s+([A-Za-z]{3})\s+(\d{1,2})\s+([0-9:]{8})(?:\.[0-9]+)?\s+(\d{4})\]/) {
                my ($mon, $mday, $time, $year) = ($1, $2, $3, $4);
                my $epoch = eval { Time::Piece->strptime("$year $mon $mday $time", "%Y %b %d %T")->epoch };
                if ($@) {
                    $keep = 1;
                } else {
                    $keep = $epoch >= $cutoff ? 1 : 0;
                }
            }

            push @collected, $line if $keep;
        }

        print @collected;
    ' "$APACHE_ERROR_LOG" "$cutoff_epoch" 2>/dev/null || true)

    ERROR_LOG_SAMPLE_LINES=$(wc -l <<< "$LOG_SAMPLE_CONTENT")

    if [[ "$ERROR_LOG_SAMPLE_LINES" -eq 0 ]]; then
        LOG_REVIEW_STATUS="empty"
        LOG_REVIEW_MESSAGE="Apache error log is empty in the last ${ERROR_LOG_LOOKBACK_HOURS} hours ($APACHE_ERROR_LOG)"
    fi
}

count_log_pattern() {
    local pattern="$1"

    if [[ -z "$LOG_SAMPLE_CONTENT" ]]; then
        echo 0
        return
    fi

    echo "$LOG_SAMPLE_CONTENT" | grep -Eai "$pattern" | wc -l | awk '{print $1}'
}

analyze_restarts() {
    local log_content="$1"

    RESTART_TOTAL=0
    RESTART_DAY_COUNT=0
    RESTART_MAX_PER_DAY=0

    local restart_counts
    restart_counts=$(echo "$log_content" | awk '
        match($0,/^\[([A-Za-z]{3}) ([A-Za-z]{3}) ([ 0-9]{2}) ([0-9:]{8})(\.[0-9]+)? ([0-9]{4})\]/,a) {
            if ($0 ~ /(resuming normal operations|Graceful restart|Graceful restart requested|caught SIGTERM|caught SIGUSR1|caught SIGHUP)/) {
                day=sprintf("%s-%02d-%s", a[2], a[3], a[6]);
                counts[day]++;
                total++;
            }
        }
        END {
            for (d in counts) printf("%s %s\n", counts[d], d);
            printf("TOTAL %s\n", total);
        }
    ')

    while IFS= read -r line; do
        [[ -z "$line" ]] && continue

        if [[ "$line" =~ ^TOTAL[[:space:]]+([0-9]+) ]]; then
            RESTART_TOTAL="${BASH_REMATCH[1]}"
            continue
        fi

        if [[ "$line" =~ ^([0-9]+)[[:space:]]+(.+) ]]; then
            local count="${BASH_REMATCH[1]}"
            ((RESTART_DAY_COUNT++))
            if (( count > RESTART_MAX_PER_DAY )); then
                RESTART_MAX_PER_DAY=$count
            fi
        fi
    done <<< "$restart_counts"
}

analyze_error_log() {
    reset_log_review

    detect_error_log_path

    if [[ -z "$APACHE_ERROR_LOG" ]]; then
        LOG_REVIEW_STATUS="not_found"
        LOG_REVIEW_MESSAGE="No readable Apache error log detected."
        return
    fi

    load_error_log_sample

    if [[ "$ERROR_LOG_SAMPLE_LINES" -eq 0 ]]; then
        [[ -z "$LOG_REVIEW_MESSAGE" ]] && LOG_REVIEW_MESSAGE="Apache error log is empty ($APACHE_ERROR_LOG)"
        return
    fi

    LOG_REVIEW_STATUS="ready"
    LOG_REVIEW_MESSAGE="Analyzed the last ${ERROR_LOG_LOOKBACK_HOURS} hours (${ERROR_LOG_SAMPLE_LINES} lines)"

    SCOREBOARD_HITS=$(count_log_pattern "scoreboard is full|server reached MaxRequestWorkers setting|server reached MaxRequestWorkers|MaxRequestWorkers setting, consider raising the MaxRequestWorkers setting")
    SERVER_LIMIT_HITS=$(count_log_pattern "ServerLimit")

    for key in "${!COMMON_ERROR_PATTERNS[@]}"; do
        COMMON_ERROR_COUNTS[$key]=$(count_log_pattern "${COMMON_ERROR_PATTERNS[$key]}")
    done

    analyze_restarts "$LOG_SAMPLE_CONTENT"
}

infer_mpm_from_config() {
    local SEARCH_PATHS=()

    [[ -n "$HTTPD_CONF" && -f "$HTTPD_CONF" ]] && SEARCH_PATHS+=("$HTTPD_CONF")
    [[ -d "$APACHE_ROOT/conf" ]] && SEARCH_PATHS+=("$APACHE_ROOT/conf")
    [[ -d "$APACHE_ROOT/conf.d" ]] && SEARCH_PATHS+=("$APACHE_ROOT/conf.d")
    [[ -n "$INCLUDES_DIR" && -d "$INCLUDES_DIR" ]] && SEARCH_PATHS+=("$INCLUDES_DIR")

    if ((${#SEARCH_PATHS[@]} == 0)); then
        echo ""
        return
    fi

    local FIRST_MATCH
    FIRST_MATCH=$(grep -R -h -E 'mpm_(prefork|worker|event)_module' "${SEARCH_PATHS[@]}" 2>/dev/null | head -n1)

    if [[ -z "$FIRST_MATCH" ]]; then
        FIRST_MATCH=$(grep -R -h -E '<IfModule[[:space:]]+mpm_(prefork|worker|event)_module>' "${SEARCH_PATHS[@]}" 2>/dev/null | head -n1)
    fi

    if [[ -n "$FIRST_MATCH" ]]; then
        echo "$FIRST_MATCH" | awk '{
            for (i=1; i<=NF; i++) {
                if ($i ~ /mpm_(prefork|worker|event)_module/) {
                    sub(/.*mpm_/, "", $i)
                    sub(/_module.*/, "", $i)
                    print tolower($i)
                    break
                }
            }
        }'
        return
    fi

    echo ""
}

set_tier_params() { 
    if [[ "$TOTAL_MEM" -le 2048 || "$CORES" -le 2 ]]; then
        TIER="LOW"
        APACHE_PCT="0.30"
        PROCS_PER_CORE_CAP=64
        THREADS_PER_CORE_CAP=200
        START_SERVERS=1
        MIN_SPARE=1
        MAX_SPARE=2
        MAX_CONN_PER_CHILD=2000
        MRW_CAP_PREFORK=128
        MRW_CAP_THREADED=128
    elif [[ "$TOTAL_MEM" -le 8192 && "$CORES" -ge 2 ]]; then
        TIER="LOW-MID"
        APACHE_PCT="0.35"
        PROCS_PER_CORE_CAP=64
        THREADS_PER_CORE_CAP=220
        START_SERVERS=2
        MIN_SPARE=2
        MAX_SPARE=5
        MAX_CONN_PER_CHILD=4000
        MRW_CAP_PREFORK=256
        MRW_CAP_THREADED=256
    elif [[ "$TOTAL_MEM" -le 16384 && "$CORES" -ge 4 ]]; then
        TIER="MID"
        APACHE_PCT="0.40"
        PROCS_PER_CORE_CAP=48
        THREADS_PER_CORE_CAP=200
        START_SERVERS=3
        MIN_SPARE=3
        MAX_SPARE=8
        MAX_CONN_PER_CHILD=6000
        MRW_CAP_PREFORK=1024
        MRW_CAP_THREADED=1024
    elif [[ "$TOTAL_MEM" -le 32768 && "$CORES" -ge 8 ]]; then
        TIER="MID-HIGH"
        APACHE_PCT="0.43"
        PROCS_PER_CORE_CAP=44
        THREADS_PER_CORE_CAP=200
        START_SERVERS=4
        MIN_SPARE=4
        MAX_SPARE=10
        MAX_CONN_PER_CHILD=8000
        MRW_CAP_PREFORK=2048
        MRW_CAP_THREADED=2048
    else
        TIER="HIGH"
        APACHE_PCT="0.45"
        PROCS_PER_CORE_CAP=40
        THREADS_PER_CORE_CAP=200
        START_SERVERS=5
        MIN_SPARE=5
        MAX_SPARE=12
        MAX_CONN_PER_CHILD=10000
        MRW_CAP_PREFORK=4096
        MRW_CAP_THREADED=4096
    fi

    if [[ -n "$BUDGET_OVERRIDE" ]]; then
        APACHE_PCT="$BUDGET_OVERRIDE"
        BUDGET_SOURCE="override"
    else
        BUDGET_SOURCE="tier"
    fi
}

measure_apache_procs() {
    detect_apache_process_name

    APACHE_PROCS=$(ps -C "$APACHE_PROC_NAME" -o rss= 2>/dev/null | wc -l || echo 0)
    APACHE_RUNNING=0
    [[ "$APACHE_PROCS" -gt 0 ]] && APACHE_RUNNING=1

    AVG_PROC_MB=$(ps -C "$APACHE_PROC_NAME" -o rss= 2>/dev/null | awk '{sum+=$1; n++} END { if(n>0) printf "%.0f", sum/n/1024; else print 0 }')

    if [[ -z "$AVG_PROC_MB" || "$AVG_PROC_MB" -le 0 ]]; then
        if [[ "$MPM_TYPE" == "prefork" ]]; then
            AVG_PROC_MB=35
        else
            AVG_PROC_MB=15
        fi
    fi

    APACHE_BUDGET_MB=$(echo "$TOTAL_MEM * $APACHE_PCT" | bc | awk '{printf "%.0f", $0}')
}

apache_configtest() {
    if [[ -z "${APACHE_CONFIGTEST_CMD:-}" ]]; then
        detect_configtest_cmd
    fi

    case "$APACHE_CONFIGTEST_CMD" in
        apachectl|apache2ctl)
            "$APACHE_CONFIGTEST_CMD" configtest
            ;;
        httpd)
            httpd -t
            ;;
        *)
            echo "ERROR: No suitable Apache binary found for configtest."
            return 1
            ;;
    esac
}

json_bool() {
    if [[ "$1" -eq 0 ]]; then
        echo "false"
    else
        echo "true"
    fi
}

format_batch_value() {
    local value="$1"
    local fallback="$2"

    value=$(sanitize_value "${value:-}")
    if [[ -z "$value" ]]; then
        value="$fallback"
    fi

    echo "$value"
}

extract_tuner_block() {
    local FILE="$1"
    [[ -f "$FILE" ]] || return

    sed -n '/# BEGIN APACHE_SMART_TUNER/,/# END APACHE_SMART_TUNER/p' "$FILE"
}

# Normalize whitespace so block comparisons ignore cosmetic drift.
normalize_block_for_compare() {
    printf "%s" "$1" | sed 's/[[:space:]]*$//'
}

# Drive the main analysis path that turns runtime inputs into a config block.
run_analysis_pipeline() {
    set_tier_params
    measure_apache_procs
    get_current_values_with_precedence
    analyze_error_log
    detect_configtest_cmd
    build_recommended_block
}

build_json_output() {
    local block_json
    block_json=${RECOMMENDED_BLOCK//$'\n'/\\n}

    cat <<EOF
{
  "version": "$VERSION",
  "mode": "$MODE",
  "mpm": "$MPM_TYPE",
  "mpm_source": "$MPM_SOURCE",
  "tier": "$TIER",
  "total_ram_mb": $TOTAL_MEM,
  "cpu_cores": $CORES,
  "apache_running": $(json_bool "$APACHE_RUNNING"),
  "observed_processes": $APACHE_PROCS,
  "avg_httpd_mb": $AVG_PROC_MB,
  "apache_budget_mb": $APACHE_BUDGET_MB,
  "apache_budget_pct": $APACHE_PCT,
  "apache_budget_source": "$BUDGET_SOURCE",
  "log_file": "$(sanitize_value "${LOG_FILE:-}")",
  "export_path": "$(sanitize_value "${EXPORT_PATH:-}")",
  "recommended_block": "$block_json",
  "log_review": {
    "status": "$LOG_REVIEW_STATUS",
    "message": "$(sanitize_value "$LOG_REVIEW_MESSAGE")",
    "error_log_path": "$(sanitize_value "$APACHE_ERROR_LOG")",
    "sampled_lines": $ERROR_LOG_SAMPLE_LINES,
    "scoreboard_hits": $SCOREBOARD_HITS,
    "server_limit_hits": $SERVER_LIMIT_HITS,
    "restarts_total": $RESTART_TOTAL,
    "restart_days_observed": $RESTART_DAY_COUNT,
    "restart_max_per_day": $RESTART_MAX_PER_DAY,
    "common_errors": {
      "segfaults": ${COMMON_ERROR_COUNTS[segfaults]:-0},
      "timeouts": ${COMMON_ERROR_COUNTS[timeouts]:-0},
      "denied": ${COMMON_ERROR_COUNTS[denied]:-0}
    }
  },
  "current": {
    "ServerLimit": "$(sanitize_value "$CUR_SERVER_LIMIT")",
    "MaxRequestWorkers": "$(sanitize_value "$CUR_MAX_WORKERS")",
    "ThreadsPerChild": "$(sanitize_value "$CUR_THREADS_PER_CHILD")",
    "StartServers": "$(sanitize_value "$CUR_START_SERVERS")",
    "MinSpareThreads": "$(sanitize_value "$CUR_MIN_SPARE_THREADS")",
    "MaxSpareThreads": "$(sanitize_value "$CUR_MAX_SPARE_THREADS")",
    "MinSpareServers": "$(sanitize_value "$CUR_MIN_SPARE_SERVERS")",
    "MaxSpareServers": "$(sanitize_value "$CUR_MAX_SPARE_SERVERS")",
    "MaxConnectionsPerChild": "$(sanitize_value "$CUR_MAX_CONN_PER_CHILD")"
  },
  "recommended": {
    "ServerLimit": "$(sanitize_value "$SERVER_LIMIT")",
    "MaxRequestWorkers": "$(sanitize_value "$MAX_WORKERS")",
    "ThreadsPerChild": "$(sanitize_value "${THREADS_PER_CHILD:-}")",
    "StartServers": "$(sanitize_value "$START_SERVERS_ROUNDED")",
    "MinSpareThreads": "$(sanitize_value "${MIN_SPARE_THREADS:-}")",
    "MaxSpareThreads": "$(sanitize_value "${MAX_SPARE_THREADS:-}")",
    "MinSpareServers": "$(sanitize_value "${MIN_SPARE_SERVERS:-}")",
    "MaxSpareServers": "$(sanitize_value "${MAX_SPARE_SERVERS:-}")",
    "MaxConnectionsPerChild": "$(sanitize_value "$MAX_CONN_PER_CHILD")"
  }
}
EOF
}

build_batch_output() {
    local -a fields=()

    fields+=("version=$(sanitize_value "$VERSION")")
    fields+=("mode=$MODE")
    fields+=("mpm=$MPM_TYPE")
    fields+=("mpm_source=$(sanitize_value "$MPM_SOURCE")")
    fields+=("tier=$TIER")
    fields+=("total_ram_mb=$TOTAL_MEM")
    fields+=("cpu_cores=$CORES")
    fields+=("apache_running=$APACHE_RUNNING")
    fields+=("observed_processes=$APACHE_PROCS")
    fields+=("avg_httpd_mb=$AVG_PROC_MB")
    fields+=("apache_budget_mb=$APACHE_BUDGET_MB")
    fields+=("apache_budget_pct=$APACHE_PCT")
    fields+=("apache_budget_source=$BUDGET_SOURCE")
    fields+=("log_file=$(format_batch_value "${LOG_FILE:-}" "none")")
    fields+=("export_path=$(format_batch_value "${EXPORT_PATH:-}" "none")")
    fields+=("error_log_path=$(format_batch_value "$APACHE_ERROR_LOG" "unknown")")
    fields+=("error_log_sampled_lines=$ERROR_LOG_SAMPLE_LINES")
    fields+=("log_review_status=$LOG_REVIEW_STATUS")
    fields+=("scoreboard_hits=$SCOREBOARD_HITS")
    fields+=("server_limit_hits=$SERVER_LIMIT_HITS")
    fields+=("restarts_total=$RESTART_TOTAL")
    fields+=("restart_max_per_day=$RESTART_MAX_PER_DAY")
    fields+=("common_errors_segfaults=${COMMON_ERROR_COUNTS[segfaults]:-0}")
    fields+=("common_errors_timeouts=${COMMON_ERROR_COUNTS[timeouts]:-0}")
    fields+=("common_errors_denied=${COMMON_ERROR_COUNTS[denied]:-0}")

    fields+=("current_server_limit=$(format_batch_value "$CUR_SERVER_LIMIT" "unknown")")
    fields+=("current_max_requestworkers=$(format_batch_value "$CUR_MAX_WORKERS" "unknown")")
    fields+=("current_start_servers=$(format_batch_value "$CUR_START_SERVERS" "unknown")")
    fields+=("current_max_connections_per_child=$(format_batch_value "$CUR_MAX_CONN_PER_CHILD" "unknown")")

    if [[ "$MPM_TYPE" == "prefork" ]]; then
        fields+=("current_threads_per_child=na")
        fields+=("current_min_spare_threads=na")
        fields+=("current_max_spare_threads=na")
        fields+=("current_min_spare_servers=$(format_batch_value "$CUR_MIN_SPARE_SERVERS" "unknown")")
        fields+=("current_max_spare_servers=$(format_batch_value "$CUR_MAX_SPARE_SERVERS" "unknown")")

        fields+=("recommended_threads_per_child=na")
        fields+=("recommended_min_spare_threads=na")
        fields+=("recommended_max_spare_threads=na")
        fields+=("recommended_min_spare_servers=$(format_batch_value "$MIN_SPARE_SERVERS" "unknown")")
        fields+=("recommended_max_spare_servers=$(format_batch_value "$MAX_SPARE_SERVERS" "unknown")")
    else
        fields+=("current_threads_per_child=$(format_batch_value "$CUR_THREADS_PER_CHILD" "unknown")")
        fields+=("current_min_spare_threads=$(format_batch_value "$CUR_MIN_SPARE_THREADS" "unknown")")
        fields+=("current_max_spare_threads=$(format_batch_value "$CUR_MAX_SPARE_THREADS" "unknown")")
        fields+=("current_min_spare_servers=na")
        fields+=("current_max_spare_servers=na")

        fields+=("recommended_threads_per_child=$(format_batch_value "$THREADS_PER_CHILD" "unknown")")
        fields+=("recommended_min_spare_threads=$(format_batch_value "$MIN_SPARE_THREADS" "unknown")")
        fields+=("recommended_max_spare_threads=$(format_batch_value "$MAX_SPARE_THREADS" "unknown")")
        fields+=("recommended_min_spare_servers=na")
        fields+=("recommended_max_spare_servers=na")
    fi

    fields+=("recommended_server_limit=$(format_batch_value "$SERVER_LIMIT" "unknown")")
    fields+=("recommended_max_requestworkers=$(format_batch_value "$MAX_WORKERS" "unknown")")
    fields+=("recommended_start_servers=$(format_batch_value "$START_SERVERS_ROUNDED" "unknown")")
    fields+=("recommended_max_connections_per_child=$(format_batch_value "$MAX_CONN_PER_CHILD" "unknown")")

    printf "%s\n" "${fields[*]}"
}

export_recommended_block() {
    local DEST="$1"

    if [[ -z "$DEST" ]]; then
        echo "ERROR: --export requires a valid file path"
        exit 1
    fi

    local DEST_DIR
    DEST_DIR=$(dirname "$DEST")
    if [[ ! -d "$DEST_DIR" ]]; then
        echo "ERROR: Export directory does not exist: $DEST_DIR"
        exit 1
    fi

    printf "%b" "$RECOMMENDED_BLOCK" > "$DEST"

    if [[ "$OUTPUT_FORMAT" == "text" ]]; then
        echo "Smart Tuner block exported to: $DEST"
    fi

    log_message "INFO" "Exported Smart Tuner block to $DEST"
}

# ----------------- current config parsing -----------------

parse_file_for_directives() {
    local FILE="$1"
    [[ ! -f "$FILE" ]] && return

    while IFS= read -r line; do
        [[ "$line" =~ ^[[:space:]]*# ]] && continue
        local clean
        clean=$(echo "$line" | sed -e 's/^[[:space:]]*//')

        case "$clean" in
            ServerLimit*)
                [[ -z "${CUR_SERVER_LIMIT:-}" || "$CUR_SERVER_LIMIT" == "unknown" ]] \
                    && CUR_SERVER_LIMIT=$(echo "$clean" | awk '{print $2}')
                ;;
            MaxRequestWorkers*|MaxClients*)
                [[ -z "${CUR_MAX_WORKERS:-}" || "$CUR_MAX_WORKERS" == "unknown" ]] \
                    && CUR_MAX_WORKERS=$(echo "$clean" | awk '{print $2}')
                ;;
            StartServers*)
                [[ -z "${CUR_START_SERVERS:-}" || "$CUR_START_SERVERS" == "unknown" ]] \
                    && CUR_START_SERVERS=$(echo "$clean" | awk '{print $2}')
                ;;
            MinSpareServers*)
                [[ -z "${CUR_MIN_SPARE_SERVERS:-}" || "$CUR_MIN_SPARE_SERVERS" == "unknown" ]] \
                    && CUR_MIN_SPARE_SERVERS=$(echo "$clean" | awk '{print $2}')
                ;;
            MaxSpareServers*)
                [[ -z "${CUR_MAX_SPARE_SERVERS:-}" || "$CUR_MAX_SPARE_SERVERS" == "unknown" ]] \
                    && CUR_MAX_SPARE_SERVERS=$(echo "$clean" | awk '{print $2}')
                ;;
            MinSpareThreads*)
                [[ -z "${CUR_MIN_SPARE_THREADS:-}" || "$CUR_MIN_SPARE_THREADS" == "unknown" ]] \
                    && CUR_MIN_SPARE_THREADS=$(echo "$clean" | awk '{print $2}')
                ;;
            MaxSpareThreads*)
                [[ -z "${CUR_MAX_SPARE_THREADS:-}" || "$CUR_MAX_SPARE_THREADS" == "unknown" ]] \
                    && CUR_MAX_SPARE_THREADS=$(echo "$clean" | awk '{print $2}')
                ;;
            ThreadsPerChild*)
                [[ -z "${CUR_THREADS_PER_CHILD:-}" || "$CUR_THREADS_PER_CHILD" == "unknown" ]] \
                    && CUR_THREADS_PER_CHILD=$(echo "$clean" | awk '{print $2}')
                ;;
            MaxConnectionsPerChild*|MaxRequestsPerChild*)
                [[ -z "${CUR_MAX_CONN_PER_CHILD:-}" || "$CUR_MAX_CONN_PER_CHILD" == "unknown" ]] \
                    && CUR_MAX_CONN_PER_CHILD=$(echo "$clean" | awk '{print $2}')
                ;;
        esac
    done < <(grep -E 'ServerLimit|MaxRequestWorkers|MaxClients|StartServers|MinSpareServers|MaxSpareServers|MinSpareThreads|MaxSpareThreads|ThreadsPerChild|MaxConnectionsPerChild|MaxRequestsPerChild' "$FILE" 2>/dev/null || true)
}

get_current_values_with_precedence() {
    CUR_SERVER_LIMIT=""
    CUR_MAX_WORKERS=""
    CUR_THREADS_PER_CHILD=""
    CUR_START_SERVERS=""
    CUR_MIN_SPARE_THREADS=""
    CUR_MAX_SPARE_THREADS=""
    CUR_MIN_SPARE_SERVERS=""
    CUR_MAX_SPARE_SERVERS=""
    CUR_MAX_CONN_PER_CHILD=""

    # 1) cPanel include wins
    if [[ "$IS_CPANEL" -eq 1 && -n "${PREFORK_INCLUDE:-}" && -f "$PREFORK_INCLUDE" ]]; then
        parse_file_for_directives "$PREFORK_INCLUDE"
    fi

    # 2) Runtime (DUMP_RUN_CFG) fills blanks
    local CURRENT_DUMP=""
    [[ ${#APACHE_BINARIES[@]} -eq 0 ]] && discover_apache_binaries

    for bin in "${APACHE_BINARIES[@]}"; do
        case "$bin" in
            apachectl|apache2ctl|httpd)
                CURRENT_DUMP=$("$bin" -t -D DUMP_RUN_CFG 2>/dev/null || true)
                [[ -n "$CURRENT_DUMP" ]] && break
                ;;
        esac
    done

    if [[ -n "$CURRENT_DUMP" ]] && echo "$CURRENT_DUMP" | grep -q "ServerLimit:" 2>/dev/null; then
        [[ -z "$CUR_SERVER_LIMIT" || "$CUR_SERVER_LIMIT" == "unknown" ]] \
            && CUR_SERVER_LIMIT=$(echo "$CURRENT_DUMP" | awk '/ServerLimit:/ {print $2; exit}')
        local _mrw
        _mrw=$(echo "$CURRENT_DUMP" | awk '/MaxRequestWorkers:/ {print $2; found=1} /MaxClients:/ && !found {print $2; found=1} END {if(!found) print ""}')
        [[ -n "$_mrw" && ( -z "$CUR_MAX_WORKERS" || "$CUR_MAX_WORKERS" == "unknown" ) ]] && CUR_MAX_WORKERS="$_mrw"
        [[ -z "$CUR_THREADS_PER_CHILD" || "$CUR_THREADS_PER_CHILD" == "unknown" ]] \
            && CUR_THREADS_PER_CHILD=$(echo "$CURRENT_DUMP" | awk '/ThreadsPerChild:/ {print $2; exit}')
        [[ -z "$CUR_START_SERVERS" || "$CUR_START_SERVERS" == "unknown" ]] \
            && CUR_START_SERVERS=$(echo "$CURRENT_DUMP" | awk '/StartServers:/ {print $2; exit}')
        [[ -z "$CUR_MIN_SPARE_THREADS" || "$CUR_MIN_SPARE_THREADS" == "unknown" ]] \
            && CUR_MIN_SPARE_THREADS=$(echo "$CURRENT_DUMP" | awk '/MinSpareThreads:/ {print $2; exit}')
        [[ -z "$CUR_MAX_SPARE_THREADS" || "$CUR_MAX_SPARE_THREADS" == "unknown" ]] \
            && CUR_MAX_SPARE_THREADS=$(echo "$CURRENT_DUMP" | awk '/MaxSpareThreads:/ {print $2; exit}')
        [[ -z "$CUR_MIN_SPARE_SERVERS" || "$CUR_MIN_SPARE_SERVERS" == "unknown" ]] \
            && CUR_MIN_SPARE_SERVERS=$(echo "$CURRENT_DUMP" | awk '/MinSpareServers:/ {print $2; exit}')
        [[ -z "$CUR_MAX_SPARE_SERVERS" || "$CUR_MAX_SPARE_SERVERS" == "unknown" ]] \
            && CUR_MAX_SPARE_SERVERS=$(echo "$CURRENT_DUMP" | awk '/MaxSpareServers:/ {print $2; exit}')
        local _mcp
        _mcp=$(echo "$CURRENT_DUMP" | awk '/MaxConnectionsPerChild:/ {print $2; found=1} /MaxRequestsPerChild:/ && !found {print $2; found=1} END {if(!found) print ""}')
        [[ -n "$_mcp" && ( -z "$CUR_MAX_CONN_PER_CHILD" || "$CUR_MAX_CONN_PER_CHILD" == "unknown" ) ]] && CUR_MAX_CONN_PER_CHILD="$_mcp"
    fi

    # 3) httpd.conf as last fallback
    parse_file_for_directives "$HTTPD_CONF"

    : "${CUR_SERVER_LIMIT:=unknown}"
    : "${CUR_MAX_WORKERS:=unknown}"
    : "${CUR_THREADS_PER_CHILD:=unknown}"
    : "${CUR_START_SERVERS:=unknown}"
    : "${CUR_MIN_SPARE_THREADS:=unknown}"
    : "${CUR_MAX_SPARE_THREADS:=unknown}"
    : "${CUR_MIN_SPARE_SERVERS:=unknown}"
    : "${CUR_MAX_SPARE_SERVERS:=unknown}"
    : "${CUR_MAX_CONN_PER_CHILD:=unknown}"
}

# ----------------- recommendation logic -----------------

build_recommended_block() {
    local HEADER COMMON
    HEADER="# ${VERSION_NAME} (Tier: $TIER, MPM: $MPM_TYPE)\n"
    COMMON="Timeout 120\nKeepAlive On\nMaxKeepAliveRequests 100\nKeepAliveTimeout 5\n\n"

    if [[ "$MPM_TYPE" == "prefork" ]]; then
        RAW_MAX_WORKERS=$(echo "$APACHE_BUDGET_MB / $AVG_PROC_MB" | bc | awk '{printf "%.0f", $0}')
        [[ -z "$RAW_MAX_WORKERS" ]] && RAW_MAX_WORKERS=32

        if [[ "$TIER" == "LOW" || "$TIER" == "LOW-MID" ]]; then
            RAW_MAX_WORKERS=$(( RAW_MAX_WORKERS * 2 ))
        fi

        CPU_CAP=$(( CORES * PROCS_PER_CORE_CAP ))
        (( CPU_CAP < 128 )) && CPU_CAP=128
        (( RAW_MAX_WORKERS > CPU_CAP )) && RAW_MAX_WORKERS=$CPU_CAP

        MAX_WORKERS=$(round_down_to_multiple_of_8 "$RAW_MAX_WORKERS")
        (( MAX_WORKERS > MRW_CAP_PREFORK )) && MAX_WORKERS=$MRW_CAP_PREFORK
        (( MAX_WORKERS < 128 )) && MAX_WORKERS=128

        SERVER_LIMIT=$(calculate_server_limit_with_buffer "$MAX_WORKERS")
        MIN_SPARE_SERVERS=$(round_down_to_multiple_of_8 "$MIN_SPARE")
        MAX_SPARE_SERVERS=$(round_down_to_multiple_of_8 "$MAX_SPARE")
        START_SERVERS_ROUNDED=$(round_down_to_multiple_of_8 "$START_SERVERS")

        (( MAX_SPARE_SERVERS > MAX_WORKERS )) && MAX_SPARE_SERVERS=$MAX_WORKERS
        (( MIN_SPARE_SERVERS > MAX_WORKERS )) && MIN_SPARE_SERVERS=$MAX_WORKERS
        (( MIN_SPARE_SERVERS > MAX_SPARE_SERVERS )) && MIN_SPARE_SERVERS=$MAX_SPARE_SERVERS

        PREFORK_BLOCK="<IfModule prefork.c>\n"
        PREFORK_BLOCK+="    ServerLimit            $SERVER_LIMIT\n"
        PREFORK_BLOCK+="    MaxRequestWorkers      $MAX_WORKERS\n"
        PREFORK_BLOCK+="    StartServers           $START_SERVERS_ROUNDED\n"
        PREFORK_BLOCK+="    MinSpareServers        $MIN_SPARE_SERVERS\n"
        PREFORK_BLOCK+="    MaxSpareServers        $MAX_SPARE_SERVERS\n"
        PREFORK_BLOCK+="    MaxConnectionsPerChild $MAX_CONN_PER_CHILD\n"
        PREFORK_BLOCK+="</IfModule>\n"

        RECOMMENDED_BLOCK="# BEGIN APACHE_SMART_TUNER\n${HEADER}${COMMON}${PREFORK_BLOCK}# END APACHE_SMART_TUNER\n"

    else
        THREADS_PER_CHILD=25

        RAW_MAX_PROCS=$(echo "$APACHE_BUDGET_MB / $AVG_PROC_MB" | bc | awk '{printf "%.0f", $0}')
        [[ -z "$RAW_MAX_PROCS" ]] && RAW_MAX_PROCS=8

        if [[ "$TIER" == "LOW" || "$TIER" == "LOW-MID" ]]; then
            RAW_MAX_PROCS=$(( RAW_MAX_PROCS * 2 ))
        fi

        CPU_THREAD_CAP=$(( CORES * THREADS_PER_CORE_CAP ))
        CPU_PROC_CAP=$(( CPU_THREAD_CAP / THREADS_PER_CHILD ))
        (( RAW_MAX_PROCS > CPU_PROC_CAP )) && RAW_MAX_PROCS=$CPU_PROC_CAP

        MAX_PROCS=$(round_down_to_multiple_of_8 "$RAW_MAX_PROCS")
        (( MAX_PROCS < 1 )) && MAX_PROCS=1

        MAX_WORKERS=$(( MAX_PROCS * THREADS_PER_CHILD ))
        (( MAX_WORKERS > MRW_CAP_THREADED )) && MAX_WORKERS=$MRW_CAP_THREADED
        (( MAX_WORKERS < 128 )) && MAX_WORKERS=128

        MAX_PROCS=$(( (MAX_WORKERS + THREADS_PER_CHILD - 1) / THREADS_PER_CHILD ))
        MAX_PROCS=$(round_down_to_multiple_of_8 "$MAX_PROCS")
        (( MAX_PROCS < 1 )) && MAX_PROCS=1
        MAX_WORKERS=$(( MAX_PROCS * THREADS_PER_CHILD ))
        SERVER_LIMIT=$(calculate_server_limit_with_buffer "$MAX_PROCS")

        MIN_SPARE_THREADS=$(( MIN_SPARE * THREADS_PER_CHILD ))
        MAX_SPARE_THREADS=$(( MAX_SPARE * THREADS_PER_CHILD ))

        MIN_SPARE_THREADS=$(round_down_to_multiple_of_8 "$MIN_SPARE_THREADS")
        MAX_SPARE_THREADS=$(round_down_to_multiple_of_8 "$MAX_SPARE_THREADS")
        START_SERVERS_ROUNDED=$(round_down_to_multiple_of_8 "$START_SERVERS")

        (( MAX_SPARE_THREADS > MAX_WORKERS )) && MAX_SPARE_THREADS=$MAX_WORKERS
        (( MIN_SPARE_THREADS > MAX_WORKERS )) && MIN_SPARE_THREADS=$MAX_WORKERS
        (( MIN_SPARE_THREADS > MAX_SPARE_THREADS )) && MIN_SPARE_THREADS=$MAX_SPARE_THREADS

        MPM_BLOCK="<IfModule mpm_${MPM_TYPE}_module>\n"
        MPM_BLOCK+="    ServerLimit            $SERVER_LIMIT\n"
        MPM_BLOCK+="    StartServers           $START_SERVERS_ROUNDED\n"
        MPM_BLOCK+="    MaxRequestWorkers      $MAX_WORKERS\n"
        MPM_BLOCK+="    ThreadsPerChild        $THREADS_PER_CHILD\n"
        MPM_BLOCK+="    MinSpareThreads        $MIN_SPARE_THREADS\n"
        MPM_BLOCK+="    MaxSpareThreads        $MAX_SPARE_THREADS\n"
        MPM_BLOCK+="    MaxConnectionsPerChild $MAX_CONN_PER_CHILD\n"
        MPM_BLOCK+="</IfModule>\n"

        RECOMMENDED_BLOCK="# BEGIN APACHE_SMART_TUNER\n${HEADER}${COMMON}${MPM_BLOCK}# END APACHE_SMART_TUNER\n"
    fi
}

# ----------------- apply config safely -----------------

reload_apache() {
    if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files >/dev/null 2>&1; then
        for unit in httpd apache2; do
            if systemctl list-unit-files | grep -q "^${unit}\.service"; then
                systemctl reload "$unit" || systemctl restart "$unit"
                return
            fi
        done
    fi

    if command -v service >/dev/null 2>&1; then
        for svc in httpd apache2; do
            if service "$svc" status >/dev/null 2>&1; then
                service "$svc" reload || service "$svc" restart
                return
            fi
        done
    fi

    if command -v apachectl >/dev/null 2>&1; then
        apachectl graceful || apachectl restart
    elif command -v apache2ctl >/dev/null 2>&1; then
        apache2ctl graceful || apache2ctl restart
    fi
}

config_matches_recommendation() {
    local FILE="$1"
    local EXISTING_BLOCK

    EXISTING_BLOCK=$(extract_tuner_block "$FILE")
    [[ -z "$EXISTING_BLOCK" ]] && return 1

    [[ "$(normalize_block_for_compare "$EXISTING_BLOCK")" == "$(normalize_block_for_compare "$RECOMMENDED_BLOCK")" ]]
}

# Decide how to respond after applying the block (rollback, log, or reload).
handle_post_apply() {
    local configtest_status="$1"
    local backup_file="$2"
    local target_file="$3"

    if [[ "$configtest_status" == "fail" ]]; then
        echo "ERROR: Apache configtest FAILED after applying changes." >&2
        echo "Restoring backup: $backup_file" >&2
        cp -p "$backup_file" "$target_file"
        log_message "ERROR" "Post-apply configtest failed; restored backup $backup_file"
        if [[ "$IS_CPANEL" -eq 1 && -x /scripts/rebuildhttpdconf ]]; then
            /scripts/rebuildhttpdconf
        fi
        apache_configtest || echo "WARNING: Apache config still failing after rollback; manual intervention required." >&2
        exit 1
    fi

    if [[ "${APACHE_RUNNING:-0}" -ne 1 ]]; then
        print_if_text "Configtest PASSED. Apache appears stopped; skipping reload."
        print_if_text "Start Apache manually to apply new settings, e.g. systemctl start httpd"
        print_if_text "Apache Smart Tuner: configuration written; awaiting manual Apache start."
        log_message "WARN" "Config applied but Apache not running; skipped reload"
        log_message "INFO" "Applied Smart Tuner configuration to $target_file with tier $TIER and MPM $MPM_TYPE"
    elif [[ "$RELOAD_AFTER_APPLY" -eq 0 ]]; then
        print_if_text "Configtest PASSED. Reload skipped due to --no-reload."
        print_if_text "Apache Smart Tuner: configuration written; reload manually to activate."
        log_message "INFO" "Config applied to $target_file with reload intentionally skipped"
    else
        print_if_text "Configtest PASSED. Reloading Apache..."
        reload_apache
        log_message "INFO" "Reloaded Apache after applying Smart Tuner configuration"

        print_if_text "Apache Smart Tuner: configuration successfully applied."
        log_message "INFO" "Applied Smart Tuner configuration to $target_file with tier $TIER and MPM $MPM_TYPE"
    fi
}

# Apply the recommended block with backups, cleanup, and optional reloads.
apply_recommended_config() {
    if [[ "$EUID" -ne 0 ]]; then
        echo "ERROR: --apply must be run as root." >&2
        exit 1
    fi

    touch "$LOG_FILE" 2>/dev/null || true

    print_if_text "Running preflight Apache configtest..."
    if ! apache_configtest; then
        echo "ERROR: Apache configtest FAILED before making changes." >&2
        log_message "ERROR" "Preflight Apache configtest failed; aborting apply"
        exit 1
    fi

    if [[ "$IS_CPANEL" -eq 1 && -d "$INCLUDES_DIR" ]]; then
        if [[ -n "${PREFORK_INCLUDE:-}" && -f "$PREFORK_INCLUDE" ]]; then
            TARGET_FILE="$PREFORK_INCLUDE"
        else
            TARGET_FILE="${INCLUDES_DIR}/pre_main_global.conf"
        fi
    else
        mkdir -p "$INCLUDES_DIR"
        TARGET_FILE="${INCLUDES_DIR}/apache-smart-tuner.conf"
    fi

    [[ ! -f "$TARGET_FILE" ]] && touch "$TARGET_FILE"

    if config_matches_recommendation "$TARGET_FILE"; then
        print_if_text "Smart Tuner block already matches recommendation in $TARGET_FILE; skipping backup and reload."
        log_message "INFO" "Skipped apply: $TARGET_FILE already contains current Smart Tuner block"
        return
    fi

    local TS BACKUP_FILE
    TS=$(date +%Y%m%d%H%M%S)
    BACKUP_FILE="${TARGET_FILE}.bk-${TS}"
    cp -p "$TARGET_FILE" "$BACKUP_FILE"
    print_if_text "Backup created at: $BACKUP_FILE"
    log_message "INFO" "Backup created at $BACKUP_FILE for target $TARGET_FILE"

    local tmpfile
    tmpfile=$(mktemp)
    awk '
        BEGIN {
            in_tuner = 0;
            in_prefork = 0;
            in_worker = 0;
            in_event = 0;
            in_mpm = 0;
        }
        /BEGIN APACHE_SMART_TUNER/ { in_tuner=1; next }
        /END APACHE_SMART_TUNER/   { in_tuner=0; next }

        /<IfModule[[:space:]]+prefork\.c>/         { in_prefork=1; next }
        /<IfModule[[:space:]]+worker\.c>/          { in_worker=1;  next }
        /<IfModule[[:space:]]+event\.c>/           { in_event=1;   next }
        /<IfModule[[:space:]]+mpm_(prefork|worker|event)_module>/ { in_mpm=1; next }

        /<\/IfModule>/ {
            if (in_prefork || in_worker || in_event || in_mpm) {
                in_prefork=0; in_worker=0; in_event=0; in_mpm=0;
                next;
            }
        }

        in_tuner || in_prefork || in_worker || in_event || in_mpm { next }

        { print }
    ' "$TARGET_FILE" > "$tmpfile"

    mv "$tmpfile" "$TARGET_FILE"
    printf "%b" "$RECOMMENDED_BLOCK" >> "$TARGET_FILE"

    if [[ "$IS_CPANEL" -eq 1 && -x /scripts/rebuildhttpdconf ]]; then
        print_if_text "Detected cPanel. Running /scripts/rebuildhttpdconf..."
        /scripts/rebuildhttpdconf
        log_message "INFO" "Ran /scripts/rebuildhttpdconf after applying Smart Tuner block"
    fi

    print_if_text "Running Apache configtest after applying changes..."
    if apache_configtest; then
        handle_post_apply "pass" "$BACKUP_FILE" "$TARGET_FILE"
    else
        handle_post_apply "fail" "$BACKUP_FILE" "$TARGET_FILE"
    fi
}

# ----------------- locate directives (QUIET) -----------------

locate_current_mpm_directives() {
    local SEARCH_DIRS=()
    [[ -d "$APACHE_ROOT/conf" ]]   && SEARCH_DIRS+=("$APACHE_ROOT/conf")
    [[ -d "$APACHE_ROOT/conf.d" ]] && SEARCH_DIRS+=("$APACHE_ROOT/conf.d")
    [[ -n "$INCLUDES_DIR" && -d "$INCLUDES_DIR" ]] && SEARCH_DIRS+=("$INCLUDES_DIR")

    if ((${#SEARCH_DIRS[@]} == 0)); then
        echo "No Apache config directories found to search."
        return
    fi

    echo "Apache config search roots:"
    for d in "${SEARCH_DIRS[@]}"; do
        echo "  $d"
    done
    echo

    echo "Files defining MPM modules (prefork/worker/event):"
    grep -RIlE 'mpm_(prefork|worker|event)_module|prefork.c' "${SEARCH_DIRS[@]}" 2>/dev/null \
        | grep -E '\.conf(\.|$)' \
        | sort -u || echo "  (none found)"
    echo

    echo "Files defining concurrency / spare settings:"
    grep -RIlE 'ServerLimit|MaxRequestWorkers|MaxClients|ThreadsPerChild|MinSpareThreads|MaxSpareThreads|MinSpareServers|MaxSpareServers|MaxRequestsPerChild|MaxConnectionsPerChild' "${SEARCH_DIRS[@]}" 2>/dev/null \
        | grep -E '\.conf(\.|$)' \
        | sort -u || echo "  (none found)"
}

# Normalize values before printing to avoid control characters (e.g., \r)
sanitize_value() {
    local val="$1"
    # Remove carriage returns/tabs that can arrive from config dumps
    val=${val//$'\r'/}
    val=${val//$'\t'/}
    echo "$val"
}

format_health_line() {
    local status="$1"
    local label="$2"
    local detail="$3"

    printf "  %-4s %-32s %s\n" "$status" "$label" "$detail"
}

print_log_review() {
    if [[ "$LOG_REVIEW_STATUS" != "ready" ]]; then
        format_health_line "[--]" "Log availability" "$LOG_REVIEW_MESSAGE"
        return
    fi

    format_health_line "[--]" "Log source" "$APACHE_ERROR_LOG (last ${ERROR_LOG_SAMPLE_LINES} lines)"

    if (( SCOREBOARD_HITS > 0 )); then
        format_health_line "[!!]" "Worker saturation" "$SCOREBOARD_HITS entries showing MaxRequestWorkers/scoreboard exhaustion"
    else
        format_health_line "[OK]" "Worker saturation" "No MaxRequestWorkers saturation observed"
    fi

    if (( SERVER_LIMIT_HITS > 0 )); then
        format_health_line "[!!]" "ServerLimit warnings" "$SERVER_LIMIT_HITS ServerLimit/MaxRequestWorkers notices"
    else
        format_health_line "[OK]" "ServerLimit warnings" "No ServerLimit notices detected"
    fi

    for key in "${LOG_REVIEW_KEYS[@]}"; do
        local label="${COMMON_ERROR_LABELS[$key]}"
        local count=${COMMON_ERROR_COUNTS[$key]:-0}
        local status="[OK]"
        local detail="No $label observed in sampled lines."

        if (( count > 0 )); then
            status="[--]"
            detail="$count entries related to $label."
            [[ "$key" == "segfaults" ]] && status="[!!]"
        fi

        format_health_line "$status" "$label" "$detail"
    done

    local restart_status="[OK]"
    local restart_detail="No restart events in sampled log."

    if (( RESTART_TOTAL > 0 )); then
        restart_detail="$RESTART_TOTAL restart events; max $RESTART_MAX_PER_DAY/day across $RESTART_DAY_COUNT day(s)."
        restart_status="[--]"
        if (( RESTART_MAX_PER_DAY > 3 )); then
            restart_status="[!!]"
        fi
    fi

    format_health_line "$restart_status" "Apache restarts" "$restart_detail"
}

print_current_vs_proposed() {
    local label
    local current
    local proposed

    label=$(sanitize_value "$1")
    current=$(sanitize_value "$2")
    proposed=$(sanitize_value "$3")
    printf "  %-22s %-12s => %s\n" "$label" "$current" "$proposed"
}

# ----------------- main -----------------

parse_cli_arguments "$@"

preflight_validate_options

bootstrap_environment

if [[ "$MODE" == "locate" ]]; then
    locate_current_mpm_directives
    exit 0
fi

start_progress_indicator "Collecting Apache metrics and log data"
run_analysis_pipeline
stop_progress_indicator

if [[ -n "$EXPORT_PATH" ]]; then
    export_recommended_block "$EXPORT_PATH"
fi

if [[ "$OUTPUT_FORMAT" == "json" ]]; then
    build_json_output
elif [[ "$OUTPUT_FORMAT" == "batch" ]]; then
    build_batch_output
else
    echo "------------------------------------------------"
    echo " Apache Smart Tuner Analysis"
    echo "------------------------------------------------"
    echo "Detected MPM:           $MPM_TYPE (source: $MPM_SOURCE)"
    echo "Tier:                   $TIER"
    echo "Total RAM:              ${TOTAL_MEM} MB"
    echo "CPU Cores:              ${CORES}"
    if [[ "$APACHE_RUNNING" -eq 1 ]]; then
        echo "Apache running:         yes (${APACHE_PROCS} processes observed)"
    else
        echo "Apache running:         no (using default ${MPM_TYPE} memory model)"
    fi
    echo "Apache RAM Budget:      ${APACHE_BUDGET_MB} MB (${APACHE_PCT} of total; source: ${BUDGET_SOURCE})"
    echo "Avg httpd proc size:    ${AVG_PROC_MB} MB"
    echo "------------------------------------------------"
    echo " Error log & stability review"
    echo "------------------------------------------------"
    print_log_review
    echo "------------------------------------------------"
    echo "Current vs Proposed (${MPM_TYPE}) (current => proposed):"

    if [[ "$MPM_TYPE" == "prefork" ]]; then
        print_current_vs_proposed "ServerLimit" "$CUR_SERVER_LIMIT" "$SERVER_LIMIT"
        print_current_vs_proposed "MaxRequestWorkers" "$CUR_MAX_WORKERS" "$MAX_WORKERS"
        print_current_vs_proposed "StartServers" "$CUR_START_SERVERS" "$START_SERVERS_ROUNDED"
        print_current_vs_proposed "MinSpareServers" "$CUR_MIN_SPARE_SERVERS" "$MIN_SPARE_SERVERS"
        print_current_vs_proposed "MaxSpareServers" "$CUR_MAX_SPARE_SERVERS" "$MAX_SPARE_SERVERS"
        print_current_vs_proposed "MaxConnectionsPerChild" "$CUR_MAX_CONN_PER_CHILD" "$MAX_CONN_PER_CHILD"
    else
        print_current_vs_proposed "ServerLimit" "$CUR_SERVER_LIMIT" "$SERVER_LIMIT"
        print_current_vs_proposed "MaxRequestWorkers" "$CUR_MAX_WORKERS" "$MAX_WORKERS"
        print_current_vs_proposed "ThreadsPerChild" "$CUR_THREADS_PER_CHILD" "$THREADS_PER_CHILD"
        print_current_vs_proposed "StartServers" "$CUR_START_SERVERS" "$START_SERVERS_ROUNDED"
        print_current_vs_proposed "MinSpareThreads" "$CUR_MIN_SPARE_THREADS" "$MIN_SPARE_THREADS"
        print_current_vs_proposed "MaxSpareThreads" "$CUR_MAX_SPARE_THREADS" "$MAX_SPARE_THREADS"
        print_current_vs_proposed "MaxConnectionsPerChild" "$CUR_MAX_CONN_PER_CHILD" "$MAX_CONN_PER_CHILD"
    fi

    echo "------------------------------------------------"
    echo -e "$RECOMMENDED_BLOCK"
    echo "------------------------------------------------"
fi

if [[ "$MODE" == "apply" ]]; then
    apply_recommended_config
fi