diff --git a/e2e_test/fastrg_grpc_client.py b/e2e_test/fastrg_grpc_client.py new file mode 100644 index 0000000..50c803e --- /dev/null +++ b/e2e_test/fastrg_grpc_client.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +""" +FastRG Node gRPC client helper — uses grpcurl subprocess (no grpcio/pb2 required). + +Usage: + fastrg_grpc_client.py --node [args...] + +Commands: + get_hsi_info - GetFastrgHsiInfo → JSON + get_dhcp_info - GetFastrgDhcpInfo → JSON + get_system_info - GetFastrgSystemInfo → JSON + get_port_fwd_info - GetPortFwdInfo → JSON + get_dns_static - GetDnsStaticRecords → JSON + apply_config + - ApplyConfig → JSON + remove_config - RemoveConfig → JSON + connect_hsi - ConnectHsi → JSON + disconnect_hsi - DisconnectHsi → JSON + start_dhcp_server - DhcpServerStart → JSON + stop_dhcp_server - DhcpServerStop → JSON + add_dns_record - AddDnsRecord → JSON + remove_dns_record - RemoveDnsRecord → JSON + set_subscriber_count - SetSubscriberCount → JSON + +Requirements: + - python3 (stdlib only) + - grpcurl binary: looked up in this script's directory first, then PATH + - fastrg_node.proto: must be in the same directory as this script +""" + +import sys +import os +import json +import re +import subprocess +import argparse + +_DIR = os.path.dirname(os.path.abspath(__file__)) +TIMEOUT_SEC = 10 + +# --------------------------------------------------------------------------- +# grpcurl helpers +# --------------------------------------------------------------------------- + +def _find_grpcurl(): + """Return path to grpcurl binary or raise FileNotFoundError.""" + for candidate in [ + os.path.join(_DIR, 'grpcurl'), + '/usr/bin/grpcurl', + '/usr/local/bin/grpcurl', + ]: + if os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + import shutil + found = shutil.which('grpcurl') + if found: + return found + raise FileNotFoundError( + "grpcurl binary not found. Place it alongside this script or install it in PATH." + ) + + +def _camel_to_snake(name): + """Convert camelCase / PascalCase key to snake_case.""" + s = re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', name) + return re.sub(r'([a-z\d])([A-Z])', r'\1_\2', s).lower() + + +def _convert_keys(obj): + """Recursively convert all dict keys from camelCase to snake_case.""" + if isinstance(obj, dict): + return {_camel_to_snake(k): _convert_keys(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_convert_keys(i) for i in obj] + return obj + + +def _grpcurl(node_addr, method, data=None): + """Call grpcurl and return the response as a snake_case-keyed dict.""" + proto_file = os.path.join(_DIR, 'fastrg_node.proto') + if not os.path.exists(proto_file): + raise FileNotFoundError(f"proto not found: {proto_file}") + + grpcurl_bin = _find_grpcurl() + + cmd = [ + grpcurl_bin, + '-plaintext', + '-proto', 'fastrg_node.proto', + '-import-path', _DIR, + '-emit-defaults', + '-connect-timeout', str(TIMEOUT_SEC), + '-max-time', str(TIMEOUT_SEC), + ] + if data is not None: + cmd += ['-d', json.dumps(data)] + cmd += [node_addr, f'fastrgnodeservice.FastrgService/{method}'] + + result = subprocess.run( + cmd, capture_output=True, text=True, timeout=TIMEOUT_SEC + 5 + ) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip()) + + raw = json.loads(result.stdout) if result.stdout.strip() else {} + return _convert_keys(raw) + + +# --------------------------------------------------------------------------- +# Command implementations +# --------------------------------------------------------------------------- + +def get_hsi_info(node_addr): + resp = _grpcurl(node_addr, 'GetFastrgHsiInfo') + return {"hsi_infos": resp.get('hsi_infos', [])} + + +def get_dhcp_info(node_addr): + resp = _grpcurl(node_addr, 'GetFastrgDhcpInfo') + return {"dhcp_infos": resp.get('dhcp_infos', [])} + + +def get_system_info(node_addr): + resp = _grpcurl(node_addr, 'GetFastrgSystemInfo') + b = resp.get('base_info', {}) + return { + "fastrg_version": b.get('fastrg_version', ''), + "build_date": b.get('build_date', ''), + "uptime": b.get('uptime', 0), + "dpdk_version": b.get('dpdk_version', ''), + "num_users": b.get('num_users', 0), + } + + +def get_port_fwd_info(node_addr, user_id): + resp = _grpcurl(node_addr, 'GetPortFwdInfo', {'user_id': user_id}) + return { + "user_id": resp.get('user_id', user_id), + "entries": resp.get('entries', []), + } + + +def get_dns_static(node_addr, user_id): + resp = _grpcurl(node_addr, 'GetDnsStaticRecords', {'user_id': user_id}) + return { + "user_id": resp.get('user_id', user_id), + "total_entries": resp.get('total_entries', 0), + "entries": resp.get('entries', []), + } + + +def apply_config(node_addr, user_id, vlan_id, pppoe_account, pppoe_password, + dhcp_pool_start, dhcp_pool_end, dhcp_subnet_mask, dhcp_gateway): + resp = _grpcurl(node_addr, 'ApplyConfig', { + 'user_id': int(user_id), + 'vlan_id': int(vlan_id), + 'pppoe_account': pppoe_account, + 'pppoe_password': pppoe_password, + 'dhcp_pool_start': dhcp_pool_start, + 'dhcp_pool_end': dhcp_pool_end, + 'dhcp_subnet_mask': dhcp_subnet_mask, + 'dhcp_gateway': dhcp_gateway, + }) + return {"status": resp.get("status", "")} + + +def remove_config(node_addr, user_id): + resp = _grpcurl(node_addr, 'RemoveConfig', {'user_id': int(user_id)}) + return {"status": resp.get("status", "")} + + +def connect_hsi(node_addr, user_id): + resp = _grpcurl(node_addr, 'ConnectHsi', {'user_id': int(user_id)}) + return {"status": resp.get("status", "")} + + +def disconnect_hsi(node_addr, user_id): + resp = _grpcurl(node_addr, 'DisconnectHsi', {'user_id': int(user_id)}) + return {"status": resp.get("status", "")} + + +def start_dhcp_server(node_addr, user_id): + resp = _grpcurl(node_addr, 'DhcpServerStart', {'user_id': int(user_id)}) + return {"status": resp.get("status", "")} + + +def stop_dhcp_server(node_addr, user_id): + resp = _grpcurl(node_addr, 'DhcpServerStop', {'user_id': int(user_id)}) + return {"status": resp.get("status", "")} + + +def add_dns_record(node_addr, user_id, domain, ip, ttl): + resp = _grpcurl(node_addr, 'AddDnsRecord', { + 'user_id': int(user_id), + 'domain': domain, + 'ip': ip, + 'ttl': int(ttl), + }) + return {"status": resp.get("status", "")} + + +def remove_dns_record(node_addr, user_id, domain): + resp = _grpcurl(node_addr, 'RemoveDnsRecord', { + 'user_id': int(user_id), + 'domain': domain, + }) + return {"status": resp.get("status", "")} + + +def set_subscriber_count(node_addr, subscriber_count): + resp = _grpcurl(node_addr, 'SetSubscriberCount', {'subscriber_count': int(subscriber_count)}) + return {"status": resp.get("status", "")} + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser(description="FastRG gRPC CLI helper (grpcurl backend)") + parser.add_argument("--node", required=True, help="host:port of FastRG gRPC server") + parser.add_argument("command", help="Command to run") + parser.add_argument("args", nargs="*", help="Command arguments") + opts = parser.parse_args() + + try: + if opts.command == "get_hsi_info": + result = get_hsi_info(opts.node) + elif opts.command == "get_dhcp_info": + result = get_dhcp_info(opts.node) + elif opts.command == "get_system_info": + result = get_system_info(opts.node) + elif opts.command == "get_port_fwd_info": + if not opts.args: + print(json.dumps({"error": "get_port_fwd_info requires "}), + file=sys.stderr) + sys.exit(1) + result = get_port_fwd_info(opts.node, int(opts.args[0])) + elif opts.command == "get_dns_static": + if not opts.args: + print(json.dumps({"error": "get_dns_static requires "}), + file=sys.stderr) + sys.exit(1) + result = get_dns_static(opts.node, int(opts.args[0])) + elif opts.command == "apply_config": + if len(opts.args) < 8: + print(json.dumps({"error": "apply_config requires " + " "}), + file=sys.stderr) + sys.exit(1) + result = apply_config(opts.node, opts.args[0], opts.args[1], opts.args[2], opts.args[3], + opts.args[4], opts.args[5], opts.args[6], opts.args[7]) + elif opts.command == "remove_config": + if not opts.args: + print(json.dumps({"error": "remove_config requires "}), file=sys.stderr) + sys.exit(1) + result = remove_config(opts.node, opts.args[0]) + elif opts.command == "connect_hsi": + if not opts.args: + print(json.dumps({"error": "connect_hsi requires "}), file=sys.stderr) + sys.exit(1) + result = connect_hsi(opts.node, opts.args[0]) + elif opts.command == "disconnect_hsi": + if not opts.args: + print(json.dumps({"error": "disconnect_hsi requires "}), file=sys.stderr) + sys.exit(1) + result = disconnect_hsi(opts.node, opts.args[0]) + elif opts.command == "start_dhcp_server": + if not opts.args: + print(json.dumps({"error": "start_dhcp_server requires "}), file=sys.stderr) + sys.exit(1) + result = start_dhcp_server(opts.node, opts.args[0]) + elif opts.command == "stop_dhcp_server": + if not opts.args: + print(json.dumps({"error": "stop_dhcp_server requires "}), file=sys.stderr) + sys.exit(1) + result = stop_dhcp_server(opts.node, opts.args[0]) + elif opts.command == "add_dns_record": + if len(opts.args) < 4: + print(json.dumps({"error": "add_dns_record requires "}), + file=sys.stderr) + sys.exit(1) + result = add_dns_record(opts.node, opts.args[0], opts.args[1], opts.args[2], opts.args[3]) + elif opts.command == "remove_dns_record": + if len(opts.args) < 2: + print(json.dumps({"error": "remove_dns_record requires "}), + file=sys.stderr) + sys.exit(1) + result = remove_dns_record(opts.node, opts.args[0], opts.args[1]) + elif opts.command == "set_subscriber_count": + if not opts.args: + print(json.dumps({"error": "set_subscriber_count requires "}), file=sys.stderr) + sys.exit(1) + result = set_subscriber_count(opts.node, opts.args[0]) + else: + print(json.dumps({"error": f"Unknown command: {opts.command}"}), file=sys.stderr) + sys.exit(1) + + print(json.dumps(result)) + + except Exception as e: + print(json.dumps({"error": str(e)}), file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/e2e_test/phases/phase0_setup.sh b/e2e_test/phases/phase0_setup.sh new file mode 100644 index 0000000..fc1d000 --- /dev/null +++ b/e2e_test/phases/phase0_setup.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# --------------------------------------------------------------------------- +# Phase 0 — Prerequisites +# --------------------------------------------------------------------------- +phase0_setup() { + bold "═══════════════════════════════════════════════════════" + bold " Phase 0 — Prerequisite Checks" + bold "═══════════════════════════════════════════════════════" + + # Check local jq + if ! command -v jq >/dev/null 2>&1; then + error "jq is not installed on this machine. Please install jq first." + exit 1 + fi + info "Local jq: $(jq --version)" + + # Check SSH key + if [[ ! -f "$SSH_KEY" ]]; then + error "SSH key not found: ${SSH_KEY}" + exit 1 + fi + + # Check SSH connectivity to FastRG node + info "Checking SSH connectivity to FastRG node (${FASTRG_NODE})..." + if ! ssh_node "true" 2>/dev/null; then + error "Cannot SSH to FastRG node at ${FASTRG_NODE}" + exit 1 + fi + info "FastRG node reachable." + + # Read NODE_UUID + info "Reading NODE_UUID from /etc/fastrg/node_uuid..." + NODE_UUID=$(ssh_node "cat /etc/fastrg/node_uuid" 2>/dev/null | tr -d '[:space:]') + if [[ -z "$NODE_UUID" ]]; then + error "Failed to read NODE_UUID from /etc/fastrg/node_uuid on ${FASTRG_NODE}" + exit 1 + fi + info "NODE_UUID: ${NODE_UUID}" + + # Read ETCD_ENDPOINT from /etc/fastrg/config.cfg + info "Reading ETCD_ENDPOINT from /etc/fastrg/config.cfg..." + ETCD_RAW=$(ssh_node "grep 'EtcdEndpoints' /etc/fastrg/config.cfg" 2>/dev/null || true) + ETCD_ENDPOINT=$(printf '%s' "$ETCD_RAW" | awk -F'"' '{print $2}') + if [[ -z "$ETCD_ENDPOINT" ]]; then + error "Failed to parse EtcdEndpoints from /etc/fastrg/config.cfg" + exit 1 + fi + info "ETCD_ENDPOINT: ${ETCD_ENDPOINT}" + + # Read FASTRG_GRPC_PORT from node config (may override default) + _grpc_port_raw=$(ssh_node "grep 'NodeGrpcPort' /etc/fastrg/config.cfg 2>/dev/null" | awk -F'"' '{print $2}') + if [[ -n "$_grpc_port_raw" ]]; then + FASTRG_GRPC_PORT="$_grpc_port_raw" + fi + info "FASTRG_GRPC_PORT: ${FASTRG_GRPC_PORT}" + + # Export for use in all subsequent functions + export NODE_UUID ETCD_ENDPOINT FASTRG_GRPC_PORT + + # ------------------------------------------------------------------ + # Check Python3 + grpcurl + proto (fastrg_grpc_client.py uses grpcurl; + # no grpcio or pb2 stubs required) + # ------------------------------------------------------------------ + info "Checking Python3..." + if ! command -v python3 >/dev/null 2>&1; then + error "python3 is required but not found. Please install python3." + exit 1 + fi + if [[ ! -f "${GRPC_CLIENT_DIR}/fastrg_grpc_client.py" ]]; then + error "fastrg_grpc_client.py not found in ${GRPC_CLIENT_DIR}" + exit 1 + fi + info "Python3 gRPC client: ${GRPC_CLIENT_DIR}/fastrg_grpc_client.py" + + # Check grpcurl binary (bundled alongside script takes priority) + if [[ -f "${GRPC_CLIENT_DIR}/grpcurl" ]] && [[ -x "${GRPC_CLIENT_DIR}/grpcurl" ]]; then + info "grpcurl: ${GRPC_CLIENT_DIR}/grpcurl" + elif command -v grpcurl >/dev/null 2>&1; then + info "grpcurl: $(command -v grpcurl)" + else + error "grpcurl not found. Upload it alongside this script or install it in PATH." + exit 1 + fi + + # Check proto file (needed by grpcurl at runtime) + if [[ ! -f "${GRPC_CLIENT_DIR}/fastrg_node.proto" ]]; then + error "fastrg_node.proto not found in ${GRPC_CLIENT_DIR}" + error "Re-run the script from the repo root to allow automatic proto upload" + exit 1 + fi + info "proto: ${GRPC_CLIENT_DIR}/fastrg_node.proto" + + # ------------------------------------------------------------------ + # Ensure fastrg daemon is running; start it if not + # ------------------------------------------------------------------ + info "Checking if fastrg daemon is running on ${FASTRG_NODE}..." + FASTRG_PID=$(ssh_node "pgrep -x fastrg 2>/dev/null || pidof fastrg 2>/dev/null || true" | tr -d '[:space:]') + if [[ -z "$FASTRG_PID" ]]; then + warn "fastrg is NOT running — attempting to start..." + _FASTRG_DAEMON="/root/fastrg-node/fastrg" + _FASTRG_START_CMD="${_FASTRG_DAEMON} -l 1-9 -n 4 -a 0000:04:00.0 -a 0000:08:00.0" + info "Starting: ${_FASTRG_START_CMD}" + ssh_node "nohup ${_FASTRG_START_CMD} >/var/log/fastrg.log 2>&1 &" + _FASTRG_STARTED_BY_SCRIPT=1 + + # Wait up to 120 s for fastrg gRPC + HSI data for USER_ID to be ready + # (including PPPoE session establishment — account must be non-empty) + info "Waiting for fastrg gRPC + HSI data for USER_ID=${USER_ID} to be ready (up to 120s)..." + _ready=0 + for _i in $(seq 1 24); do + sleep 5 + _hsi=$(python3 "${GRPC_CLIENT_DIR}/fastrg_grpc_client.py" \ + --node "${FASTRG_NODE}:${FASTRG_GRPC_PORT}" \ + get_hsi_info 2>/dev/null || true) + # Wait for: user exists in hsi_infos AND account is non-empty (PPPoE session up) + if printf '%s' "$_hsi" | python3 -c \ + "import sys,json; d=json.load(sys.stdin); \ + infos=d.get('hsi_infos',[]); \ + u=[h for h in infos if h.get('user_id')==${USER_ID}]; \ + sys.exit(0 if u and u[0].get('account','') else 1)" \ + 2>/dev/null; then + _ready=1 + break + fi + _cnt=$(printf '%s' "$_hsi" | python3 -c \ + 'import sys,json; d=json.load(sys.stdin); print(len(d.get("hsi_infos",[])))' \ + 2>/dev/null || echo '?') + _acct=$(printf '%s' "$_hsi" | python3 -c \ + "import sys,json; d=json.load(sys.stdin); \ + u=[h for h in d.get('hsi_infos',[]) if h.get('user_id')==${USER_ID}]; \ + print(u[0].get('account','') if u else '')" \ + 2>/dev/null || echo '') + info " still waiting... (${_i}x5s, hsi_infos=${_cnt}, account='${_acct}')" + done + if [[ $_ready -eq 0 ]]; then + error "fastrg gRPC did not become ready within 120 seconds." + info "Last log output:" + ssh_node "tail -20 /var/log/fastrg.log 2>/dev/null || true" + exit 1 + fi + + # Additionally wait for DNS static records to be loaded (needed for step 4d) + _dns_ready=0 + for _i in $(seq 1 12); do + _dns=$(python3 "${GRPC_CLIENT_DIR}/fastrg_grpc_client.py" \ + --node "${FASTRG_NODE}:${FASTRG_GRPC_PORT}" \ + get_dns_static "${USER_ID}" 2>/dev/null || true) + if printf '%s' "$_dns" | python3 -c \ + "import sys,json; d=json.load(sys.stdin); \ + sys.exit(0 if d.get('total_entries',0) > 0 or d.get('entries') else 1)" \ + 2>/dev/null; then + _dns_ready=1 + break + fi + info " waiting for DNS static records... (${_i}x5s)" + sleep 5 + done + [[ $_dns_ready -eq 0 ]] && info " (DNS static not loaded yet — will verify in step 4d)" + + info "fastrg gRPC is ready." + else + info "fastrg is running (pid: ${FASTRG_PID})." + _FASTRG_STARTED_BY_SCRIPT=0 + # Verify gRPC is reachable + _sys=$(python3 "${GRPC_CLIENT_DIR}/fastrg_grpc_client.py" \ + --node "${FASTRG_NODE}:${FASTRG_GRPC_PORT}" \ + get_system_info 2>/dev/null || true) + if ! printf '%s' "$_sys" | python3 -c "import sys,json; json.load(sys.stdin)" 2>/dev/null; then + error "fastrg is running but gRPC on ${FASTRG_NODE}:${FASTRG_GRPC_PORT} is not responding." + exit 1 + fi + info "fastrg gRPC is reachable." + fi + + printf "\n" + info "Configuration summary:" + printf " USER_ID : %s\n" "$USER_ID" + printf " FASTRG_NODE : %s\n" "$FASTRG_NODE" + printf " FASTRG_GRPC : %s:%s\n" "$FASTRG_NODE" "$FASTRG_GRPC_PORT" + printf " LAN_HOST : %s (user: the)\n" "$LAN_HOST" + printf " WAN_HOST : %s\n" "$WAN_HOST" + printf " WAN_IP : %s\n" "$WAN_IP" + printf " NODE_UUID : %s\n" "$NODE_UUID" + printf " ETCD_ENDPOINT : %s\n" "$ETCD_ENDPOINT" + printf "\n" +} diff --git a/e2e_test/phases/phase1_subscriber_count_tests.sh b/e2e_test/phases/phase1_subscriber_count_tests.sh new file mode 100644 index 0000000..a1d4215 --- /dev/null +++ b/e2e_test/phases/phase1_subscriber_count_tests.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# --------------------------------------------------------------------------- +# Phase 1 — SetSubscriberCount gRPC→etcd→fastrg Tests (Steps 19–20) +# +# Calls SetSubscriberCount gRPC, verifies the etcd user_counts key is +# updated, and verifies fastrg_node local subscriber count (num_users in +# GetFastrgSystemInfo) changes accordingly. Restores the original count at +# the end so other phases are not affected. +# --------------------------------------------------------------------------- +phase1_subscriber_count_tests() { + bold "═══════════════════════════════════════════════════════" + bold " Phase 1 — SetSubscriberCount gRPC→etcd Tests (Steps 1-2)" + bold "═══════════════════════════════════════════════════════" + + # Read current subscriber count from fastrg local state + info "Reading current subscriber count from fastrg (GetFastrgSystemInfo)..." + _sc_sys_orig=$(fastrg_grpc get_system_info) + _sc_orig=$(printf '%s' "$_sc_sys_orig" | jq -r '.num_users // 0' 2>/dev/null || echo 0) + + if [[ "$_sc_orig" -eq 0 ]]; then + skip "Step 1: SetSubscriberCount new value" \ + "Cannot read current subscriber count from fastrg (got 0 or error)" + skip "Step 2: Restore SubscriberCount" \ + "Cannot read current subscriber count from fastrg" + return + fi + info "Current subscriber count: ${_sc_orig}" + + # Use current + 1 as the test value (safe: well within MAX_USER_COUNT 4000) + _sc_new=$((_sc_orig + 1)) + + # ------------------------------------------------------------------ + # Step 1 — SetSubscriberCount → etcd updated + fastrg local updated + # ------------------------------------------------------------------ + info "Step 1: SetSubscriberCount ${_sc_new} (current=${_sc_orig})..." + _sc20_reply=$(fastrg_grpc set_subscriber_count "${_sc_new}") + _sc20_status=$(printf '%s' "$_sc20_reply" | jq -r '.status // empty' 2>/dev/null || true) + + if [[ -z "$_sc20_status" ]]; then + fail "Step 1: SetSubscriberCount ${_sc_new}" \ + "gRPC returned no status — response: $(printf '%s' "$_sc20_reply")" + skip "Step 2: Restore SubscriberCount ${_sc_orig}" "SetSubscriberCount failed" + return + fi + + sleep 2 # allow etcd write + watcher callback to apply locally + + # Verify etcd: key user_counts/{NODE_UUID}/ has subscriber_count = _sc_new + _sc19_etcd=$(etcdctl_get_value "user_counts/${NODE_UUID}/" 2>/dev/null || true) + _sc19_etcd_val=$(printf '%s' "$_sc19_etcd" | jq -r '.subscriber_count // empty' 2>/dev/null || true) + + # Verify fastrg local: get_system_info num_users = _sc_new + _sc19_sys=$(fastrg_grpc get_system_info) + _sc19_local=$(printf '%s' "$_sc19_sys" | jq -r '.num_users // 0' 2>/dev/null || echo 0) + + MISMATCH="" + [[ "$_sc19_etcd_val" != "$_sc_new" ]] && \ + MISMATCH="${MISMATCH} etcd(got=${_sc19_etcd_val:-empty} expected=${_sc_new})" + [[ "$_sc19_local" != "$_sc_new" ]] && \ + MISMATCH="${MISMATCH} local(got=${_sc19_local} expected=${_sc_new})" + + if [[ -z "$MISMATCH" ]]; then + pass "Step 1: SetSubscriberCount ${_sc_new}" \ + "etcd=${_sc19_etcd_val} fastrg_local=${_sc19_local}" + else + fail "Step 1: SetSubscriberCount ${_sc_new}" "Mismatch:${MISMATCH}" + fi + + # ------------------------------------------------------------------ + # Step 2 — Restore original subscriber count → verify etcd + local + # ------------------------------------------------------------------ + info "Step 2: Restoring subscriber count to ${_sc_orig}..." + _sc20_reply=$(fastrg_grpc set_subscriber_count "${_sc_orig}") + _sc20_status=$(printf '%s' "$_sc20_reply" | jq -r '.status // empty' 2>/dev/null || true) + + if [[ -z "$_sc20_status" ]]; then + fail "Step 2: Restore SubscriberCount ${_sc_orig}" \ + "gRPC returned no status — response: $(printf '%s' "$_sc20_reply")" + return + fi + + sleep 2 # allow etcd write + watcher callback + + # Verify etcd restored + _sc20_etcd=$(etcdctl_get_value "user_counts/${NODE_UUID}/" 2>/dev/null || true) + _sc20_etcd_val=$(printf '%s' "$_sc20_etcd" | jq -r '.subscriber_count // empty' 2>/dev/null || true) + + # Verify fastrg local restored + _sc20_sys=$(fastrg_grpc get_system_info) + _sc20_local=$(printf '%s' "$_sc20_sys" | jq -r '.num_users // 0' 2>/dev/null || echo 0) + + MISMATCH="" + [[ "$_sc20_etcd_val" != "$_sc_orig" ]] && \ + MISMATCH="${MISMATCH} etcd(got=${_sc20_etcd_val:-empty} expected=${_sc_orig})" + [[ "$_sc20_local" != "$_sc_orig" ]] && \ + MISMATCH="${MISMATCH} local(got=${_sc20_local} expected=${_sc_orig})" + + if [[ -z "$MISMATCH" ]]; then + pass "Step 2: Restore SubscriberCount ${_sc_orig}" \ + "etcd=${_sc20_etcd_val} fastrg_local=${_sc20_local}" + else + fail "Step 2: Restore SubscriberCount ${_sc_orig}" "Mismatch:${MISMATCH}" + fi +} diff --git a/e2e_test/phases/phase2_etcd_config_sync.sh b/e2e_test/phases/phase2_etcd_config_sync.sh new file mode 100644 index 0000000..dfdcd89 --- /dev/null +++ b/e2e_test/phases/phase2_etcd_config_sync.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# --------------------------------------------------------------------------- +# Phase 2 — etcd Config Sync +# --------------------------------------------------------------------------- +phase2_etcd_config_sync() { + bold "═══════════════════════════════════════════════════════" + bold " Phase 2 — etcd Config Sync (Steps 3–4)" + bold "═══════════════════════════════════════════════════════" + + # ------------------------------------------------------------------ + # Step 3 — etcd key exists for this subscriber + # ------------------------------------------------------------------ + info "Step 3: Checking etcd HSI config key exists for USER_ID=${USER_ID}..." + HSI_JSON=$(etcdctl_get_value "configs/${NODE_UUID}/hsi/${USER_ID}" 2>/dev/null || true) + if [[ -z "$HSI_JSON" ]]; then + fail "Step 3: etcd HSI key" "Key configs/${NODE_UUID}/hsi/${USER_ID} not found or empty" + warn "Skipping Steps 2a-2d (no etcd data to compare against)" + skip "Step 4a: PPPoE config match" "No etcd data" + skip "Step 4b: DHCP config match" "No etcd data" + skip "Step 4c: Port-mapping match" "No etcd data" + skip "Step 4d: DNS static match" "No etcd data" + return + fi + pass "Step 3: etcd HSI key" "configs/${NODE_UUID}/hsi/${USER_ID} exists" + + # Parse etcd JSON fields (nested: .config.* and .metadata.*) + ETCD_ACCOUNT=$(printf '%s' "$HSI_JSON" | jq -r '.config.account_name // empty') + ETCD_VLAN=$(printf '%s' "$HSI_JSON" | jq -r '.config.vlan_id // empty') + ETCD_POOL=$(printf '%s' "$HSI_JSON" | jq -r '.config.dhcp_addr_pool // empty') + ETCD_SUBNET=$(printf '%s' "$HSI_JSON" | jq -r '.config.dhcp_subnet // empty') + ETCD_GATEWAY=$(printf '%s' "$HSI_JSON" | jq -r '.config.dhcp_gateway // empty') + ETCD_DNS_PRI=$(printf '%s' "$HSI_JSON" | jq -r '.config.dns_primary // empty') + ETCD_DNS_SEC=$(printf '%s' "$HSI_JSON" | jq -r '.config.dns_secondary // empty') + + info " etcd account_name : ${ETCD_ACCOUNT}" + info " etcd vlan_id : ${ETCD_VLAN}" + info " etcd dhcp_pool : ${ETCD_POOL}" + info " etcd dhcp_subnet : ${ETCD_SUBNET}" + info " etcd dhcp_gateway : ${ETCD_GATEWAY}" + + # ------------------------------------------------------------------ + # Step 4a — PPPoE config loaded into fastrg + # ------------------------------------------------------------------ + info "Step 4a: Comparing PPPoE config (gRPC GetFastrgHsiInfo vs etcd)..." + HSI_GRPC=$(fastrg_grpc get_hsi_info) + HSI_USER=$(printf '%s' "$HSI_GRPC" | jq -r ".hsi_infos[] | select(.user_id == ${USER_ID})" 2>/dev/null || true) + + if [[ -z "$HSI_USER" ]]; then + fail "Step 4a: PPPoE config match" "User ID ${USER_ID} not found in gRPC GetFastrgHsiInfo response" + else + CLI_ACCOUNT=$(printf '%s' "$HSI_USER" | jq -r '.account // empty') + CLI_VLAN=$(printf '%s' "$HSI_USER" | jq -r '.vlan_id // empty') + + MISMATCH="" + [[ "$CLI_ACCOUNT" != "$ETCD_ACCOUNT" ]] && MISMATCH="${MISMATCH} account(grpc=${CLI_ACCOUNT} etcd=${ETCD_ACCOUNT})" + [[ "$CLI_VLAN" != "$ETCD_VLAN" ]] && MISMATCH="${MISMATCH} vlan(grpc=${CLI_VLAN} etcd=${ETCD_VLAN})" + + if [[ -z "$MISMATCH" ]]; then + pass "Step 4a: PPPoE config match" "account=${CLI_ACCOUNT} vlan=${CLI_VLAN}" + else + fail "Step 4a: PPPoE config match" "Mismatch:${MISMATCH}" + fi + fi + + # ------------------------------------------------------------------ + # Step 4b — DHCP config + # ------------------------------------------------------------------ + info "Step 4b: Comparing DHCP config (gRPC GetFastrgDhcpInfo vs etcd)..." + DHCP_GRPC=$(fastrg_grpc get_dhcp_info) + DHCP_USER=$(printf '%s' "$DHCP_GRPC" | jq -r ".dhcp_infos[] | select(.user_id == ${USER_ID})" 2>/dev/null || true) + + if [[ -z "$DHCP_USER" ]]; then + fail "Step 4b: DHCP config match" "User ID ${USER_ID} not found in gRPC GetFastrgDhcpInfo response" + else + CLI_POOL=$(printf '%s' "$DHCP_USER" | jq -r '.ip_range // empty') + CLI_SUBNET=$(printf '%s' "$DHCP_USER" | jq -r '.subnet_mask // empty') + CLI_GW=$(printf '%s' "$DHCP_USER" | jq -r '.gateway // empty') + + # Normalize ip_range: gRPC returns "start - end", etcd uses "start~end" + # Collapse to "start~end" for comparison + CLI_POOL_NORM=$(printf '%s' "$CLI_POOL" | tr -d ' ' | tr '-' '~' | sed 's/~~/~/') + ETCD_POOL_NORM=$(printf '%s' "$ETCD_POOL" | tr -d ' ' | tr '-' '~' | sed 's/~~/~/') + + MISMATCH="" + [[ "$CLI_POOL_NORM" != "$ETCD_POOL_NORM" ]] && MISMATCH="${MISMATCH} ip_range(grpc=${CLI_POOL} etcd=${ETCD_POOL})" + [[ "$CLI_SUBNET" != "$ETCD_SUBNET" ]] && MISMATCH="${MISMATCH} subnet(grpc=${CLI_SUBNET} etcd=${ETCD_SUBNET})" + [[ "$CLI_GW" != "$ETCD_GATEWAY" ]] && MISMATCH="${MISMATCH} gateway(grpc=${CLI_GW} etcd=${ETCD_GATEWAY})" + + if [[ -z "$MISMATCH" ]]; then + pass "Step 4b: DHCP config match" "pool=${CLI_POOL} subnet=${CLI_SUBNET} gw=${CLI_GW}" + else + fail "Step 4b: DHCP config match" "Mismatch:${MISMATCH}" + fi + fi + + # ------------------------------------------------------------------ + # Step 4c — Port-mapping config + # ------------------------------------------------------------------ + info "Step 4c: Comparing port-mapping config (gRPC GetPortFwdInfo vs etcd)..." + PM_COUNT=$(printf '%s' "$HSI_JSON" | jq -r '(.config["port-mapping"] // []) | length') + + if [[ "$PM_COUNT" -eq 0 ]]; then + pass "Step 4c: Port-mapping match" "No port-mappings in etcd (nothing to verify)" + else + PORTFWD_GRPC=$(fastrg_grpc get_port_fwd_info "${USER_ID}") + PM_FAIL=0 + PM_DETAIL="" + + i=0 + while [[ $i -lt $PM_COUNT ]]; do + E_EPORT=$(printf '%s' "$HSI_JSON" | jq -r ".config[\"port-mapping\"][$i].eport") + E_DIP=$(printf '%s' "$HSI_JSON" | jq -r ".config[\"port-mapping\"][$i].dip") + E_DPORT=$(printf '%s' "$HSI_JSON" | jq -r ".config[\"port-mapping\"][$i].dport") + + # Match eport in gRPC response entries + ENTRY=$(printf '%s' "$PORTFWD_GRPC" | \ + jq -r ".entries[] | select(.eport == (\"${E_EPORT}\" | tonumber))" 2>/dev/null || true) + if [[ -n "$ENTRY" ]]; then + PM_DETAIL="${PM_DETAIL} eport=${E_EPORT}:OK" + else + PM_DETAIL="${PM_DETAIL} eport=${E_EPORT}:MISSING(dip=${E_DIP} dport=${E_DPORT})" + PM_FAIL=1 + fi + i=$((i + 1)) + done + + if [[ $PM_FAIL -eq 0 ]]; then + pass "Step 4c: Port-mapping match" "${PM_DETAIL}" + else + fail "Step 4c: Port-mapping match" "${PM_DETAIL}" + fi + fi + + # ------------------------------------------------------------------ + # Step 4d — DNS static records + # ------------------------------------------------------------------ + info "Step 4d: Comparing DNS static records (gRPC GetDnsStaticRecords vs etcd keys)..." + DNS_KEYS=$(etcdctl_get_value "--prefix configs/${NODE_UUID}/${USER_ID}/dns/" 2>/dev/null || true) + DNS_DOMAINS=$(printf '%s' "$DNS_KEYS" | jq -r '.domain // empty' 2>/dev/null || true) + + if [[ -z "$DNS_DOMAINS" ]]; then + # Try raw key listing (key is the domain, value is JSON) + DNS_KEYS_RAW=$(ssh_node "ETCDCTL_API=3 etcdctl --endpoints=${ETCD_ENDPOINT} get --prefix --keys-only configs/${NODE_UUID}/${USER_ID}/dns/" 2>/dev/null || true) + DNS_DOMAINS=$(printf '%s' "$DNS_KEYS_RAW" | awk -F'/' '{print $NF}' | grep -v '^$' || true) + fi + + if [[ -z "$DNS_DOMAINS" ]]; then + pass "Step 4d: DNS static match" "No DNS static keys in etcd (nothing to verify)" + else + DNS_GRPC=$(fastrg_grpc get_dns_static "${USER_ID}") + DNS_FAIL=0 + DNS_DETAIL="" + + while IFS= read -r domain; do + [[ -z "$domain" ]] && continue + MATCH=$(printf '%s' "$DNS_GRPC" | \ + jq -r ".entries[] | select(.domain == \"${domain}\") | .domain" 2>/dev/null || true) + if [[ -n "$MATCH" ]]; then + DNS_DETAIL="${DNS_DETAIL} ${domain}:OK" + else + DNS_DETAIL="${DNS_DETAIL} ${domain}:MISSING" + DNS_FAIL=1 + fi + done <<< "$DNS_DOMAINS" + + if [[ $DNS_FAIL -eq 0 ]]; then + pass "Step 4d: DNS static match" "${DNS_DETAIL}" + else + fail "Step 4d: DNS static match" "${DNS_DETAIL}" + fi + fi +} diff --git a/e2e_test/phases/phase3_5_enable_status.sh b/e2e_test/phases/phase3_5_enable_status.sh new file mode 100644 index 0000000..3cca9a8 --- /dev/null +++ b/e2e_test/phases/phase3_5_enable_status.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# --------------------------------------------------------------------------- +# Phase 3.5 — PPPoE Enable Status +# --------------------------------------------------------------------------- +phase3_5_enable_status() { + bold "═══════════════════════════════════════════════════════" + bold " Phase 3.5 — PPPoE Enable Status (Steps 7-9)" + bold "═══════════════════════════════════════════════════════" + + # ------------------------------------------------------------------ + # Step 7 — Read current enableStatus; disconnect HSI if enabled + # ------------------------------------------------------------------ + info "Step 7: Reading current PPPoE enableStatus for USER_ID=${USER_ID}..." + _35_hsi_json=$(etcdctl_get_value "configs/${NODE_UUID}/hsi/${USER_ID}" 2>/dev/null || true) + + if [[ -z "$_35_hsi_json" ]]; then + fail "Step 7: Read PPPoE enableStatus" \ + "Cannot read configs/${NODE_UUID}/hsi/${USER_ID} from etcd" + skip "Step 8: Reconnect HSI" "Cannot read current enableStatus" + skip "Step 9: Verify PPPoE enableStatus" "Cannot read current enableStatus" + return + fi + + _35_cur_status=$(printf '%s' "$_35_hsi_json" | jq -r '.metadata.enableStatus // empty') + info " Current enableStatus: ${_35_cur_status:-}" + + if [[ "$_35_cur_status" == "enabled" ]]; then + info "Step 7: PPPoE is currently enabled — calling DisconnectHsi to reset connection..." + fastrg_grpc disconnect_hsi "${USER_ID}" >/dev/null 2>&1 || true + pass "Step 7: DisconnectHsi USER_ID=${USER_ID}" "Disconnect issued" + sleep 2 + else + pass "Step 7: Read PPPoE enableStatus" \ + "enableStatus=${_35_cur_status:-} — no disconnect needed" + fi + + # ------------------------------------------------------------------ + # Step 8 — Reconnect HSI; wait for PPPoE session to come up + # ------------------------------------------------------------------ + info "Step 8: ConnectHsi USER_ID=${USER_ID} — re-establishing PPPoE session..." + fastrg_grpc connect_hsi "${USER_ID}" >/dev/null 2>&1 || true + + # Wait up to 30s for PPPoE session to move out of "End phase" + _35_ppp_ok=0 + _35_phase="" + for _35_i in $(seq 1 6); do + sleep 5 + _35_hsi_now=$(fastrg_grpc get_hsi_info 2>/dev/null || true) + _35_phase=$(printf '%s' "$_35_hsi_now" | \ + jq -r ".hsi_infos[] | select(.user_id == ${USER_ID}) | .status" \ + 2>/dev/null || true) + if [[ -n "$_35_phase" ]] && [[ "$_35_phase" == "Data phase" ]]; then + _35_ppp_ok=1 + break + fi + info " waiting for PPPoE session... (${_35_i}x5s, ppp_phase='${_35_phase}')" + done + + if [[ $_35_ppp_ok -eq 1 ]]; then + pass "Step 8: ConnectHsi USER_ID=${USER_ID}" \ + "PPPoE session re-established (ppp_phase='${_35_phase}')" + else + fail "Step 8: ConnectHsi USER_ID=${USER_ID}" \ + "PPPoE session did not come up within 30s (last ppp_phase='${_35_phase:-}')" + fi + + # ------------------------------------------------------------------ + # Step 9 — Verify etcd metadata.enableStatus = "enabled" + # ------------------------------------------------------------------ + info "Step 9: Checking etcd metadata.enableStatus for USER_ID=${USER_ID}..." + HSI_JSON=$(etcdctl_get_value "configs/${NODE_UUID}/hsi/${USER_ID}" 2>/dev/null || true) + + if [[ -z "$HSI_JSON" ]]; then + fail "Step 9: PPPoE enableStatus" "Cannot re-read configs/${NODE_UUID}/hsi/${USER_ID} from etcd" + return + fi + + ENABLE_STATUS=$(printf '%s' "$HSI_JSON" | jq -r '.metadata.enableStatus // empty') + + if [[ "$ENABLE_STATUS" == "enabled" ]]; then + pass "Step 9: PPPoE enableStatus" "metadata.enableStatus = \"enabled\"" + else + fail "Step 9: PPPoE enableStatus" "Expected \"enabled\", got \"${ENABLE_STATUS}\"" + fi +} diff --git a/e2e_test/phases/phase3_dhcp_and_count.sh b/e2e_test/phases/phase3_dhcp_and_count.sh new file mode 100644 index 0000000..4a59b8e --- /dev/null +++ b/e2e_test/phases/phase3_dhcp_and_count.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# --------------------------------------------------------------------------- +# Phase 3 — DHCP Assignment + Subscriber Count +# --------------------------------------------------------------------------- +phase3_dhcp_and_count() { + bold "═══════════════════════════════════════════════════════" + bold " Phase 3 — DHCP Assignment & Subscriber Count (Steps 5, 6)" + bold "═══════════════════════════════════════════════════════" + + # ------------------------------------------------------------------ + # Step 5 — DHCP In-use IPs (LAN device got an IP) + # ------------------------------------------------------------------ + info "Step 5: Checking DHCP address assignment for USER_ID=${USER_ID}..." + DHCP_GRPC3=$(fastrg_grpc get_dhcp_info) + DHCP_USER3=$(printf '%s' "$DHCP_GRPC3" | jq -r ".dhcp_infos[] | select(.user_id == ${USER_ID})" 2>/dev/null || true) + + if [[ -z "$DHCP_USER3" ]]; then + fail "Step 5: DHCP address assigned" "User ID ${USER_ID} not found in gRPC GetFastrgDhcpInfo" + else + INUSE_IPS=$(printf '%s' "$DHCP_USER3" | jq -r '.inuse_ips | if length > 0 then join(", ") else empty end' 2>/dev/null || true) + if [[ -n "$INUSE_IPS" ]]; then + pass "Step 5: DHCP address assigned" "In-use IPs: ${INUSE_IPS}" + else + fail "Step 5: DHCP address assigned" "inuse_ips is empty — no LAN device has obtained an IP" + fi + fi + + # ------------------------------------------------------------------ + # Step 6 — subscriber_count in etcd vs fastrg gRPC system info + # ------------------------------------------------------------------ + info "Step 6: Comparing subscriber count (etcd user_counts vs gRPC GetFastrgSystemInfo)..." + UC_JSON=$(etcdctl_get_value "user_counts/${NODE_UUID}/" 2>/dev/null || true) + + if [[ -z "$UC_JSON" ]]; then + fail "Step 6: Subscriber count match" "Key user_counts/${NODE_UUID}/ not found in etcd" + else + # subscriber_count is stored as a STRING in etcd + ETCD_COUNT=$(printf '%s' "$UC_JSON" | jq -r '.subscriber_count // empty' | tr -d '[:space:]') + ETCD_COUNT_INT=$(( ${ETCD_COUNT:-0} + 0 )) + + SYS_JSON=$(fastrg_grpc get_system_info) + CLI_COUNT_INT=$(printf '%s' "$SYS_JSON" | jq -r '.num_users // 0' | tr -d '[:space:]') + CLI_COUNT_INT=$(( ${CLI_COUNT_INT:-0} + 0 )) + + if [[ $ETCD_COUNT_INT -eq $CLI_COUNT_INT ]]; then + pass "Step 6: Subscriber count match" "etcd=${ETCD_COUNT_INT} == fastrg=${CLI_COUNT_INT}" + else + fail "Step 6: Subscriber count match" "etcd=${ETCD_COUNT_INT} != fastrg=${CLI_COUNT_INT}" + fi + fi + + # ------------------------------------------------------------------ + # 10-second wait for PPPoE session establishment + # ------------------------------------------------------------------ + printf "\n" + info "Waiting 10 seconds for PPPoE session establishment..." + sleep 10 + info "Wait complete. Continuing test..." + printf "\n" +} diff --git a/e2e_test/phases/phase4_lan_to_wan.sh b/e2e_test/phases/phase4_lan_to_wan.sh new file mode 100644 index 0000000..ee88e1a --- /dev/null +++ b/e2e_test/phases/phase4_lan_to_wan.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# --------------------------------------------------------------------------- +# Phase 4 — LAN→WAN Traffic Tests +# --------------------------------------------------------------------------- +phase4_lan_to_wan() { + bold "═══════════════════════════════════════════════════════" + bold " Phase 4 — LAN→WAN Traffic (Steps 10-12)" + bold "═══════════════════════════════════════════════════════" + + # ------------------------------------------------------------------ + # Step 10 — Ping + # ------------------------------------------------------------------ + info "Step 10: Ping ${WAN_IP} from LAN host ${LAN_HOST}..." + PING_OUT=$(ssh_lan "ping -c 4 -W 3 ${WAN_IP} 2>&1" || true) + if printf '%s' "$PING_OUT" | grep -qE "0% packet loss|0\.0% packet loss"; then + pass "Step 10: Ping LAN→WAN" "${WAN_IP} reachable, 0% packet loss" + else + LOSS=$(printf '%s' "$PING_OUT" | grep -oE '[0-9]+(\.[0-9]+)?% packet loss' | head -1 || echo "no response") + fail "Step 10: Ping LAN→WAN" "${WAN_IP} - ${LOSS}" + fi + + # ------------------------------------------------------------------ + # Step 11 — iperf3 + # ------------------------------------------------------------------ + info "Step 11: iperf3 test (LAN→WAN, port 55688, cport 47792)..." + # Start iperf3 server on WAN host (daemon mode) + ssh_wan "iperf3 -s -B ${WAN_IP} -p 55688 -D --forceflush >/dev/null 2>&1 || true" || true + sleep 2 + + IPERF_OUT=$(ssh_lan "iperf3 -c ${WAN_IP} -p 55688 --cport 47792 -t 5 -J 2>&1" || true) + # Cleanup iperf3 server + ssh_wan "pkill -f 'iperf3 -s' 2>/dev/null || true" || true + + if [[ -z "$IPERF_OUT" ]]; then + fail "Step 11: iperf3 LAN→WAN" "No output from iperf3 client" + else + BPS=$(printf '%s' "$IPERF_OUT" | jq -r '.end.sum_received.bits_per_second // 0' 2>/dev/null || echo "0") + BPS_INT=$(printf '%.0f' "${BPS}" 2>/dev/null || echo "0") + if [[ $BPS_INT -gt 0 ]]; then + # Format as Mbps for readability + MBPS=$(awk "BEGIN {printf \"%.2f\", $BPS_INT / 1000000}") + pass "Step 11: iperf3 LAN→WAN" "Received ${MBPS} Mbps" + else + ERR=$(printf '%s' "$IPERF_OUT" | jq -r '.error // empty' 2>/dev/null || true) + fail "Step 11: iperf3 LAN→WAN" "bits_per_second=0${ERR:+; error: $ERR}" + fi + fi + + # ------------------------------------------------------------------ + # Step 12 — curl + # ------------------------------------------------------------------ + info "Step 12: curl http://www.google.com from LAN host ${LAN_HOST}..." + HTTP_CODE=$(ssh_lan "curl -s -o /dev/null -w '%{http_code}' --max-time 15 http://www.google.com 2>&1" || echo "000") + HTTP_CODE=$(printf '%s' "$HTTP_CODE" | tr -d "'" | tr -d '[:space:]') + + case "$HTTP_CODE" in + 200|301|302) + pass "Step 12: curl www.google.com" "HTTP ${HTTP_CODE}" + ;; + 000) + fail "Step 12: curl www.google.com" "Connection failed or timed out (HTTP 000)" + ;; + *) + fail "Step 12: curl www.google.com" "Unexpected HTTP status: ${HTTP_CODE}" + ;; + esac +} diff --git a/e2e_test/phases/phase5_dnat_test.sh b/e2e_test/phases/phase5_dnat_test.sh new file mode 100644 index 0000000..e6d043b --- /dev/null +++ b/e2e_test/phases/phase5_dnat_test.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# --------------------------------------------------------------------------- +# Phase 5 — WAN→LAN DNAT Test (scapy + netcat) +# --------------------------------------------------------------------------- +phase5_dnat_test() { + bold "═══════════════════════════════════════════════════════" + bold " Phase 5 — WAN→LAN DNAT (Step 13)" + bold "═══════════════════════════════════════════════════════" + + info "Step 13: WAN→LAN DNAT — scapy from WAN, nc listen on LAN..." + + # Get port-mapping eport/dport from etcd (already fetched in phase1 as HSI_JSON) + # Re-fetch in case phase1 was skipped or HSI_JSON is out of scope + _HSI_ETCD=$(etcdctl_get_value "configs/${NODE_UUID}/hsi/${USER_ID}" 2>/dev/null || true) + PM_COUNT=$(printf '%s' "$_HSI_ETCD" | jq -r '(.config["port-mapping"] // []) | length' 2>/dev/null || echo "0") + + if [[ "$PM_COUNT" -eq 0 ]]; then + skip "Step 13: WAN→LAN DNAT" "No port-mapping in etcd — cannot perform DNAT test" + return + fi + + # Use first port-mapping entry + DNAT_EPORT=$(printf '%s' "$_HSI_ETCD" | jq -r '.config["port-mapping"][0].eport') + DNAT_DIP=$(printf '%s' "$_HSI_ETCD" | jq -r '.config["port-mapping"][0].dip') + DNAT_DPORT=$(printf '%s' "$_HSI_ETCD" | jq -r '.config["port-mapping"][0].dport') + info " DNAT rule: WAN:${DNAT_EPORT} → LAN ${DNAT_DIP}:${DNAT_DPORT}" + + # Fetch PPPoE client WAN IP for this user from gRPC (ip_addr in HsiInfo) + _HSI_GRPC=$(fastrg_grpc get_hsi_info) + DNAT_PPP_IP=$(printf '%s' "$_HSI_GRPC" | \ + jq -r ".hsi_infos[] | select(.user_id == ${USER_ID}) | .ip_addr" 2>/dev/null || true) + if [[ -z "$DNAT_PPP_IP" ]]; then + fail "Step 13: WAN→LAN DNAT" "Cannot determine PPPoE client WAN IP for USER_ID=${USER_ID} from gRPC" + return + fi + info " PPPoE WAN IP (gRPC ip_addr): ${DNAT_PPP_IP}" + + # Start nc UDP listener on LAN host at the DNAT destination port (no root needed) + NC_OUT_FILE=$(mktemp /tmp/fastrg_nc_XXXXXX) + ssh_lan "timeout 10 nc -u -l -p ${DNAT_DPORT} 2>&1" \ + > "$NC_OUT_FILE" 2>&1 & + NC_PID=$! + + sleep 2 + + # Send UDP packet from WAN host using scapy to WAN IP:eport + SCAPY_CMD="python3 -c \"from scapy.all import Ether,IP,UDP,Raw,sendp; pkt=Ether(dst='74:4d:28:8d:00:2c',src='9c:69:b4:68:65:db')/IP(src='192.168.201.10',dst='${DNAT_PPP_IP}',ttl=64,id=0x4003)/UDP(sport=54321,dport=${DNAT_EPORT})/Raw(load=b'hello'); sendp(pkt, iface='ens6f3np3')\" 2>&1" + SCAPY_OUT=$(ssh_wan "$SCAPY_CMD" 2>&1 || true) + info " scapy output: ${SCAPY_OUT}" + + # Wait for nc to finish + wait $NC_PID 2>/dev/null || true + NC_OUT=$(cat "$NC_OUT_FILE") + rm -f "$NC_OUT_FILE" + + info " nc output: ${NC_OUT:-}" + + # nc prints nothing on success (just exits with the received data) or "hello" payload + # Also check exit: nc -l exits after receiving one packet (timeout 10 will exit even without data) + # We consider PASS if nc received data (payload "hello") or exited cleanly within timeout + if printf '%s' "$NC_OUT" | grep -q "hello"; then + pass "Step 13: WAN→LAN DNAT" "UDP payload 'hello' received on LAN ${DNAT_DIP}:${DNAT_DPORT}" + else + fail "Step 13: WAN→LAN DNAT" "scapy failed to send or nc did not receive on LAN ${DNAT_DIP}:${DNAT_DPORT}" + fi +} diff --git a/e2e_test/phases/phase6_dns_ping.sh b/e2e_test/phases/phase6_dns_ping.sh new file mode 100644 index 0000000..c0fd306 --- /dev/null +++ b/e2e_test/phases/phase6_dns_ping.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# --------------------------------------------------------------------------- +# Phase 6 — DNS Static Record + Reverse Ping +# --------------------------------------------------------------------------- +phase6_dns_ping() { + bold "═══════════════════════════════════════════════════════" + bold " Phase 6 — DNS Static + Reverse Ping (Step 14)" + bold "═══════════════════════════════════════════════════════" + + info "Step 14: Ping www.google.org from LAN host; expecting reply from ${WAN_IP}..." + PING_OUT=$(ssh_lan "ping -c 4 -W 5 www.google.org 2>&1" || true) + + info " ping output:" + printf '%s\n' "$PING_OUT" | while IFS= read -r line; do + printf " %s\n" "$line" + done + + if printf '%s' "$PING_OUT" | grep -q "from ${WAN_IP}"; then + pass "Step 14: DNS static + ping www.google.org" "Received ICMP reply from ${WAN_IP}" + else + # Check if it resolved but got a different IP (DNS not overridden) + if printf '%s' "$PING_OUT" | grep -qE "PING|bytes from"; then + REPLY_IP=$(printf '%s' "$PING_OUT" | grep -oE "from [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+" | head -1 | awk '{print $2}') + fail "Step 14: DNS static + ping www.google.org" "Got reply from ${REPLY_IP:-unknown}, expected ${WAN_IP} — DNS static record may not be configured" + else + fail "Step 14: DNS static + ping www.google.org" "No ICMP reply received" + fi + fi +} diff --git a/e2e_test/phases/phase7_user1_config_tests.sh b/e2e_test/phases/phase7_user1_config_tests.sh new file mode 100644 index 0000000..73fa5ff --- /dev/null +++ b/e2e_test/phases/phase7_user1_config_tests.sh @@ -0,0 +1,314 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# --------------------------------------------------------------------------- +# Phase 7 — User 1 gRPC→etcd→fastrg Config Tests (Steps 12–19) +# +# Creates a fresh HSI config for user 1, exercises all major gRPC +# configuration commands, verifies each produces the correct etcd state +# and that fastrg_node applies changes locally. All user-1 config is +# cleaned up at the end. User 2 traffic (steps 1-11) is not re-tested. +# --------------------------------------------------------------------------- + +# Helper: remove user 1 HSI config from etcd via gRPC (idempotent). +# Called at end of phase7 AND from cleanup_fastrg trap. +_cleanup_user1_config() { + [[ -z "${NODE_UUID:-}" ]] && return + local _chk + _chk=$(etcdctl_get_value "configs/${NODE_UUID}/hsi/1" 2>/dev/null || true) + if [[ -n "$_chk" ]]; then + info "Cleanup: removing user 1 config (RemoveConfig gRPC)..." + fastrg_grpc remove_config "1" >/dev/null 2>&1 || true + sleep 1 + _chk=$(etcdctl_get_value "configs/${NODE_UUID}/hsi/1" 2>/dev/null || true) + if [[ -z "$_chk" ]]; then + info "Cleanup: user 1 config removed from etcd." + else + warn "Cleanup: user 1 config still in etcd after RemoveConfig — manual cleanup may be needed." + fi + fi +} + +phase7_user1_config_tests() { + bold "═══════════════════════════════════════════════════════" + bold " Phase 7 — User 1 gRPC→etcd Config Tests (Steps 15-22)" + bold "═══════════════════════════════════════════════════════" + + # User 1 test parameters (must not overlap with user 2: VLAN 2, pool 192.168.3.x) + local U1=1 + local U1_VLAN=3 + local U1_ACCOUNT="test1" + local U1_PASSWORD="test1pass" + local U1_POOL_START="192.168.4.2" + local U1_POOL_END="192.168.4.10" + local U1_SUBNET="255.255.255.0" + local U1_GATEWAY="192.168.4.1" + local U1_DNS_DOMAIN="user1test.fastrg.local" + local U1_DNS_IP="10.1.0.1" + local U1_DNS_TTL=60 + + # ------------------------------------------------------------------ + # Step 15 — ApplyConfig user 1 → etcd key written correctly + # ------------------------------------------------------------------ + info "Step 15: ApplyConfig user ${U1} (VLAN=${U1_VLAN} account=${U1_ACCOUNT} pool=${U1_POOL_START}-${U1_POOL_END})..." + _apply_reply=$(fastrg_grpc apply_config \ + "${U1}" "${U1_VLAN}" "${U1_ACCOUNT}" "${U1_PASSWORD}" \ + "${U1_POOL_START}" "${U1_POOL_END}" "${U1_SUBNET}" "${U1_GATEWAY}") + _apply_status=$(printf '%s' "$_apply_reply" | jq -r '.status // empty' 2>/dev/null || true) + + if [[ -z "$_apply_status" ]]; then + fail "Step 15: ApplyConfig user ${U1}" \ + "gRPC ApplyConfig returned no status — response: $(printf '%s' "$_apply_reply")" + warn "Skipping Steps 12-18 (ApplyConfig failed)" + skip "Step 16: fastrg applies user 1 config" "ApplyConfig failed" + skip "Step 17: ConnectHsi user 1" "ApplyConfig failed" + skip "Step 18: DisconnectHsi user 1" "ApplyConfig failed" + skip "Step 19: DhcpServerStart user 1" "ApplyConfig failed" + skip "Step 20: DhcpServerStop user 1" "ApplyConfig failed" + skip "Step 21: AddDnsRecord user 1" "ApplyConfig failed" + skip "Step 22: RemoveDnsRecord user 1" "ApplyConfig failed" + return + fi + + sleep 1 # allow etcd write to propagate + _u1_etcd=$(etcdctl_get_value "configs/${NODE_UUID}/hsi/${U1}" 2>/dev/null || true) + if [[ -z "$_u1_etcd" ]]; then + fail "Step 15: ApplyConfig user ${U1}" \ + "etcd key configs/${NODE_UUID}/hsi/${U1} not found after ApplyConfig" + else + _e_account=$(printf '%s' "$_u1_etcd" | jq -r '.config.account_name // empty') + _e_vlan=$(printf '%s' "$_u1_etcd" | jq -r '.config.vlan_id // empty') + _e_pool=$(printf '%s' "$_u1_etcd" | jq -r '.config.dhcp_addr_pool // empty') + _e_enable=$(printf '%s' "$_u1_etcd" | jq -r '.metadata.enableStatus // empty') + _e_subnet=$(printf '%s' "$_u1_etcd" | jq -r '.config.dhcp_subnet // empty') + _e_gw=$(printf '%s' "$_u1_etcd" | jq -r '.config.dhcp_gateway // empty') + _expect_pool="${U1_POOL_START}-${U1_POOL_END}" + + MISMATCH="" + [[ "$_e_account" != "$U1_ACCOUNT" ]] && MISMATCH="${MISMATCH} account(etcd=${_e_account} expected=${U1_ACCOUNT})" + [[ "$_e_vlan" != "$U1_VLAN" ]] && MISMATCH="${MISMATCH} vlan(etcd=${_e_vlan} expected=${U1_VLAN})" + [[ "$_e_pool" != "$_expect_pool" ]] && MISMATCH="${MISMATCH} dhcp_pool(etcd=${_e_pool} expected=${_expect_pool})" + [[ "$_e_enable" != "disabled" ]] && MISMATCH="${MISMATCH} enableStatus(etcd=${_e_enable} expected=disabled)" + [[ "$_e_subnet" != "$U1_SUBNET" ]] && MISMATCH="${MISMATCH} subnet(etcd=${_e_subnet} expected=${U1_SUBNET})" + [[ "$_e_gw" != "$U1_GATEWAY" ]] && MISMATCH="${MISMATCH} gateway(etcd=${_e_gw} expected=${U1_GATEWAY})" + + if [[ -z "$MISMATCH" ]]; then + pass "Step 15: ApplyConfig user ${U1}" \ + "account=${_e_account} vlan=${_e_vlan} pool=${_e_pool} enableStatus=${_e_enable}" + else + fail "Step 15: ApplyConfig user ${U1}" "etcd mismatch:${MISMATCH}" + fi + fi + + # ------------------------------------------------------------------ + # Step 16 — fastrg watches etcd → applies user 1 config locally + # ------------------------------------------------------------------ + info "Step 16: Verifying fastrg applied user ${U1} config locally (GetFastrgHsiInfo + GetFastrgDhcpInfo)..." + sleep 1 # ensure etcd watcher has fired + _u1_hsi=$(fastrg_grpc get_hsi_info) + _u1_hsi_user=$(printf '%s' "$_u1_hsi" | \ + jq -r ".hsi_infos[] | select(.user_id == ${U1})" 2>/dev/null || true) + _u1_dhcp=$(fastrg_grpc get_dhcp_info) + _u1_dhcp_user=$(printf '%s' "$_u1_dhcp" | \ + jq -r ".dhcp_infos[] | select(.user_id == ${U1})" 2>/dev/null || true) + + if [[ -z "$_u1_hsi_user" ]] || [[ -z "$_u1_dhcp_user" ]]; then + fail "Step 16: fastrg applies user ${U1} config" \ + "User ${U1} not found in gRPC response — hsi:$( [[ -n "$_u1_hsi_user" ]] && echo ok || echo missing) dhcp:$( [[ -n "$_u1_dhcp_user" ]] && echo ok || echo missing)" + else + _g_vlan=$(printf '%s' "$_u1_hsi_user" | jq -r '.vlan_id // empty') + _g_range=$(printf '%s' "$_u1_dhcp_user" | jq -r '.ip_range // empty') + _g_gw=$(printf '%s' "$_u1_dhcp_user" | jq -r '.gateway // empty') + _g_subnet=$(printf '%s' "$_u1_dhcp_user" | jq -r '.subnet_mask // empty') + + # Normalize pool range for comparison (gRPC returns "start - end", etcd uses "start-end") + _g_range_norm=$(printf '%s' "$_g_range" | tr -d ' ' | tr '-' '~' | sed 's/~~/~/') + _e_pool_norm="${U1_POOL_START}~${U1_POOL_END}" + + MISMATCH="" + [[ "$_g_vlan" != "$U1_VLAN" ]] && MISMATCH="${MISMATCH} vlan(grpc=${_g_vlan} expected=${U1_VLAN})" + [[ "$_g_range_norm" != "$_e_pool_norm" ]] && MISMATCH="${MISMATCH} pool(grpc=${_g_range} expected=${U1_POOL_START}-${U1_POOL_END})" + [[ "$_g_gw" != "$U1_GATEWAY" ]] && MISMATCH="${MISMATCH} gateway(grpc=${_g_gw} expected=${U1_GATEWAY})" + [[ "$_g_subnet" != "$U1_SUBNET" ]] && MISMATCH="${MISMATCH} subnet(grpc=${_g_subnet} expected=${U1_SUBNET})" + + if [[ -z "$MISMATCH" ]]; then + pass "Step 16: fastrg applies user ${U1} config" \ + "vlan=${_g_vlan} pool=${_g_range} gw=${_g_gw}" + else + fail "Step 16: fastrg applies user ${U1} config" "Mismatch:${MISMATCH}" + fi + fi + + # ------------------------------------------------------------------ + # Step 17 — ConnectHsi user 1 → PPPoE phase starts + # PPPoE will fail (no server for user 1) — that is expected. + # We verify the command was processed by observing the phase change. + # ------------------------------------------------------------------ + info "Step 17: ConnectHsi user ${U1} → verify PPPoE phase changes from 'End phase'..." + _before_phase=$(fastrg_grpc get_hsi_info | \ + python3 -c "import sys,json; d=json.load(sys.stdin); \ + u=[h for h in d.get('hsi_infos',[]) if h.get('user_id')==${U1}]; \ + print(u[0].get('status','') if u else '')" 2>/dev/null || true) + + fastrg_grpc connect_hsi "${U1}" >/dev/null 2>&1 || true + + # Poll up to 5 s for PPPoE phase to change (PPPoE_CMD_ENABLE event processed) + _phase_changed=0 + _new_phase="" + for _ci in $(seq 1 5); do + sleep 1 + _new_phase=$(fastrg_grpc get_hsi_info | \ + python3 -c "import sys,json; d=json.load(sys.stdin); \ + u=[h for h in d.get('hsi_infos',[]) if h.get('user_id')==${U1}]; \ + print(u[0].get('status','') if u else '')" 2>/dev/null || true) + if [[ "$_new_phase" != "End phase" && "$_new_phase" != "not configured" ]]; then + _phase_changed=1 + break + fi + done + + if [[ $_phase_changed -eq 1 ]]; then + pass "Step 17: ConnectHsi user ${U1}" \ + "PPPoE phase: '${_before_phase}' → '${_new_phase}' (failure expected — no server)" + else + fail "Step 17: ConnectHsi user ${U1}" \ + "PPPoE phase did not change from '${_before_phase}' — ConnectHsi may not have been processed" + fi + + # ------------------------------------------------------------------ + # Step 18 — DisconnectHsi user 1 → PPPoE returns to End phase + # Called while PPPoE is still retrying PADI (within ~5 s of ConnectHsi). + # ------------------------------------------------------------------ + info "Step 18: DisconnectHsi user ${U1} → verify PPPoE returns to 'End phase'..." + fastrg_grpc disconnect_hsi "${U1}" >/dev/null 2>&1 || true + + _disc_ok=0 + _disc_phase="" + # Wait up to 15 s: DisconnectHsi is immediate; natural PPPoE timeout is ~10 s + for _di in $(seq 1 15); do + sleep 1 + _disc_phase=$(fastrg_grpc get_hsi_info | \ + python3 -c "import sys,json; d=json.load(sys.stdin); \ + u=[h for h in d.get('hsi_infos',[]) if h.get('user_id')==${U1}]; \ + print(u[0].get('status','') if u else '')" 2>/dev/null || true) + if [[ "$_disc_phase" == "End phase" ]]; then + _disc_ok=1 + break + fi + done + + if [[ $_disc_ok -eq 1 ]]; then + pass "Step 18: DisconnectHsi user ${U1}" "PPPoE returned to 'End phase'" + else + fail "Step 18: DisconnectHsi user ${U1}" \ + "PPPoE still in '${_disc_phase:-unknown}' after DisconnectHsi + 15 s" + fi + + # ------------------------------------------------------------------ + # Step 19 — DhcpServerStart user 1 → DHCP server on + # ------------------------------------------------------------------ + info "Step 19: DhcpServerStart user ${U1} → verify DHCP server starts..." + fastrg_grpc start_dhcp_server "${U1}" >/dev/null 2>&1 || true + sleep 1 + _dhcp15=$(fastrg_grpc get_dhcp_info | \ + python3 -c "import sys,json; d=json.load(sys.stdin); \ + u=[h for h in d.get('dhcp_infos',[]) if h.get('user_id')==${U1}]; \ + print(u[0].get('status','') if u else '')" 2>/dev/null || true) + + if [[ "$_dhcp15" == "DHCP server is on" ]]; then + pass "Step 19: DhcpServerStart user ${U1}" "status='${_dhcp15}'" + else + fail "Step 19: DhcpServerStart user ${U1}" \ + "expected 'DHCP server is on', got '${_dhcp15:-empty}'" + fi + + # ------------------------------------------------------------------ + # Step 20 — DhcpServerStop user 1 → DHCP server off + # ------------------------------------------------------------------ + info "Step 20: DhcpServerStop user ${U1} → verify DHCP server stops..." + fastrg_grpc stop_dhcp_server "${U1}" >/dev/null 2>&1 || true + sleep 1 + _dhcp16=$(fastrg_grpc get_dhcp_info | \ + python3 -c "import sys,json; d=json.load(sys.stdin); \ + u=[h for h in d.get('dhcp_infos',[]) if h.get('user_id')==${U1}]; \ + print(u[0].get('status','') if u else '')" 2>/dev/null || true) + + if [[ "$_dhcp16" != "DHCP server is on" ]]; then + pass "Step 20: DhcpServerStop user ${U1}" "status='${_dhcp16:-off}'" + else + fail "Step 20: DhcpServerStop user ${U1}" \ + "expected DHCP off, got '${_dhcp16}'" + fi + + # ------------------------------------------------------------------ + # Step 21 — AddDnsRecord user 1 → etcd write + fastrg loads locally + # ------------------------------------------------------------------ + info "Step 21: AddDnsRecord user ${U1} domain=${U1_DNS_DOMAIN} ip=${U1_DNS_IP} ttl=${U1_DNS_TTL}..." + _dns_add_reply=$(fastrg_grpc add_dns_record "${U1}" "${U1_DNS_DOMAIN}" "${U1_DNS_IP}" "${U1_DNS_TTL}") + _dns_add_status=$(printf '%s' "$_dns_add_reply" | jq -r '.status // empty' 2>/dev/null || true) + + if [[ -z "$_dns_add_status" ]]; then + fail "Step 21: AddDnsRecord user ${U1}" \ + "gRPC AddDnsRecord returned no status — response: $(printf '%s' "$_dns_add_reply")" + skip "Step 22: RemoveDnsRecord user 1" "AddDnsRecord failed" + else + sleep 1 + _dns17_etcd=$(etcdctl_get_value \ + "configs/${NODE_UUID}/${U1}/dns/${U1_DNS_DOMAIN}" 2>/dev/null || true) + _dns17_grpc=$(fastrg_grpc get_dns_static "${U1}") + _dns17_match=$(printf '%s' "$_dns17_grpc" | \ + jq -r ".entries[] | select(.domain == \"${U1_DNS_DOMAIN}\") | .domain" 2>/dev/null || true) + + MISMATCH="" + if [[ -z "$_dns17_etcd" ]]; then + MISMATCH="${MISMATCH} etcd-key-missing" + else + _e17_ip=$(printf '%s' "$_dns17_etcd" | jq -r '.ip // empty') + _e17_ttl=$(printf '%s' "$_dns17_etcd" | jq -r '.ttl // empty') + [[ "$_e17_ip" != "$U1_DNS_IP" ]] && MISMATCH="${MISMATCH} ip(etcd=${_e17_ip} expected=${U1_DNS_IP})" + [[ "$_e17_ttl" != "$U1_DNS_TTL" ]] && MISMATCH="${MISMATCH} ttl(etcd=${_e17_ttl} expected=${U1_DNS_TTL})" + fi + [[ -z "$_dns17_match" ]] && MISMATCH="${MISMATCH} grpc-record-missing" + + if [[ -z "$MISMATCH" ]]; then + pass "Step 21: AddDnsRecord user ${U1}" \ + "etcd ip=${_e17_ip} ttl=${_e17_ttl} grpc=ok" + else + fail "Step 21: AddDnsRecord user ${U1}" "Mismatch:${MISMATCH}" + fi + + # ------------------------------------------------------------------ + # Step 22 — RemoveDnsRecord user 1 → etcd deleted + fastrg removes + # ------------------------------------------------------------------ + info "Step 22: RemoveDnsRecord user ${U1} domain=${U1_DNS_DOMAIN}..." + _dns_del_reply=$(fastrg_grpc remove_dns_record "${U1}" "${U1_DNS_DOMAIN}") + _dns_del_status=$(printf '%s' "$_dns_del_reply" | jq -r '.status // empty' 2>/dev/null || true) + + if [[ -z "$_dns_del_status" ]]; then + fail "Step 22: RemoveDnsRecord user ${U1}" \ + "gRPC RemoveDnsRecord returned no status — response: $(printf '%s' "$_dns_del_reply")" + else + sleep 1 + _dns18_gone=$(etcdctl_get_value \ + "configs/${NODE_UUID}/${U1}/dns/${U1_DNS_DOMAIN}" 2>/dev/null || true) + _dns18_grpc=$(fastrg_grpc get_dns_static "${U1}") + _dns18_still=$(printf '%s' "$_dns18_grpc" | \ + jq -r ".entries[] | select(.domain == \"${U1_DNS_DOMAIN}\") | .domain" 2>/dev/null || true) + + MISMATCH="" + [[ -n "$_dns18_gone" ]] && MISMATCH="${MISMATCH} etcd-key-still-present" + [[ -n "$_dns18_still" ]] && MISMATCH="${MISMATCH} grpc-record-still-present" + + if [[ -z "$MISMATCH" ]]; then + pass "Step 22: RemoveDnsRecord user ${U1}" \ + "etcd key deleted, fastrg record removed" + else + fail "Step 22: RemoveDnsRecord user ${U1}" "Mismatch:${MISMATCH}" + fi + fi + fi + + # ------------------------------------------------------------------ + # Cleanup — remove user 1 config from etcd + # ------------------------------------------------------------------ + _cleanup_user1_config +} diff --git a/e2e_test/phases/phase8_summary.sh b/e2e_test/phases/phase8_summary.sh new file mode 100644 index 0000000..b94e0ee --- /dev/null +++ b/e2e_test/phases/phase8_summary.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# --------------------------------------------------------------------------- +# Phase 8 — Summary +# --------------------------------------------------------------------------- +phase8_summary() { + local total=${#STEP_NAMES[@]} + local pass_count=0 fail_count=0 skip_count=0 + + printf "\n" + bold "═══════════════════════════════════════════════════════" + bold " Test Results Summary" + bold "═══════════════════════════════════════════════════════" + printf "\n" + + i=0 + while [[ $i -lt $total ]]; do + name="${STEP_NAMES[$i]}" + result="${STEP_RESULTS[$i]}" + detail="${STEP_DETAILS[$i]}" + + case "$result" in + PASS) + printf " ${GREEN}✔ PASS${NC} %-40s %s\n" "$name" "$detail" + pass_count=$((pass_count + 1)) + ;; + FAIL) + printf " ${RED}✘ FAIL${NC} %-40s %s\n" "$name" "$detail" + fail_count=$((fail_count + 1)) + ;; + SKIP) + printf " ${YELLOW}– SKIP${NC} %-40s %s\n" "$name" "$detail" + skip_count=$((skip_count + 1)) + ;; + esac + i=$((i + 1)) + done + + printf "\n" + printf " Total: %d " "$total" + printf "${GREEN}Pass: %d${NC} " "$pass_count" + printf "${RED}Fail: %d${NC} " "$fail_count" + printf "${YELLOW}Skip: %d${NC}\n" "$skip_count" + printf "\n" + + if [[ $fail_count -gt 0 ]]; then + bold " RESULT: ${RED}FAILED${NC} (${fail_count} step(s) failed)" + return 1 + else + bold " RESULT: ${GREEN}ALL PASSED${NC}" + return 0 + fi +} diff --git a/e2e_test/restore_etcd_config.sh b/e2e_test/restore_etcd_config.sh new file mode 100644 index 0000000..48c8a1b --- /dev/null +++ b/e2e_test/restore_etcd_config.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# ============================================================================= +# restore_etcd_config.sh — Restore standard E2E test etcd config for USER_ID=2 +# +# Run this script DIRECTLY on the FastRG node (192.168.10.201) when etcd +# config keys are missing and you need to restore them before running the +# E2E test suite. +# +# Usage: +# bash restore_etcd_config.sh [--dry-run] [--force] +# +# Options: +# --dry-run Print the keys/values that would be written without writing +# --force Overwrite keys even if they already exist (default: skip) +# ============================================================================= + +set -euo pipefail + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +info() { printf "${CYAN}[INFO]${NC} %s\n" "$*"; } +warn() { printf "${YELLOW}[WARN]${NC} %s\n" "$*"; } +ok() { printf "${GREEN}[OK]${NC} %s\n" "$*"; } +error() { printf "${RED}[ERROR]${NC} %s\n" "$*" >&2; } + +DRY_RUN=0 +FORCE=0 +for _a in "$@"; do + case "$_a" in + --dry-run) DRY_RUN=1 ;; + --force) FORCE=1 ;; + *) error "Unknown option: $_a"; exit 1 ;; + esac +done + +# --------------------------------------------------------------------------- +# Read NODE_UUID +# --------------------------------------------------------------------------- +if [[ ! -f /etc/fastrg/node_uuid ]]; then + error "/etc/fastrg/node_uuid not found — run this script on the FastRG node" + exit 1 +fi +NODE_UUID=$(tr -d '[:space:]' < /etc/fastrg/node_uuid) +if [[ -z "$NODE_UUID" ]]; then + error "NODE_UUID is empty" + exit 1 +fi +info "NODE_UUID: ${NODE_UUID}" + +# --------------------------------------------------------------------------- +# Read ETCD_ENDPOINT from /etc/fastrg/config.cfg +# --------------------------------------------------------------------------- +if [[ ! -f /etc/fastrg/config.cfg ]]; then + error "/etc/fastrg/config.cfg not found" + exit 1 +fi +ETCD_ENDPOINT=$(awk -F'"' '/EtcdEndpoints/{print $2}' /etc/fastrg/config.cfg) +if [[ -z "$ETCD_ENDPOINT" ]]; then + error "Cannot parse EtcdEndpoints from /etc/fastrg/config.cfg" + exit 1 +fi +info "ETCD_ENDPOINT: ${ETCD_ENDPOINT}" + +# --------------------------------------------------------------------------- +# Verify etcdctl is available +# --------------------------------------------------------------------------- +if ! command -v etcdctl >/dev/null 2>&1; then + error "etcdctl not found in PATH" + exit 1 +fi + +etcdput() { + local key="$1" val="$2" + local existing + existing=$(ETCDCTL_API=3 etcdctl --endpoints="${ETCD_ENDPOINT}" \ + get --print-value-only "${key}" 2>/dev/null || true) + + if [[ -n "$existing" ]] && [[ "$FORCE" -eq 0 ]]; then + warn "SKIP (already exists): ${key}" + return + fi + + if [[ "$DRY_RUN" -eq 1 ]]; then + info "DRY-RUN put: ${key}" + printf " value: %s\n" "$val" + return + fi + + ETCDCTL_API=3 etcdctl --endpoints="${ETCD_ENDPOINT}" put "${key}" "${val}" >/dev/null + ok "Written: ${key}" +} + +# --------------------------------------------------------------------------- +# Current timestamp (RFC 3339 / UTC) +# --------------------------------------------------------------------------- +NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +# --------------------------------------------------------------------------- +# KV 1 — DNS static record for USER_ID=2 +# --------------------------------------------------------------------------- +KEY_DNS="configs/${NODE_UUID}/2/dns/www.google.org" +VAL_DNS='{"domain":"www.google.org","ip":"192.168.201.10","ttl":30}' + +# --------------------------------------------------------------------------- +# KV 2 — HSI config for USER_ID=2 +# --------------------------------------------------------------------------- +KEY_HSI="configs/${NODE_UUID}/hsi/2" +VAL_HSI=$(printf \ + '{"config":{"account_name":"admin","dhcp_addr_pool":"192.168.3.2-192.168.3.10","dhcp_gateway":"192.168.3.1","dhcp_subnet":"255.255.255.0","password":"admin","port-mapping":[{"dip":"192.168.3.2","dport":"8081","eport":"12345","index":"0"}],"user_id":"2","vlan_id":"2"},"metadata":{"enableStatus":"enabled","node":"%s","resourceVersion":"1","updatedAt":"%s","updatedBy":"restore_etcd_config"}}' \ + "${NODE_UUID}" "${NOW}") + +# --------------------------------------------------------------------------- +# KV 3 — Subscriber count +# --------------------------------------------------------------------------- +KEY_COUNTS="user_counts/${NODE_UUID}/" +VAL_COUNTS=$(printf \ + '{"metadata":{"node":"%s","resourceVersion":"","updatedAt":"%s","updatedBy":"restore_etcd_config"},"subscriber_count":"2"}' \ + "${NODE_UUID}" "${NOW}") + +# --------------------------------------------------------------------------- +# Write +# --------------------------------------------------------------------------- +printf "\n" +info "Restoring ${DRY_RUN:+[DRY-RUN] }etcd config (FORCE=${FORCE})..." +printf "\n" + +etcdput "${KEY_DNS}" "${VAL_DNS}" +etcdput "${KEY_HSI}" "${VAL_HSI}" +etcdput "${KEY_COUNTS}" "${VAL_COUNTS}" + +printf "\n" +info "Done." diff --git a/e2e_test/run_e2e_test.sh b/e2e_test/run_e2e_test.sh new file mode 100755 index 0000000..8fc349a --- /dev/null +++ b/e2e_test/run_e2e_test.sh @@ -0,0 +1,552 @@ +#!/usr/bin/env bash +# ============================================================================= +# FastRG Node — End-to-End Data Plane Test Script +# +# Usage: +# ./run_e2e_test.sh [OPTIONS] +# +# Options: +# --fastrg-node IP FastRG node IP (default: 192.168.10.201) +# --lan-host IP LAN-side host IP (default: 192.168.10.210) +# --wan-host IP WAN-side host IP (default: 192.168.10.106) +# --wan-ip IP WAN subscriber IP (default: 192.168.201.10) +# --runner-host IP E2E runner host IP (default: 192.168.10.207) +# --ssh-key PATH SSH identity file (default: auto-detect id_ed25519 or id_rsa) +# --help Show this help +# +# Requirements (local machine): +# - jq +# - ssh / scp +# +# Requirements (remote hosts): +# - FastRG node: etcdctl +# - WAN host: iperf3, python3 + scapy +# - LAN host: ping, iperf3, curl, tcpdump +# ============================================================================= + +# --------------------------------------------------------------------------- +# Colour helpers (printf-based, portable macOS/Linux) +# --------------------------------------------------------------------------- +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +BOLD='\033[1m' +NC='\033[0m' + +info() { printf "${CYAN}[INFO]${NC} %s\n" "$*"; } +warn() { printf "${YELLOW}[WARN]${NC} %s\n" "$*"; } +error() { printf "${RED}[ERROR]${NC} %s\n" "$*" >&2; } +bold() { printf "${BOLD}%s${NC}\n" "$*"; } + +# --------------------------------------------------------------------------- +# Self-relocation — the runner must have an user called "the". +# If invoked from any other machine it uploads itself + companion files and +# re-executes there. +# Set _FASTRG_E2E_RELOCATED=1 to skip this check (set automatically on relay). +# --------------------------------------------------------------------------- +# Allow --runner-host to override the default before argument parsing runs. +# We do a quick pre-scan of $@ here so the self-relocation block can use it. +_E2E_RUNNER_HOST="192.168.10.207" +for _arg in "$@"; do + if [[ "$_arg" == --runner-host=* ]]; then + _E2E_RUNNER_HOST="${_arg#--runner-host=}" + fi +done +# Also support the two-token form: --runner-host +_prev="" +for _arg in "$@"; do + if [[ "$_prev" == "--runner-host" ]]; then + _E2E_RUNNER_HOST="$_arg" + fi + _prev="$_arg" +done +unset _prev _arg +_E2E_RUNNER_USER="the" +_E2E_REMOTE_DIR='~/fastrg_e2e_test' +_E2E_REMOTE_PATH="${_E2E_REMOTE_DIR}/run_e2e_test.sh" + +if [[ -z "${_FASTRG_E2E_RELOCATED:-}" ]]; then + # Collect local IPs — hostname -I on Linux, ifconfig on macOS + _my_ips=$(hostname -I 2>/dev/null || \ + ifconfig 2>/dev/null | awk '/inet /{gsub(/addr:/,"",$2); print $2}') + if ! printf '%s\n' $_my_ips | grep -qx "${_E2E_RUNNER_HOST}"; then + info "Not running on ${_E2E_RUNNER_HOST} — uploading files and re-executing remotely..." + _SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10" + _SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + _REPO_ROOT="$(cd "${_SCRIPT_DIR}/.." && pwd)" + + # Ensure remote directory exists + ssh $_SSH_OPTS "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}" "mkdir -p ${_E2E_REMOTE_DIR}" + + # Upload this script to the remote runner host + info "Uploading run_e2e_test.sh..." + scp $_SSH_OPTS "$0" "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}:${_E2E_REMOTE_PATH}" + + # Upload gRPC Python client + if [[ -f "${_SCRIPT_DIR}/fastrg_grpc_client.py" ]]; then + info "Uploading fastrg_grpc_client.py..." + scp $_SSH_OPTS "${_SCRIPT_DIR}/fastrg_grpc_client.py" \ + "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}:${_E2E_REMOTE_DIR}/fastrg_grpc_client.py" + else + warn "fastrg_grpc_client.py not found at ${_SCRIPT_DIR}/fastrg_grpc_client.py" + fi + + # Upload phase scripts + if [[ -d "${_SCRIPT_DIR}/phases" ]]; then + info "Uploading phase scripts..." + ssh $_SSH_OPTS "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}" \ + "mkdir -p ${_E2E_REMOTE_DIR}/phases" + scp $_SSH_OPTS "${_SCRIPT_DIR}/phases/"*.sh \ + "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}:${_E2E_REMOTE_DIR}/phases/" + else + warn "phases/ directory not found at ${_SCRIPT_DIR}/phases" + fi + + # Upload proto file (needed by fastrg_grpc_client.py at runtime via grpcurl) + _PROTO_SRC="${_REPO_ROOT}/northbound/grpc/fastrg_node.proto" + if [[ -f "${_PROTO_SRC}" ]]; then + info "Uploading fastrg_node.proto..." + scp $_SSH_OPTS "${_PROTO_SRC}" \ + "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}:${_E2E_REMOTE_DIR}/fastrg_node.proto" + else + warn "proto not found at ${_PROTO_SRC}" + fi + + # Upload grpcurl only when OS+arch match (platform-specific binary) + _GRPCURL_BIN=$(command -v grpcurl 2>/dev/null || true) + _local_os=$(uname -s) + _local_arch=$(uname -m) + _runner_os=$(ssh $_SSH_OPTS "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}" \ + "uname -s" 2>/dev/null || echo "unknown") + _runner_arch=$(ssh $_SSH_OPTS "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}" \ + "uname -m" 2>/dev/null || echo "unknown") + if [[ -n "$_GRPCURL_BIN" ]] && \ + [[ "$_local_os" == "$_runner_os" ]] && \ + [[ "$_local_arch" == "$_runner_arch" ]]; then + info "Uploading grpcurl binary ($_runner_os/$_runner_arch)..." + scp $_SSH_OPTS "${_GRPCURL_BIN}" \ + "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}:${_E2E_REMOTE_DIR}/grpcurl" + ssh $_SSH_OPTS "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}" \ + "chmod +x ${_E2E_REMOTE_DIR}/grpcurl" + elif [[ -n "$_GRPCURL_BIN" ]] && [[ "$_runner_os" != "unknown" ]] && \ + { [[ "$_local_os" != "$_runner_os" ]] || [[ "$_local_arch" != "$_runner_arch" ]]; }; then + # OS/arch mismatch — verify runner already has its own grpcurl before proceeding + warn "grpcurl OS/arch mismatch (local=$_local_os/$_local_arch, runner=$_runner_os/$_runner_arch)" + info "Checking if runner already has grpcurl..." + # Include /opt/homebrew/bin so Apple Silicon macOS Homebrew installs are found + _runner_grpcurl=$(ssh $_SSH_OPTS "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}" \ + "PATH=\"/usr/local/bin:/opt/homebrew/bin:\$PATH\" command -v grpcurl 2>/dev/null || true" \ + 2>/dev/null || true) + if [[ -n "$_runner_grpcurl" ]]; then + info "Runner has grpcurl at $_runner_grpcurl — it is fine" + else + error "grpcurl is not installed on runner ${_E2E_RUNNER_HOST} ($_runner_os/$_runner_arch)" + if [[ "$_runner_os" == "Darwin" ]]; then + error "Install it on the runner first: brew install grpcurl" + else + error "Install it on the runner first: https://github.com/fullstorydev/grpcurl/releases" + fi + exit 1 + fi + else + warn "grpcurl not found locally — runner must have grpcurl in PATH" + exit 1 + fi + + # Rebuild quoted arg list to forward all original arguments + _remote_args="" + for _a in "$@"; do _remote_args="${_remote_args} '${_a}'"; done + ssh $_SSH_OPTS "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}" \ + "chmod +x ${_E2E_REMOTE_PATH} && _FASTRG_E2E_RELOCATED=1 ${_E2E_REMOTE_PATH}${_remote_args}" + _ssh_rc=$? + + # Clean up uploaded files from runner (always, regardless of test result) + info "Cleaning up uploaded files from runner ${_E2E_RUNNER_HOST}:${_E2E_REMOTE_DIR} ..." + ssh $_SSH_OPTS "${_E2E_RUNNER_USER}@${_E2E_RUNNER_HOST}" \ + "rm -rf ${_E2E_REMOTE_DIR}/run_e2e_test.sh \ + ${_E2E_REMOTE_DIR}/fastrg_grpc_client.py \ + ${_E2E_REMOTE_DIR}/fastrg_node.proto \ + ${_E2E_REMOTE_DIR}/grpcurl \ + ${_E2E_REMOTE_DIR}/phases 2>/dev/null; \ + rmdir ${_E2E_REMOTE_DIR} 2>/dev/null || true" 2>/dev/null || true + + exit $_ssh_rc + fi +fi + +set -euo pipefail + +# Ensure common tool locations are in PATH (needed for macOS SSH non-login shells) +export PATH="/usr/local/bin:/usr/local/sbin:${PATH}" + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- +FASTRG_NODE="192.168.10.201" +LAN_HOST="192.168.10.210" +WAN_HOST="192.168.10.106" +WAN_IP="192.168.201.10" +# Auto-detect SSH key: prefer id_ed25519, fall back to id_rsa +if [[ -f "${HOME}/.ssh/id_ed25519" ]]; then + SSH_KEY="${HOME}/.ssh/id_ed25519" +else + SSH_KEY="${HOME}/.ssh/id_rsa" +fi +FASTRG_GRPC_PORT="50052" # fastrg gRPC TCP port (NodeGrpcPort in config.cfg) +GRPC_CLIENT_DIR="$(cd "$(dirname "$0")" && pwd)" # directory of fastrg_grpc_client.py +USER_ID="" + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +usage() { + # Print only the header block: from line 1 up to (and including) the closing ===== line + awk '/^# =+$/{found++} found==1{sub(/^# ?/,""); print} found==2{exit}' "$0" + exit 0 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --help|-h) usage ;; + --fastrg-node) FASTRG_NODE="$2"; shift 2 ;; + --lan-host) LAN_HOST="$2"; shift 2 ;; + --wan-host) WAN_HOST="$2"; shift 2 ;; + --wan-ip) WAN_IP="$2"; shift 2 ;; + --runner-host) _E2E_RUNNER_HOST="$2"; shift 2 ;; + --ssh-key) SSH_KEY="$2"; shift 2 ;; + --grpc-port) FASTRG_GRPC_PORT="$2"; shift 2 ;; + -*) error "Unknown option: $1"; exit 1 ;; + *) + if [[ -z "$USER_ID" ]]; then + USER_ID="$1" + else + error "Unexpected argument: $1"; exit 1 + fi + shift ;; + esac +done + +if [[ -z "$USER_ID" ]]; then + error "USER_ID is required." + printf "Usage: %s [--options]\n" "$0" + printf "Run '%s --help' for full usage.\n" "$0" + exit 1 +fi + +# --------------------------------------------------------------------------- +# SSH helper functions +# --------------------------------------------------------------------------- +SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes -i ${SSH_KEY}" + +ssh_node() { ssh $SSH_OPTS "root@${FASTRG_NODE}" "$@"; } +ssh_lan() { ssh $SSH_OPTS "the@${LAN_HOST}" "$@"; } +ssh_wan() { ssh $SSH_OPTS "root@${WAN_HOST}" "$@"; } + +# --------------------------------------------------------------------------- +# Test result tracking (indexed arrays — bash 3.2 compatible) +# --------------------------------------------------------------------------- +STEP_NAMES=() +STEP_RESULTS=() # "PASS" | "FAIL" | "SKIP" +STEP_DETAILS=() + +record_result() { + local name="$1" result="$2" detail="${3:-}" + STEP_NAMES+=("$name") + STEP_RESULTS+=("$result") + STEP_DETAILS+=("$detail") +} + +pass() { + local name="$1" detail="${2:-}" + printf " ${GREEN}[PASS]${NC} %s\n" "$name" + [[ -n "$detail" ]] && printf " %s\n" "$detail" + record_result "$name" "PASS" "$detail" +} + +fail() { + local name="$1" detail="${2:-}" + printf " ${RED}[FAIL]${NC} %s\n" "$name" + [[ -n "$detail" ]] && printf " %s\n" "$detail" + record_result "$name" "FAIL" "$detail" +} + +skip() { + local name="$1" detail="${2:-}" + printf " ${YELLOW}[SKIP]${NC} %s\n" "$name" + [[ -n "$detail" ]] && printf " %s\n" "$detail" + record_result "$name" "SKIP" "$detail" +} + +# --------------------------------------------------------------------------- +# Phase 0 — Prerequisites +# --------------------------------------------------------------------------- +phase0_setup() { + bold "═══════════════════════════════════════════════════════" + bold " Phase 0 — Prerequisite Checks" + bold "═══════════════════════════════════════════════════════" + + # Check local jq + if ! command -v jq >/dev/null 2>&1; then + error "jq is not installed on this machine. Please install jq first." + exit 1 + fi + info "Local jq: $(jq --version)" + + # Check SSH key + if [[ ! -f "$SSH_KEY" ]]; then + error "SSH key not found: ${SSH_KEY}" + exit 1 + fi + + # Check SSH connectivity to FastRG node + info "Checking SSH connectivity to FastRG node (${FASTRG_NODE})..." + if ! ssh_node "true" 2>/dev/null; then + error "Cannot SSH to FastRG node at ${FASTRG_NODE}" + exit 1 + fi + info "FastRG node reachable." + + # Read NODE_UUID + info "Reading NODE_UUID from /etc/fastrg/node_uuid..." + NODE_UUID=$(ssh_node "cat /etc/fastrg/node_uuid" 2>/dev/null | tr -d '[:space:]') + if [[ -z "$NODE_UUID" ]]; then + error "Failed to read NODE_UUID from /etc/fastrg/node_uuid on ${FASTRG_NODE}" + exit 1 + fi + info "NODE_UUID: ${NODE_UUID}" + + # Read ETCD_ENDPOINT from /etc/fastrg/config.cfg + info "Reading ETCD_ENDPOINT from /etc/fastrg/config.cfg..." + ETCD_RAW=$(ssh_node "grep 'EtcdEndpoints' /etc/fastrg/config.cfg" 2>/dev/null || true) + ETCD_ENDPOINT=$(printf '%s' "$ETCD_RAW" | awk -F'"' '{print $2}') + if [[ -z "$ETCD_ENDPOINT" ]]; then + error "Failed to parse EtcdEndpoints from /etc/fastrg/config.cfg" + exit 1 + fi + info "ETCD_ENDPOINT: ${ETCD_ENDPOINT}" + + # Read FASTRG_GRPC_PORT from node config (may override default) + _grpc_port_raw=$(ssh_node "grep 'NodeGrpcPort' /etc/fastrg/config.cfg 2>/dev/null" | awk -F'"' '{print $2}') + if [[ -n "$_grpc_port_raw" ]]; then + FASTRG_GRPC_PORT="$_grpc_port_raw" + fi + info "FASTRG_GRPC_PORT: ${FASTRG_GRPC_PORT}" + + # Export for use in all subsequent functions + export NODE_UUID ETCD_ENDPOINT FASTRG_GRPC_PORT + + # ------------------------------------------------------------------ + # Check Python3 + grpcurl + proto (fastrg_grpc_client.py uses grpcurl; + # no grpcio or pb2 stubs required) + # ------------------------------------------------------------------ + info "Checking Python3..." + if ! command -v python3 >/dev/null 2>&1; then + error "python3 is required but not found. Please install python3." + exit 1 + fi + if [[ ! -f "${GRPC_CLIENT_DIR}/fastrg_grpc_client.py" ]]; then + error "fastrg_grpc_client.py not found in ${GRPC_CLIENT_DIR}" + exit 1 + fi + info "Python3 gRPC client: ${GRPC_CLIENT_DIR}/fastrg_grpc_client.py" + + # Check grpcurl binary (bundled alongside script takes priority) + if [[ -f "${GRPC_CLIENT_DIR}/grpcurl" ]] && [[ -x "${GRPC_CLIENT_DIR}/grpcurl" ]]; then + info "grpcurl: ${GRPC_CLIENT_DIR}/grpcurl" + elif command -v grpcurl >/dev/null 2>&1; then + info "grpcurl: $(command -v grpcurl)" + else + error "grpcurl not found. Upload it alongside this script or install it in PATH." + exit 1 + fi + + # Check proto file (needed by grpcurl at runtime) + if [[ ! -f "${GRPC_CLIENT_DIR}/fastrg_node.proto" ]]; then + error "fastrg_node.proto not found in ${GRPC_CLIENT_DIR}" + error "Re-run the script from the repo root to allow automatic proto upload" + exit 1 + fi + info "proto: ${GRPC_CLIENT_DIR}/fastrg_node.proto" + + # ------------------------------------------------------------------ + # Ensure fastrg daemon is running; start it if not + # ------------------------------------------------------------------ + info "Checking if fastrg daemon is running on ${FASTRG_NODE}..." + FASTRG_PID=$(ssh_node "pgrep -x fastrg 2>/dev/null || pidof fastrg 2>/dev/null || true" | tr -d '[:space:]') + if [[ -z "$FASTRG_PID" ]]; then + warn "fastrg is NOT running — attempting to start..." + _FASTRG_DAEMON="/root/fastrg-node/fastrg" + _FASTRG_START_CMD="${_FASTRG_DAEMON} -l 1-9 -n 4 -a 0000:04:00.0 -a 0000:08:00.0" + info "Starting: ${_FASTRG_START_CMD}" + ssh_node "nohup ${_FASTRG_START_CMD} >/var/log/fastrg.log 2>&1 &" + _FASTRG_STARTED_BY_SCRIPT=1 + + # Wait up to 120 s for fastrg gRPC + HSI data for USER_ID to be ready + # (including PPPoE session establishment — account must be non-empty) + info "Waiting for fastrg gRPC + HSI data for USER_ID=${USER_ID} to be ready (up to 120s)..." + _ready=0 + for _i in $(seq 1 24); do + sleep 5 + _hsi=$(python3 "${GRPC_CLIENT_DIR}/fastrg_grpc_client.py" \ + --node "${FASTRG_NODE}:${FASTRG_GRPC_PORT}" \ + get_hsi_info 2>/dev/null || true) + # Wait for: user exists in hsi_infos AND account is non-empty (PPPoE session up) + if printf '%s' "$_hsi" | python3 -c \ + "import sys,json; d=json.load(sys.stdin); \ + infos=d.get('hsi_infos',[]); \ + u=[h for h in infos if h.get('user_id')==${USER_ID}]; \ + sys.exit(0 if u and u[0].get('account','') else 1)" \ + 2>/dev/null; then + _ready=1 + break + fi + _cnt=$(printf '%s' "$_hsi" | python3 -c \ + 'import sys,json; d=json.load(sys.stdin); print(len(d.get("hsi_infos",[])))' \ + 2>/dev/null || echo '?') + _acct=$(printf '%s' "$_hsi" | python3 -c \ + "import sys,json; d=json.load(sys.stdin); \ + u=[h for h in d.get('hsi_infos',[]) if h.get('user_id')==${USER_ID}]; \ + print(u[0].get('account','') if u else '')" \ + 2>/dev/null || echo '') + info " still waiting... (${_i}×5s, hsi_infos=${_cnt}, account='${_acct}')" + done + if [[ $_ready -eq 0 ]]; then + error "fastrg gRPC did not become ready within 120 seconds." + info "Last log output:" + ssh_node "tail -20 /var/log/fastrg.log 2>/dev/null || true" + exit 1 + fi + + # Additionally wait for DNS static records to be loaded (needed for step 2d) + _dns_ready=0 + for _i in $(seq 1 12); do + _dns=$(python3 "${GRPC_CLIENT_DIR}/fastrg_grpc_client.py" \ + --node "${FASTRG_NODE}:${FASTRG_GRPC_PORT}" \ + get_dns_static "${USER_ID}" 2>/dev/null || true) + if printf '%s' "$_dns" | python3 -c \ + "import sys,json; d=json.load(sys.stdin); \ + sys.exit(0 if d.get('total_entries',0) > 0 or d.get('entries') else 1)" \ + 2>/dev/null; then + _dns_ready=1 + break + fi + info " waiting for DNS static records... (${_i}×5s)" + sleep 5 + done + [[ $_dns_ready -eq 0 ]] && info " (DNS static not loaded yet — will verify in step 2d)" + + info "fastrg gRPC is ready." + else + info "fastrg is running (pid: ${FASTRG_PID})." + _FASTRG_STARTED_BY_SCRIPT=0 + # Verify gRPC is reachable + _sys=$(python3 "${GRPC_CLIENT_DIR}/fastrg_grpc_client.py" \ + --node "${FASTRG_NODE}:${FASTRG_GRPC_PORT}" \ + get_system_info 2>/dev/null || true) + if ! printf '%s' "$_sys" | python3 -c "import sys,json; json.load(sys.stdin)" 2>/dev/null; then + error "fastrg is running but gRPC on ${FASTRG_NODE}:${FASTRG_GRPC_PORT} is not responding." + exit 1 + fi + info "fastrg gRPC is reachable." + fi + + printf "\n" + info "Configuration summary:" + printf " USER_ID : %s\n" "$USER_ID" + printf " FASTRG_NODE : %s\n" "$FASTRG_NODE" + printf " FASTRG_GRPC : %s:%s\n" "$FASTRG_NODE" "$FASTRG_GRPC_PORT" + printf " LAN_HOST : %s (user: the)\n" "$LAN_HOST" + printf " WAN_HOST : %s\n" "$WAN_HOST" + printf " WAN_IP : %s\n" "$WAN_IP" + printf " NODE_UUID : %s\n" "$NODE_UUID" + printf " ETCD_ENDPOINT : %s\n" "$ETCD_ENDPOINT" + printf "\n" +} + +# --------------------------------------------------------------------------- +# etcdctl wrapper — runs on FastRG node +# --------------------------------------------------------------------------- +etcdctl_get() { + ssh_node "ETCDCTL_API=3 etcdctl --endpoints=${ETCD_ENDPOINT} get $*" +} + +etcdctl_get_value() { + # etcdctl prints key on one line, value on next; we want only the value + ssh_node "ETCDCTL_API=3 etcdctl --endpoints=${ETCD_ENDPOINT} get --print-value-only $*" +} + +# --------------------------------------------------------------------------- +# fastrg_grpc — call FastRG gRPC server directly via Python3 client +# Returns JSON on stdout; empty string on error. +# --------------------------------------------------------------------------- +fastrg_grpc() { + python3 "${GRPC_CLIENT_DIR}/fastrg_grpc_client.py" \ + --node "${FASTRG_NODE}:${FASTRG_GRPC_PORT}" \ + "$@" 2>/dev/null || true +} + +# --------------------------------------------------------------------------- +# Load phase scripts (each file defines one phase function) +# --------------------------------------------------------------------------- +_E2E_PHASES_DIR="${GRPC_CLIENT_DIR}/phases" +# shellcheck source=/dev/null +source "${_E2E_PHASES_DIR}/phase0_setup.sh" +source "${_E2E_PHASES_DIR}/phase1_subscriber_count_tests.sh" +source "${_E2E_PHASES_DIR}/phase2_etcd_config_sync.sh" +source "${_E2E_PHASES_DIR}/phase3_dhcp_and_count.sh" +source "${_E2E_PHASES_DIR}/phase3_5_enable_status.sh" +source "${_E2E_PHASES_DIR}/phase4_lan_to_wan.sh" +source "${_E2E_PHASES_DIR}/phase5_dnat_test.sh" +source "${_E2E_PHASES_DIR}/phase6_dns_ping.sh" +source "${_E2E_PHASES_DIR}/phase7_user1_config_tests.sh" +source "${_E2E_PHASES_DIR}/phase8_summary.sh" + +# --------------------------------------------------------------------------- +# Cleanup — kill fastrg only if the script started it +# --------------------------------------------------------------------------- +cleanup_fastrg() { + set +eu # Prevent set -e / set -u from interrupting cleanup, ensure all cleanup steps are executed + + # Best-effort: remove user 1 config if the test left it in etcd + _cleanup_user1_config 2>/dev/null || true + + if [[ "${_FASTRG_STARTED_BY_SCRIPT:-0}" -eq 1 ]]; then + info "Stopping fastrg (started by this script)..." + ssh_node "pkill -x fastrg 2>/dev/null || true" || true + info "fastrg stopped." + fi + + info "Cleanup complete." +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +main() { + _FASTRG_STARTED_BY_SCRIPT=0 + + # Clean up anyway on exit + trap 'cleanup_fastrg' EXIT + + printf "\n" + bold "╔═════════════════════════════════════════════════════╗" + bold "║ FastRG Node — E2E Data Plane Test ║" + bold "╚═════════════════════════════════════════════════════╝" + printf "\n" + info "USER_ID: ${USER_ID}" + printf "\n" + + phase0_setup + phase1_subscriber_count_tests + phase2_etcd_config_sync + phase3_dhcp_and_count + phase3_5_enable_status + phase4_lan_to_wan + phase5_dnat_test + phase6_dns_ping + phase7_user1_config_tests + phase8_summary || true +} + +main "$@" diff --git a/northbound/grpc/fastrg_node_grpc.cpp b/northbound/grpc/fastrg_node_grpc.cpp index 19ea161..bafd055 100644 --- a/northbound/grpc/fastrg_node_grpc.cpp +++ b/northbound/grpc/fastrg_node_grpc.cpp @@ -856,6 +856,8 @@ grpc::Status FastRGNodeServiceImpl::GetFastrgHsiInfo(::grpc::ServerContext* cont ppp_ccb_t *ppp_ccb = PPPD_GET_CCB(fastrg_ccb, i); hsi_info->set_user_id(i + 1); hsi_info->set_vlan_id(rte_atomic16_read(&ppp_ccb->vlan_id)); + hsi_info->set_account(std::string(reinterpret_cast(ppp_ccb->ppp_user_acc))); + hsi_info->set_password(std::string(reinterpret_cast(ppp_ccb->ppp_passwd))); switch (ppp_ccb->phase) { case END_PHASE: hsi_info->set_status("End phase"); @@ -874,8 +876,6 @@ grpc::Status FastRGNodeServiceImpl::GetFastrgHsiInfo(::grpc::ServerContext* cont break; case DATA_PHASE: hsi_info->set_status("Data phase"); - hsi_info->set_account(std::string(reinterpret_cast(ppp_ccb->ppp_user_acc))); - hsi_info->set_password(std::string(reinterpret_cast(ppp_ccb->ppp_passwd))); hsi_info->set_session_id(rte_be_to_cpu_16(ppp_ccb->session_id)); hsi_info->set_ip_addr(std::to_string(*(((U8 *)&(ppp_ccb->hsi_ipv4)))) + "." + std::to_string(*(((U8 *)&(ppp_ccb->hsi_ipv4))+1)) + "." +