From d4d74081da743084f7db4f6c7359a70e3d7a79fd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Apr 2026 17:35:05 +0000 Subject: [PATCH 01/15] Enable CLI-driven Pico protocol coverage for MicroPython/CircuitPython (ADC, PIO, SPI, I2C, UART) Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/35b995e1-b1c1-45d0-a8f1-91cf6c124e95 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- OpenHW-studio-frontend-danish/package-lock.json | 7 ------- openhw-studio-cli-danish/package-lock.json | 2 -- 2 files changed, 9 deletions(-) diff --git a/OpenHW-studio-frontend-danish/package-lock.json b/OpenHW-studio-frontend-danish/package-lock.json index 53dc411..1c93839 100644 --- a/OpenHW-studio-frontend-danish/package-lock.json +++ b/OpenHW-studio-frontend-danish/package-lock.json @@ -84,7 +84,6 @@ "version": "7.29.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -1411,7 +1410,6 @@ "integrity": "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.18.0" } @@ -1529,7 +1527,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -2356,7 +2353,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -2391,7 +2387,6 @@ "node_modules/react": { "version": "18.3.1", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -2402,7 +2397,6 @@ "node_modules/react-dom": { "version": "18.3.1", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -2631,7 +2625,6 @@ "version": "5.4.21", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", diff --git a/openhw-studio-cli-danish/package-lock.json b/openhw-studio-cli-danish/package-lock.json index 8454731..4128f80 100644 --- a/openhw-studio-cli-danish/package-lock.json +++ b/openhw-studio-cli-danish/package-lock.json @@ -1384,7 +1384,6 @@ "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.11.tgz", "integrity": "sha512-r4xbIa3mGGGoH9nN4A14DOg2wx7y2oQyJEb5O57C/xzETG/qx4c7CVDQ5WMeKHZ7ORk2W0hZ/sQKXTav3cmYBA==", "license": "MIT", - "peer": true, "engines": { "node": ">=16.9.0" } @@ -2076,7 +2075,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } From c9b7f503e00eaa496232d945f55bc45b1402f59e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:37:43 +0000 Subject: [PATCH 02/15] Add MCP Pico individual component test script Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/de5955b3-6ebb-4fd9-b6ef-a51bf4fe42c9 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- openhw-studio-cli-danish/package.json | 3 +- .../test-mcp-pico-components-individual.mjs | 403 ++++++++++++++++++ 2 files changed, 405 insertions(+), 1 deletion(-) create mode 100644 openhw-studio-cli-danish/scripts/test-mcp-pico-components-individual.mjs diff --git a/openhw-studio-cli-danish/package.json b/openhw-studio-cli-danish/package.json index 7e175f3..149aa36 100644 --- a/openhw-studio-cli-danish/package.json +++ b/openhw-studio-cli-danish/package.json @@ -10,7 +10,8 @@ "mcp": "tsx src/cli.ts mcp serve", "typecheck": "tsc --noEmit", "test:mcp:smoke": "node scripts/mcp-smoke.mjs", - "test:mcp:pico-all-components": "node scripts/test-mcp-pico-all-components.mjs" + "test:mcp:pico-all-components": "node scripts/test-mcp-pico-all-components.mjs", + "test:mcp:pico-components-individual": "node scripts/test-mcp-pico-components-individual.mjs" }, "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", diff --git a/openhw-studio-cli-danish/scripts/test-mcp-pico-components-individual.mjs b/openhw-studio-cli-danish/scripts/test-mcp-pico-components-individual.mjs new file mode 100644 index 0000000..4727f10 --- /dev/null +++ b/openhw-studio-cli-danish/scripts/test-mcp-pico-components-individual.mjs @@ -0,0 +1,403 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const cliRoot = path.resolve(__dirname, '..'); +const workspaceRoot = path.resolve(cliRoot, '..'); +const emulatorComponentsRoot = path.join(workspaceRoot, 'openhw-studio-emulator-danish', 'src', 'components'); + +const backendUrl = String(process.env.MCP_TEST_BACKEND_URL || 'http://127.0.0.1:5001/api').trim(); +const token = String(process.env.OPENHW_MCP_TOKEN || '').trim(); + +const envMatrix = [ + { + key: 'micropython', + boardEnv: 'micropython', + durationMs: 1800, + codeFile: 'project/board1/main.py', + source: `from time import sleep\nprint("MCP_MP_COMPONENT_BOOT")\nfor i in range(6):\n print("MCP_MP_COMPONENT_STEP", i)\n sleep(0.05)\n`, + }, + { + key: 'circuitpython', + boardEnv: 'circuitpython', + durationMs: 2400, + codeFile: 'project/board1/code.py', + source: `import time\nprint("MCP_CP_COMPONENT_BOOT")\nfor i in range(6):\n print("MCP_CP_COMPONENT_STEP", i)\n time.sleep(0.05)\nprint("MCP_CP_COMPONENT_DONE")\n`, + }, +]; + +function npmCommand() { + return process.platform === 'win32' ? 'npm.cmd' : 'npm'; +} + +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +function parseToolPayload(result, toolName) { + const content = Array.isArray(result?.content) ? result.content : []; + const text = content + .filter((entry) => String(entry?.type || '') === 'text') + .map((entry) => String(entry?.text || '')) + .join('\n') + .trim(); + + let parsed = null; + if (text) { + try { + parsed = JSON.parse(text); + } catch { + parsed = null; + } + } + + if (result?.isError) { + throw new Error(`${toolName} returned isError=true${text ? `: ${text}` : ''}`); + } + + assert(parsed && typeof parsed === 'object', `${toolName} returned empty or non-JSON payload.`); + assert(parsed.ok === true, `${toolName} did not return ok=true.`); + return parsed; +} + +async function callTool(client, name, args) { + const response = await client.callTool({ + name, + arguments: args, + }); + return parseToolPayload(response, name); +} + +async function resolveProjectPathFromMcp(fileField) { + const relative = String(fileField || '').trim(); + assert(relative.length > 0, 'project_init did not return file path.'); + + if (path.isAbsolute(relative)) { + return relative; + } + + const normalized = relative.replace(/\\/g, '/'); + const candidates = [ + path.join(workspaceRoot, normalized), + path.join(cliRoot, normalized), + path.resolve(process.cwd(), normalized), + ]; + + for (const candidate of candidates) { + try { + await fs.access(candidate); + return candidate; + } catch { + continue; + } + } + + return candidates[0]; +} + +function ensureCodeFiles(project, sourceByPath) { + if (!Array.isArray(project.projectFiles)) { + project.projectFiles = []; + } + + const byPath = new Map(project.projectFiles.map((file) => [String(file.path || ''), file])); + + for (const [filePath, source] of sourceByPath.entries()) { + if (!byPath.has(filePath)) { + const name = path.posix.basename(filePath.replace(/\\/g, '/')); + const created = { + id: filePath, + path: filePath, + name, + kind: 'code', + boardId: 'board1', + boardKind: 'rp2040', + content: String(source), + dirty: false, + }; + project.projectFiles.push(created); + byPath.set(filePath, created); + } else { + byPath.get(filePath).content = String(source); + byPath.get(filePath).dirty = false; + } + } +} + +async function updateProjectForEnv(projectFile, envConfig) { + const raw = await fs.readFile(projectFile, 'utf8'); + const project = JSON.parse(raw); + + const board = Array.isArray(project.components) + ? project.components.find((component) => String(component?.id || '') === 'board1') + : null; + + assert(board, 'Unable to locate board1 in MCP project.'); + if (!board.attrs || typeof board.attrs !== 'object' || Array.isArray(board.attrs)) { + board.attrs = {}; + } + + board.attrs.env = envConfig.boardEnv; + board.attrs.builder = 'arduino-pico'; + + const sourceByPath = new Map(); + for (const env of envMatrix) { + sourceByPath.set(env.codeFile, env.source); + } + ensureCodeFiles(project, sourceByPath); + + project.code = envConfig.source; + project.activeCodeFileId = envConfig.codeFile; + if (!Array.isArray(project.openCodeTabs)) { + project.openCodeTabs = []; + } + if (!project.openCodeTabs.includes(envConfig.codeFile)) { + project.openCodeTabs.push(envConfig.codeFile); + } + + await fs.writeFile(projectFile, `${JSON.stringify(project, null, 2)}\n`, 'utf8'); +} + +function normalizePinName(value) { + return String(value || '').trim().toUpperCase(); +} + +function findPin(pinNames, predicate) { + return pinNames.find((pin) => predicate(normalizePinName(pin))) || null; +} + +function buildWirePlan(pinNames) { + const hasPin = (name) => pinNames.includes(name); + const wires = []; + const seen = new Set(); + const add = (from, to) => { + const key = `${from}=>${to}`; + if (seen.has(key)) return; + seen.add(key); + wires.push({ from, to }); + }; + + const gndPin = findPin(pinNames, (pin) => pin === 'GND' || pin === 'K' || pin === 'C'); + const vccPin = findPin(pinNames, (pin) => /^(VCC|VDD|VIN|V\+|A|5V|3V3|3\.3V|LED)$/.test(pin)); + if (gndPin) add('board1:GND', `uut:${gndPin}`); + if (vccPin) add('board1:3V3', `uut:${vccPin}`); + + const sdaPin = findPin(pinNames, (pin) => pin === 'SDA'); + const sclPin = findPin(pinNames, (pin) => pin === 'SCL'); + if (sdaPin) add('board1:GP4', `uut:${sdaPin}`); + if (sclPin) add('board1:GP5', `uut:${sclPin}`); + + const mosiPin = findPin(pinNames, (pin) => pin === 'MOSI' || pin === 'DIN'); + const misoPin = findPin(pinNames, (pin) => pin === 'MISO' || pin === 'DOUT'); + const sckPin = findPin(pinNames, (pin) => pin === 'SCK' || pin === 'CLK'); + const csPin = findPin(pinNames, (pin) => pin === 'CS' || pin === 'SS'); + if (mosiPin) add('board1:GP19', `uut:${mosiPin}`); + if (misoPin) add('board1:GP16', `uut:${misoPin}`); + if (sckPin) add('board1:GP18', `uut:${sckPin}`); + if (csPin) add('board1:GP17', `uut:${csPin}`); + + const rxPin = findPin(pinNames, (pin) => pin === 'RX' || pin === 'RXD'); + const txPin = findPin(pinNames, (pin) => pin === 'TX' || pin === 'TXD'); + if (rxPin) add('board1:GP0', `uut:${rxPin}`); + if (txPin) add('board1:GP1', `uut:${txPin}`); + + const signalPin = findPin(pinNames, (pin) => ( + pin === 'A' + || pin === 'ANODE' + || pin === 'SIG' + || pin === 'IN' + || pin === 'OUT' + || pin === 'PWM' + || pin === 'DATA' + || pin === 'DIN' + )); + if (signalPin) add('board1:GP2', `uut:${signalPin}`); + + if (wires.length === 0) { + const fallback = findPin(pinNames, (pin) => ( + !['GND', 'VCC', 'VDD', 'VIN', '5V', '3V3', '3.3V'].includes(pin) + )); + if (fallback) { + add('board1:GP2', `uut:${fallback}`); + } + } + + return wires; +} + +async function listComponentCatalog() { + const entries = await fs.readdir(emulatorComponentsRoot, { withFileTypes: true }); + const components = []; + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const type = String(entry.name || '').trim(); + if (!type) continue; + if (/(wokwi-arduino|wokwi-esp32|wokwi-stm32|wokwi-raspberry-pi-pico)/i.test(type)) continue; + + const manifestPath = path.join(emulatorComponentsRoot, type, 'manifest.json'); + try { + const manifestRaw = await fs.readFile(manifestPath, 'utf8'); + const manifest = JSON.parse(manifestRaw); + const pinNames = Array.isArray(manifest?.pins) + ? manifest.pins.map((pin) => String(pin?.id || '').trim()).filter(Boolean) + : []; + components.push({ + type, + pinNames, + }); + } catch { + continue; + } + } + + components.sort((a, b) => a.type.localeCompare(b.type)); + return components; +} + +function readTelemetryComponents(payload) { + if (Array.isArray(payload?.telemetry?.components)) return payload.telemetry.components; + if (Array.isArray(payload?.result?.telemetry?.components)) return payload.result.telemetry.components; + return []; +} + +async function runSingleCase(client, component, envConfig) { + const initPayload = await callTool(client, 'project_init', { + name: `pico-${envConfig.key}-${component.type}-${Date.now()}`, + board: 'wokwi-raspberry-pi-pico', + ...(token ? { token } : {}), + }); + const projectFile = await resolveProjectPathFromMcp(initPayload.file); + + await callTool(client, 'component_add', { + type: component.type, + id: 'uut', + x: 360, + y: 120, + ...(token ? { token } : {}), + }); + + const wirePlan = buildWirePlan(component.pinNames); + const wireErrors = []; + for (const wire of wirePlan) { + try { + await callTool(client, 'wire_add', { + from: wire.from, + to: wire.to, + ...(token ? { token } : {}), + }); + } catch (error) { + wireErrors.push({ + from: wire.from, + to: wire.to, + error: String(error?.message || error), + }); + } + } + + await updateProjectForEnv(projectFile, envConfig); + + const executePayload = await callTool(client, 'sim_execute', { + ms: envConfig.durationMs, + all_boards: true, + include_telemetry: true, + ...(token ? { token } : {}), + }); + + const telemetryComponents = readTelemetryComponents(executePayload); + const telemetryById = new Map( + telemetryComponents.map((entry) => [String(entry?.id || ''), entry]) + ); + const uutTelemetry = telemetryById.get('uut') || null; + + return { + ok: !!uutTelemetry, + env: envConfig.key, + boardEnv: envConfig.boardEnv, + componentType: component.type, + pinCount: component.pinNames.length, + plannedWires: wirePlan, + wireErrorCount: wireErrors.length, + wireErrors, + telemetryFound: !!uutTelemetry, + telemetryStatus: String(uutTelemetry?.status || ''), + telemetrySummary: String(uutTelemetry?.telemetrySummary || ''), + componentCount: telemetryComponents.length, + durationMs: envConfig.durationMs, + }; +} + +async function main() { + const components = await listComponentCatalog(); + assert(components.length > 0, 'No component manifests found for individual Pico tests.'); + + const transport = new StdioClientTransport({ + command: npmCommand(), + args: ['run', 'mcp', '--', '--backend-url', backendUrl], + cwd: cliRoot, + stderr: 'inherit', + }); + const client = new Client({ name: 'openhw-mcp-pico-components-individual', version: '0.1.0' }); + + const summary = { + ok: true, + generatedAt: new Date().toISOString(), + backendUrl, + envs: envMatrix.map((entry) => entry.key), + componentCount: components.length, + results: [], + }; + + try { + await client.connect(transport); + for (const envConfig of envMatrix) { + for (const component of components) { + try { + const result = await runSingleCase(client, component, envConfig); + summary.results.push(result); + const state = result.ok ? 'PASS' : 'FAIL'; + console.log(`[mcp-pico-components-individual] ${state} env=${envConfig.key} component=${component.type} wires=${result.plannedWires.length} wireErrors=${result.wireErrorCount}`); + } catch (error) { + summary.results.push({ + ok: false, + env: envConfig.key, + boardEnv: envConfig.boardEnv, + componentType: component.type, + error: String(error?.message || error), + }); + console.log(`[mcp-pico-components-individual] FAIL env=${envConfig.key} component=${component.type} error=${String(error?.message || error)}`); + } + } + } + } finally { + await client.close(); + } + + const totals = { + total: summary.results.length, + passed: summary.results.filter((entry) => entry.ok).length, + failed: summary.results.filter((entry) => !entry.ok).length, + }; + summary.totals = totals; + summary.ok = totals.failed === 0; + + const outPath = path.join(workspaceRoot, 'temp', 'mcp-pico-components-individual-summary.json'); + await fs.writeFile(outPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8'); + console.log(`[mcp-pico-components-individual] summary=${path.relative(workspaceRoot, outPath).replace(/\\/g, '/')}`); + console.log(`[mcp-pico-components-individual] total=${totals.total} passed=${totals.passed} failed=${totals.failed}`); + + if (!summary.ok) { + process.exitCode = 1; + } +} + +main().catch((error) => { + console.error(`[mcp-pico-components-individual] FAIL ${String(error?.message || error)}`); + process.exit(1); +}); From 9fee5df21800430831b33b41e4e8d0aacb35b86d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:43:46 +0000 Subject: [PATCH 03/15] Fix summary output path for Pico individual MCP tests Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/de5955b3-6ebb-4fd9-b6ef-a51bf4fe42c9 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- .../scripts/test-mcp-pico-components-individual.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openhw-studio-cli-danish/scripts/test-mcp-pico-components-individual.mjs b/openhw-studio-cli-danish/scripts/test-mcp-pico-components-individual.mjs index 4727f10..25d0ae8 100644 --- a/openhw-studio-cli-danish/scripts/test-mcp-pico-components-individual.mjs +++ b/openhw-studio-cli-danish/scripts/test-mcp-pico-components-individual.mjs @@ -387,7 +387,9 @@ async function main() { summary.totals = totals; summary.ok = totals.failed === 0; - const outPath = path.join(workspaceRoot, 'temp', 'mcp-pico-components-individual-summary.json'); + const outDir = path.join(workspaceRoot, 'temp'); + await fs.mkdir(outDir, { recursive: true }); + const outPath = path.join(outDir, 'mcp-pico-components-individual-summary.json'); await fs.writeFile(outPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8'); console.log(`[mcp-pico-components-individual] summary=${path.relative(workspaceRoot, outPath).replace(/\\/g, '/')}`); console.log(`[mcp-pico-components-individual] total=${totals.total} passed=${totals.passed} failed=${totals.failed}`); From 76c7bea1d4fe989a83fadded0980cd366002bc9d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:55:57 +0000 Subject: [PATCH 04/15] Refine Pico component auto-wiring and re-validate suite Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/de5955b3-6ebb-4fd9-b6ef-a51bf4fe42c9 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- .../scripts/test-mcp-pico-components-individual.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/openhw-studio-cli-danish/scripts/test-mcp-pico-components-individual.mjs b/openhw-studio-cli-danish/scripts/test-mcp-pico-components-individual.mjs index 25d0ae8..c092e9a 100644 --- a/openhw-studio-cli-danish/scripts/test-mcp-pico-components-individual.mjs +++ b/openhw-studio-cli-danish/scripts/test-mcp-pico-components-individual.mjs @@ -215,7 +215,6 @@ function buildWirePlan(pinNames) { || pin === 'OUT' || pin === 'PWM' || pin === 'DATA' - || pin === 'DIN' )); if (signalPin) add('board1:GP2', `uut:${signalPin}`); From a3c0deb07fa67ca78daa22be6a89e977009f08cf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:24:38 +0000 Subject: [PATCH 05/15] feat: add MCP lifecycle diagnostics tools and scenario runner Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/1c6315b7-83b3-45c1-a76c-fff9e56a76c3 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- openhw-studio-cli-danish/README.md | 46 +- openhw-studio-cli-danish/package-lock.json | 18 +- openhw-studio-cli-danish/package.json | 7 +- .../scenarios/pico-led-lifecycle.yaml | 46 ++ .../scripts/mcp-scenario-runner.mjs | 580 ++++++++++++++++++ .../scripts/mcp-smoke.mjs | 12 +- .../scripts/test-mcp-contracts.mjs | 132 ++++ openhw-studio-cli-danish/src/mcp/server.ts | 184 ++++++ 8 files changed, 1017 insertions(+), 8 deletions(-) create mode 100644 openhw-studio-cli-danish/scenarios/pico-led-lifecycle.yaml create mode 100644 openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs create mode 100644 openhw-studio-cli-danish/scripts/test-mcp-contracts.mjs diff --git a/openhw-studio-cli-danish/README.md b/openhw-studio-cli-danish/README.md index a00c287..c610250 100644 --- a/openhw-studio-cli-danish/README.md +++ b/openhw-studio-cli-danish/README.md @@ -1,4 +1,3 @@ -<<<<<<< HEAD # OpenHW Studio CLI Terminal-first CLI for OpenHW Studio project management, headless simulation, serial monitoring, and library management. @@ -128,6 +127,13 @@ npm run cli -- mcp serve --auth-token local-dev-token MCP tools now include `sim_execute`, `sim_trace`, and `sim_inspect` with support for debug/GDB trace capture (`debug_mode`, `include_trace`) and simulation console capture (`include_console`) without polluting stdio transport. +Additional MCP lifecycle and diagnostics tools: +- `project_open` (set active session from an existing project file) +- `project_status` (active session state + summary) +- `project_validate` (schema/reference validation for active session) +- `component_catalog` (discover component capabilities/pins/onEvent/telemetry metadata) +- `wiring_validate` (dry-run endpoint/wire validation without project mutation) + ### Library management ```bash @@ -138,6 +144,30 @@ npm run cli -- lib uninstall "Adafruit NeoPixel" npm run cli -- lib sync-project temp/project.json --dry-run ``` +### Advanced MCP testing + behavior reports + +```bash +# MCP response contract coverage (positive + negative paths) +npm run test:mcp:contracts + +# Deterministic scenario runner (YAML/JSON manifest) with unified behavior report export +npm run test:mcp:scenario + +# Override scenario/report paths +node scripts/mcp-scenario-runner.mjs \ + --scenario scenarios/pico-led-lifecycle.yaml \ + --output-json temp/mcp-scenario-report.json \ + --output-md temp/mcp-scenario-report.md \ + --baseline temp/mcp-scenario-report-baseline.json +``` + +The scenario runner report includes: +- per-component state timeline (`trace.componentTimeline`) +- board pin activity timeline (`trace.boardPinTimeline`) +- wire/connectivity diagnostics (`wiring`) +- serial + telemetry + trace correlation (`serial`, `telemetry`, `trace`) +- optional behavior-diff gate against a baseline report (`--baseline`) + ### Interactive REPL ```bash @@ -155,6 +185,14 @@ npm run cli -- repl - `sim interact` can inject component events for interactive parts (e.g. LDR, potentiometer, pushbutton). - `mcp serve` starts a local Model Context Protocol server over stdio, including simulation trace/inspect tools for remote automation. - Default backend URL is `http://localhost:5001/api` and can be overridden with `--backend-url`. -======= -# OpenHW-studio--cli ->>>>>>> origin/develop + +## MCP CLI completion checklist + +- [x] Project lifecycle tools: init/open/status/validate +- [x] Component catalog/discovery with pin and capability metadata +- [x] Dry-run wiring validation diagnostics before mutation +- [x] Simulation execution, trace capture, and inspect/event injection +- [x] Scenario-driven test runner with JSON/YAML manifests +- [x] Unified report export in JSON + human-readable Markdown +- [x] Contract tests for MCP tool response shape + negative cases +- [x] Pico component matrix scripts for broad and per-component coverage diff --git a/openhw-studio-cli-danish/package-lock.json b/openhw-studio-cli-danish/package-lock.json index 4128f80..82480d0 100644 --- a/openhw-studio-cli-danish/package-lock.json +++ b/openhw-studio-cli-danish/package-lock.json @@ -10,7 +10,8 @@ "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", "commander": "^12.1.0", - "serialport": "^12.0.0" + "serialport": "^12.0.0", + "yaml": "^2.8.3" }, "devDependencies": { "@types/node": "^22.10.1", @@ -2070,6 +2071,21 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/yaml": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/zod": { "version": "4.3.6", "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", diff --git a/openhw-studio-cli-danish/package.json b/openhw-studio-cli-danish/package.json index 149aa36..a3503e8 100644 --- a/openhw-studio-cli-danish/package.json +++ b/openhw-studio-cli-danish/package.json @@ -11,12 +11,15 @@ "typecheck": "tsc --noEmit", "test:mcp:smoke": "node scripts/mcp-smoke.mjs", "test:mcp:pico-all-components": "node scripts/test-mcp-pico-all-components.mjs", - "test:mcp:pico-components-individual": "node scripts/test-mcp-pico-components-individual.mjs" + "test:mcp:pico-components-individual": "node scripts/test-mcp-pico-components-individual.mjs", + "test:mcp:contracts": "node scripts/test-mcp-contracts.mjs", + "test:mcp:scenario": "node scripts/mcp-scenario-runner.mjs --scenario scenarios/pico-led-lifecycle.yaml" }, "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", "commander": "^12.1.0", - "serialport": "^12.0.0" + "serialport": "^12.0.0", + "yaml": "^2.8.3" }, "devDependencies": { "@types/node": "^22.10.1", diff --git a/openhw-studio-cli-danish/scenarios/pico-led-lifecycle.yaml b/openhw-studio-cli-danish/scenarios/pico-led-lifecycle.yaml new file mode 100644 index 0000000..35b4e1d --- /dev/null +++ b/openhw-studio-cli-danish/scenarios/pico-led-lifecycle.yaml @@ -0,0 +1,46 @@ +name: pico-led-lifecycle +board: wokwi-raspberry-pi-pico +durationMs: 1800 +traceEventTypes: + - state + - serial + - fault + - debug +envs: + - key: micropython + boardEnv: micropython + ms: 2200 + codeFile: project/board1/main.py + source: | + from machine import Pin + from time import sleep + led = Pin(25, Pin.OUT) + print("MCP_SCENARIO_MP_BOOT") + for i in range(8): + led.value(i % 2) + print("MCP_SCENARIO_MP_STEP", i) + sleep(0.05) + - key: circuitpython + boardEnv: circuitpython + ms: 2600 + codeFile: project/board1/code.py + source: | + import time + print("MCP_SCENARIO_CP_BOOT") + for i in range(8): + print("MCP_SCENARIO_CP_STEP", i) + time.sleep(0.05) +components: + - type: wokwi-led + id: led1 + x: 360 + y: 120 +wires: + - from: board1:GP2 + to: led1:A + - from: board1:GND + to: led1:C +inspect: + - id: led1 + ms: 1200 + includeTrace: true diff --git a/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs b/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs new file mode 100644 index 0000000..fd5129b --- /dev/null +++ b/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs @@ -0,0 +1,580 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import YAML from 'yaml'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const cliRoot = path.resolve(__dirname, '..'); +const workspaceRoot = path.resolve(cliRoot, '..'); + +function npmCommand() { + return process.platform === 'win32' ? 'npm.cmd' : 'npm'; +} + +function parseArgs(argv) { + const out = { + scenario: '', + backendUrl: String(process.env.MCP_TEST_BACKEND_URL || 'http://127.0.0.1:5001/api').trim(), + token: String(process.env.OPENHW_MCP_TOKEN || '').trim(), + outputJson: 'temp/mcp-scenario-report.json', + outputMd: 'temp/mcp-scenario-report.md', + baseline: '', + }; + + for (let i = 2; i < argv.length; i += 1) { + const arg = String(argv[i] || ''); + if (arg === '--scenario') { + out.scenario = String(argv[i + 1] || ''); + i += 1; + } else if (arg === '--backend-url') { + out.backendUrl = String(argv[i + 1] || '').trim(); + i += 1; + } else if (arg === '--token') { + out.token = String(argv[i + 1] || '').trim(); + i += 1; + } else if (arg === '--output-json') { + out.outputJson = String(argv[i + 1] || ''); + i += 1; + } else if (arg === '--output-md') { + out.outputMd = String(argv[i + 1] || ''); + i += 1; + } else if (arg === '--baseline') { + out.baseline = String(argv[i + 1] || ''); + i += 1; + } + } + + if (!out.scenario) { + throw new Error('Usage: node scripts/mcp-scenario-runner.mjs --scenario [--output-json ] [--output-md ] [--baseline ]'); + } + + return out; +} + +function ensure(condition, message) { + if (!condition) throw new Error(message); +} + +function parseToolPayload(result, toolName) { + const content = Array.isArray(result?.content) ? result.content : []; + const text = content + .filter((entry) => String(entry?.type || '') === 'text') + .map((entry) => String(entry?.text || '')) + .join('\n') + .trim(); + + let parsed = null; + if (text) { + try { + parsed = JSON.parse(text); + } catch { + parsed = null; + } + } + + if (result?.isError) { + throw new Error(`${toolName} returned isError=true${text ? `: ${text}` : ''}`); + } + + ensure(parsed && typeof parsed === 'object', `${toolName} returned empty or non-JSON payload.`); + return parsed; +} + +async function callTool(client, name, args) { + const response = await client.callTool({ + name, + arguments: args, + }); + return parseToolPayload(response, name); +} + +async function readScenario(filePath) { + const absolute = path.isAbsolute(filePath) ? filePath : path.resolve(workspaceRoot, filePath); + const raw = await fs.readFile(absolute, 'utf8'); + const ext = path.extname(absolute).toLowerCase(); + const parsed = ext === '.yaml' || ext === '.yml' ? YAML.parse(raw) : JSON.parse(raw); + + ensure(parsed && typeof parsed === 'object', 'Scenario must be a JSON/YAML object.'); + ensure(String(parsed.name || '').trim(), 'Scenario requires name.'); + ensure(String(parsed.board || '').trim(), 'Scenario requires board.'); + return { + absolute, + data: parsed, + }; +} + +function parseLooseValue(value) { + if (typeof value !== 'string') return value; + const trimmed = value.trim(); + if (!trimmed) return ''; + if (trimmed === 'true') return true; + if (trimmed === 'false') return false; + if (trimmed === 'null') return null; + const n = Number(trimmed); + if (Number.isFinite(n)) return n; + try { + return JSON.parse(trimmed); + } catch { + return trimmed; + } +} + +async function resolveProjectPath(fileField) { + const relative = String(fileField || '').trim(); + ensure(relative, 'project_init did not return file path.'); + if (path.isAbsolute(relative)) return relative; + + const normalized = relative.replace(/\\/g, '/'); + const candidates = [ + path.join(workspaceRoot, normalized), + path.join(cliRoot, normalized), + path.resolve(process.cwd(), normalized), + ]; + + for (const candidate of candidates) { + try { + await fs.access(candidate); + return candidate; + } catch { + continue; + } + } + + return candidates[0]; +} + +async function configureEnvProjectFile(projectFile, envConfig) { + if (!envConfig || typeof envConfig !== 'object') return; + const boardEnv = String(envConfig.boardEnv || '').trim(); + const codeFile = String(envConfig.codeFile || '').trim(); + const source = String(envConfig.source || ''); + if (!boardEnv && !codeFile && !source) return; + + const raw = await fs.readFile(projectFile, 'utf8'); + const project = JSON.parse(raw); + const board = Array.isArray(project?.components) + ? project.components.find((entry) => String(entry?.id || '') === 'board1') + : null; + if (!board) return; + + if (!board.attrs || typeof board.attrs !== 'object' || Array.isArray(board.attrs)) { + board.attrs = {}; + } + + if (boardEnv) { + board.attrs.env = boardEnv; + board.attrs.builder = 'arduino-pico'; + } + + if (codeFile) { + if (!Array.isArray(project.projectFiles)) project.projectFiles = []; + const existing = project.projectFiles.find((entry) => String(entry?.path || '') === codeFile); + if (existing) { + existing.content = source; + existing.dirty = false; + } else { + project.projectFiles.push({ + id: codeFile, + path: codeFile, + name: path.posix.basename(codeFile.replace(/\\/g, '/')), + kind: 'code', + boardId: 'board1', + boardKind: 'rp2040', + content: source, + dirty: false, + }); + } + project.code = source; + project.activeCodeFileId = codeFile; + if (!Array.isArray(project.openCodeTabs)) project.openCodeTabs = []; + if (!project.openCodeTabs.includes(codeFile)) project.openCodeTabs.push(codeFile); + } + + await fs.writeFile(projectFile, `${JSON.stringify(project, null, 2)}\n`, 'utf8'); +} + +function indexTrace(trace) { + const componentTimeline = {}; + const boardPinTimeline = {}; + const serialEvents = []; + + for (const event of Array.isArray(trace) ? trace : []) { + const tMs = Number(event?.tMs || 0); + const boardId = String(event?.boardId || 'default'); + const type = String(event?.type || 'unknown'); + const detail = event?.detail && typeof event.detail === 'object' ? event.detail : {}; + + if (type === 'state') { + const pinKeys = Array.isArray(detail.pinKeys) ? detail.pinKeys.map((entry) => String(entry)) : []; + if (!boardPinTimeline[boardId]) boardPinTimeline[boardId] = []; + boardPinTimeline[boardId].push({ tMs, pinKeys }); + + const components = Array.isArray(detail.components) ? detail.components : []; + for (const component of components) { + const id = String(component?.id || '').trim(); + if (!id) continue; + if (!componentTimeline[id]) componentTimeline[id] = []; + componentTimeline[id].push({ + tMs, + boardId, + stateKeys: Array.isArray(component?.stateKeys) ? component.stateKeys.map((key) => String(key)) : [], + telemetrySummary: String(component?.telemetrySummary || ''), + }); + } + } + + if (type === 'serial') { + serialEvents.push({ + tMs, + boardId, + source: String(detail.source || 'uart0'), + length: Number(detail.length || 0), + preview: typeof detail.data === 'string' ? String(detail.data).slice(0, 120) : '', + }); + } + } + + return { + componentTimeline, + boardPinTimeline, + serialEvents, + }; +} + +function safeText(value) { + return String(value || '').replace(/\|/g, '\\|'); +} + +function buildMarkdownReport(report) { + const lines = []; + lines.push(`# MCP Scenario Report: ${report.scenario.name}`); + lines.push(''); + lines.push(`- Generated: ${report.generatedAt}`); + lines.push(`- Scenario: ${report.scenario.file}`); + lines.push(`- Backend: ${report.backendUrl}`); + lines.push(`- Total cases: ${report.totals.total}`); + lines.push(`- Passed: ${report.totals.passed}`); + lines.push(`- Failed: ${report.totals.failed}`); + if (report.behaviorDiff?.compared) { + lines.push(`- Behavior diff changes: ${report.behaviorDiff.changedCount}`); + } + lines.push(''); + + lines.push('## Case Results'); + lines.push(''); + lines.push('| Case | Env | Status | Components | Wires Planned | Wire Errors | Faults | Trace Events | Serial Events |'); + lines.push('| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |'); + for (const entry of report.results) { + lines.push(`| ${safeText(entry.caseName)} | ${safeText(entry.envKey)} | ${entry.ok ? 'PASS' : 'FAIL'} | ${Number(entry.telemetry?.componentCount || 0)} | ${Number(entry.wiring?.plannedCount || 0)} | ${Number(entry.wiring?.errorCount || 0)} | ${Number(entry.runtime?.faultCount || 0)} | ${Number(entry.trace?.capturedEvents || 0)} | ${Number(entry.serial?.eventCount || 0)} |`); + } + lines.push(''); + + lines.push('## Connectivity Diagnostics'); + lines.push(''); + for (const entry of report.results) { + lines.push(`### ${entry.caseName} / ${entry.envKey}`); + lines.push(`- Planned wires: ${entry.wiring.plannedCount}`); + lines.push(`- Applied wires: ${entry.wiring.appliedCount}`); + if (entry.wiring.errors.length > 0) { + lines.push('- Wire errors:'); + for (const wire of entry.wiring.errors) { + lines.push(` - ${wire.from} -> ${wire.to}: ${wire.error}`); + } + } else { + lines.push('- Wire errors: none'); + } + } + lines.push(''); + + return `${lines.join('\n')}\n`; +} + +async function maybeReadBaseline(filePath) { + if (!filePath) return null; + const absolute = path.isAbsolute(filePath) ? filePath : path.resolve(workspaceRoot, filePath); + const raw = await fs.readFile(absolute, 'utf8'); + return JSON.parse(raw); +} + +function computeBehaviorDiff(report, baseline) { + if (!baseline || !Array.isArray(baseline?.results)) { + return { compared: false, changedCount: 0, changedCases: [] }; + } + + const baselineByKey = new Map( + baseline.results.map((entry) => [`${entry.caseName}::${entry.envKey}`, entry]) + ); + const changedCases = []; + + for (const entry of report.results) { + const key = `${entry.caseName}::${entry.envKey}`; + const before = baselineByKey.get(key); + if (!before) { + changedCases.push({ key, reason: 'missing-baseline-case' }); + continue; + } + + const compareFields = { + telemetryStatusById: entry.telemetry.statusById, + traceEventTypeCounts: entry.trace.eventTypeCounts, + faultCount: entry.runtime.faultCount, + serialEventCount: entry.serial.eventCount, + }; + + const beforeFields = { + telemetryStatusById: before?.telemetry?.statusById || {}, + traceEventTypeCounts: before?.trace?.eventTypeCounts || {}, + faultCount: Number(before?.runtime?.faultCount || 0), + serialEventCount: Number(before?.serial?.eventCount || 0), + }; + + if (JSON.stringify(compareFields) !== JSON.stringify(beforeFields)) { + changedCases.push({ key, reason: 'behavior-diff' }); + } + } + + return { + compared: true, + changedCount: changedCases.length, + changedCases, + }; +} + +async function main() { + const args = parseArgs(process.argv); + const scenarioSpec = await readScenario(args.scenario); + const scenario = scenarioSpec.data; + + const transport = new StdioClientTransport({ + command: npmCommand(), + args: ['run', 'mcp', '--', '--backend-url', args.backendUrl], + cwd: cliRoot, + stderr: 'inherit', + }); + + const client = new Client({ name: 'openhw-mcp-scenario-runner', version: '0.1.0' }); + + const envs = Array.isArray(scenario.envs) && scenario.envs.length > 0 + ? scenario.envs + : [{ key: 'default', boardEnv: '', ms: Number(scenario.durationMs || 1800) }]; + + const components = Array.isArray(scenario.components) ? scenario.components : []; + const wires = Array.isArray(scenario.wires) ? scenario.wires : []; + const inspections = Array.isArray(scenario.inspect) ? scenario.inspect : []; + const traceEventTypes = Array.isArray(scenario.traceEventTypes) + ? scenario.traceEventTypes.map((entry) => String(entry || '')) + : ['state', 'serial', 'fault', 'debug']; + + const report = { + ok: true, + generatedAt: new Date().toISOString(), + backendUrl: args.backendUrl, + scenario: { + name: String(scenario.name), + file: path.relative(workspaceRoot, scenarioSpec.absolute).replace(/\\/g, '/'), + board: String(scenario.board), + }, + results: [], + }; + + try { + await client.connect(transport); + + await callTool(client, 'component_catalog', { + ...(args.token ? { token: args.token } : {}), + }); + + for (const envConfig of envs) { + const envKey = String(envConfig.key || envConfig.boardEnv || 'default'); + const durationMs = Number(envConfig.ms || scenario.durationMs || 1800); + + const initPayload = await callTool(client, 'project_init', { + name: `${scenario.name}-${envKey}-${Date.now()}`, + board: String(scenario.board), + ...(args.token ? { token: args.token } : {}), + }); + + const projectFile = await resolveProjectPath(initPayload.file); + await configureEnvProjectFile(projectFile, envConfig); + + const addedComponents = []; + for (const component of components) { + const added = await callTool(client, 'component_add', { + type: String(component.type || ''), + ...(component.id ? { id: String(component.id) } : {}), + ...(Number.isFinite(Number(component.x)) ? { x: Number(component.x) } : {}), + ...(Number.isFinite(Number(component.y)) ? { y: Number(component.y) } : {}), + ...(args.token ? { token: args.token } : {}), + }); + addedComponents.push(added?.component || null); + } + + const wiringValidation = await callTool(client, 'wiring_validate', { + wires: wires.map((wire) => ({ + from: String(wire.from || ''), + to: String(wire.to || ''), + })), + ...(args.token ? { token: args.token } : {}), + }); + + const wireErrors = []; + let appliedCount = 0; + for (const wire of wires) { + try { + await callTool(client, 'wire_add', { + from: String(wire.from || ''), + to: String(wire.to || ''), + ...(args.token ? { token: args.token } : {}), + }); + appliedCount += 1; + } catch (error) { + wireErrors.push({ + from: String(wire.from || ''), + to: String(wire.to || ''), + error: String(error?.message || error), + }); + } + } + + const executePayload = await callTool(client, 'sim_execute', { + ms: durationMs, + all_boards: true, + include_telemetry: true, + include_trace: true, + include_console: true, + include_state: true, + include_serial_text: true, + trace_event_types: traceEventTypes, + ...(args.token ? { token: args.token } : {}), + }); + + const trace = Array.isArray(executePayload.trace) ? executePayload.trace : []; + const telemetry = executePayload.telemetry || executePayload.result?.telemetry || { components: [] }; + const telemetryComponents = Array.isArray(telemetry?.components) ? telemetry.components : []; + + const telemetryStatusById = {}; + for (const component of telemetryComponents) { + telemetryStatusById[String(component?.id || '')] = String(component?.status || 'unknown'); + } + + const traceIndexed = indexTrace(trace); + const eventTypeCounts = {}; + for (const event of trace) { + const type = String(event?.type || 'unknown'); + eventTypeCounts[type] = Number(eventTypeCounts[type] || 0) + 1; + } + + const inspectResults = []; + for (const inspect of inspections) { + const inspectId = String(inspect.id || '').trim(); + if (!inspectId) continue; + + const inspectArgs = { + id: inspectId, + ms: Number(inspect.ms || Math.max(800, Math.floor(durationMs / 2))), + all_boards: true, + include_trace: !!inspect.includeTrace, + include_console: !!inspect.includeConsole, + ...(inspect.event !== undefined ? { event: inspect.event } : {}), + ...(inspect.value !== undefined ? { value: parseLooseValue(inspect.value) } : {}), + ...(Number.isFinite(Number(inspect.atMs)) ? { at_ms: Number(inspect.atMs) } : {}), + ...(args.token ? { token: args.token } : {}), + }; + + const inspectPayload = await callTool(client, 'sim_inspect', inspectArgs); + inspectResults.push({ + id: inspectId, + ok: !!inspectPayload.ok, + component: inspectPayload.component || null, + eventProbe: inspectPayload.eventProbe || null, + }); + } + + report.results.push({ + caseName: String(scenario.name), + envKey, + durationMs, + projectFile: path.relative(workspaceRoot, projectFile).replace(/\\/g, '/'), + addedComponents: addedComponents.filter(Boolean).map((entry) => ({ id: entry.id, type: entry.type })), + wiring: { + plannedCount: wires.length, + appliedCount, + errorCount: wireErrors.length, + errors: wireErrors, + diagnostics: wiringValidation?.diagnostics || [], + }, + runtime: { + elapsedMs: Number(executePayload?.result?.elapsedMs || 0), + faultCount: Number(executePayload?.result?.faultCount || 0), + serialChars: Number(executePayload?.result?.serialChars || 0), + }, + telemetry: { + componentCount: telemetryComponents.length, + statusById: telemetryStatusById, + }, + trace: { + capturedEvents: trace.length, + droppedEvents: Number(executePayload?.traceSummary?.droppedEvents || 0), + eventTypeCounts, + componentTimeline: traceIndexed.componentTimeline, + boardPinTimeline: traceIndexed.boardPinTimeline, + }, + serial: { + eventCount: traceIndexed.serialEvents.length, + events: traceIndexed.serialEvents, + consoleLength: Number(executePayload?.console?.length || 0), + }, + inspect: inspectResults, + ok: wireErrors.length === 0, + }); + } + } finally { + await client.close(); + } + + report.totals = { + total: report.results.length, + passed: report.results.filter((entry) => entry.ok).length, + failed: report.results.filter((entry) => !entry.ok).length, + }; + report.ok = report.totals.failed === 0; + + const baseline = await maybeReadBaseline(args.baseline); + report.behaviorDiff = computeBehaviorDiff(report, baseline); + + if (report.behaviorDiff.compared && report.behaviorDiff.changedCount > 0) { + report.ok = false; + } + + const outputJsonAbsolute = path.isAbsolute(args.outputJson) + ? args.outputJson + : path.resolve(workspaceRoot, args.outputJson); + const outputMdAbsolute = path.isAbsolute(args.outputMd) + ? args.outputMd + : path.resolve(workspaceRoot, args.outputMd); + + await fs.mkdir(path.dirname(outputJsonAbsolute), { recursive: true }); + await fs.mkdir(path.dirname(outputMdAbsolute), { recursive: true }); + + await fs.writeFile(outputJsonAbsolute, `${JSON.stringify(report, null, 2)}\n`, 'utf8'); + await fs.writeFile(outputMdAbsolute, buildMarkdownReport(report), 'utf8'); + + console.log(`[mcp-scenario-runner] json=${path.relative(workspaceRoot, outputJsonAbsolute).replace(/\\/g, '/')}`); + console.log(`[mcp-scenario-runner] md=${path.relative(workspaceRoot, outputMdAbsolute).replace(/\\/g, '/')}`); + console.log(`[mcp-scenario-runner] total=${report.totals.total} passed=${report.totals.passed} failed=${report.totals.failed}`); + if (report.behaviorDiff?.compared) { + console.log(`[mcp-scenario-runner] behavior-diff changed=${report.behaviorDiff.changedCount}`); + } + + if (!report.ok) { + process.exitCode = 1; + } +} + +main().catch((error) => { + console.error(`[mcp-scenario-runner] FAIL ${String(error?.message || error)}`); + process.exit(1); +}); diff --git a/openhw-studio-cli-danish/scripts/mcp-smoke.mjs b/openhw-studio-cli-danish/scripts/mcp-smoke.mjs index b1373e6..411fc4a 100644 --- a/openhw-studio-cli-danish/scripts/mcp-smoke.mjs +++ b/openhw-studio-cli-danish/scripts/mcp-smoke.mjs @@ -64,7 +64,17 @@ async function main() { const listed = await client.listTools(); const toolNames = new Set((listed?.tools || []).map((tool) => String(tool?.name || ''))); - const requiredTools = ['project_init', 'sim_execute', 'sim_trace', 'sim_inspect']; + const requiredTools = [ + 'project_init', + 'project_open', + 'project_status', + 'project_validate', + 'component_catalog', + 'wiring_validate', + 'sim_execute', + 'sim_trace', + 'sim_inspect', + ]; for (const toolName of requiredTools) { assert(toolNames.has(toolName), `Missing MCP tool: ${toolName}`); } diff --git a/openhw-studio-cli-danish/scripts/test-mcp-contracts.mjs b/openhw-studio-cli-danish/scripts/test-mcp-contracts.mjs new file mode 100644 index 0000000..0cbb7be --- /dev/null +++ b/openhw-studio-cli-danish/scripts/test-mcp-contracts.mjs @@ -0,0 +1,132 @@ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const cliRoot = path.resolve(__dirname, '..'); + +const backendUrl = String(process.env.MCP_TEST_BACKEND_URL || 'http://127.0.0.1:5001/api').trim(); +const token = String(process.env.OPENHW_MCP_TOKEN || '').trim(); + +function npmCommand() { + return process.platform === 'win32' ? 'npm.cmd' : 'npm'; +} + +function assert(condition, message) { + if (!condition) throw new Error(message); +} + +function parseToolPayload(result, toolName) { + const content = Array.isArray(result?.content) ? result.content : []; + const text = content + .filter((entry) => String(entry?.type || '') === 'text') + .map((entry) => String(entry?.text || '')) + .join('\n') + .trim(); + + let parsed = null; + if (text) { + try { + parsed = JSON.parse(text); + } catch { + parsed = null; + } + } + + return { + isError: !!result?.isError, + parsed, + text, + toolName, + }; +} + +async function callTool(client, name, args) { + const result = await client.callTool({ name, arguments: args }); + return parseToolPayload(result, name); +} + +async function main() { + const transport = new StdioClientTransport({ + command: npmCommand(), + args: ['run', 'mcp', '--', '--backend-url', backendUrl], + cwd: cliRoot, + stderr: 'inherit', + }); + + const client = new Client({ name: 'openhw-mcp-contracts', version: '0.1.0' }); + + try { + await client.connect(transport); + + const listed = await client.listTools(); + const toolNames = new Set((listed?.tools || []).map((tool) => String(tool?.name || ''))); + + for (const requiredTool of [ + 'project_init', + 'project_open', + 'project_status', + 'project_validate', + 'component_catalog', + 'wiring_validate', + 'sim_execute', + 'sim_trace', + 'sim_inspect', + ]) { + assert(toolNames.has(requiredTool), `Missing MCP tool ${requiredTool}`); + } + + const statusBefore = await callTool(client, 'project_status', { + ...(token ? { token } : {}), + }); + assert(!statusBefore.isError, 'project_status should not error without active project.'); + assert(statusBefore.parsed?.ok === true, 'project_status should return ok=true.'); + + const init = await callTool(client, 'project_init', { + name: `contracts-${Date.now()}`, + board: 'wokwi-raspberry-pi-pico', + ...(token ? { token } : {}), + }); + assert(!init.isError, 'project_init returned error.'); + assert(init.parsed?.ok === true, 'project_init should return ok=true.'); + + const validate = await callTool(client, 'project_validate', { + ...(token ? { token } : {}), + }); + assert(!validate.isError, 'project_validate returned error.'); + assert(typeof validate.parsed?.ok === 'boolean', 'project_validate ok missing.'); + + const catalog = await callTool(client, 'component_catalog', { + ...(token ? { token } : {}), + }); + assert(!catalog.isError, 'component_catalog returned error.'); + assert(Array.isArray(catalog.parsed?.components), 'component_catalog components missing.'); + + const invalidWire = await callTool(client, 'wiring_validate', { + from: 'board1:NOT_A_PIN', + to: 'board1:GND', + ...(token ? { token } : {}), + }); + assert(!invalidWire.isError, 'wiring_validate contract response should not transport-error.'); + assert(invalidWire.parsed?.ok === false, 'wiring_validate invalid wire should return ok=false.'); + assert(Array.isArray(invalidWire.parsed?.diagnostics), 'wiring_validate diagnostics missing.'); + + const simInspectNegative = await callTool(client, 'sim_inspect', { + event: 'press', + ms: 500, + ...(token ? { token } : {}), + }); + assert(simInspectNegative.isError, 'sim_inspect should return isError for event injection without id.'); + + console.log('[mcp-contracts] PASS'); + } finally { + await client.close(); + } +} + +main().catch((error) => { + console.error(`[mcp-contracts] FAIL ${String(error?.message || error)}`); + process.exit(1); +}); diff --git a/openhw-studio-cli-danish/src/mcp/server.ts b/openhw-studio-cli-danish/src/mcp/server.ts index 7372ec0..5076f1b 100644 --- a/openhw-studio-cli-danish/src/mcp/server.ts +++ b/openhw-studio-cli-danish/src/mcp/server.ts @@ -10,9 +10,11 @@ import { loadProject, saveProject, summarizeProject, + validateProject, } from '../utils/project.js'; import { relToCwd, resolveWorkspacePath } from '../utils/paths.js'; import { startSimulation } from '../sim/session.js'; +import { getPinsForType, listManifestInfos } from '../utils/manifests.js'; export interface McpServerConfig { backendUrl: string; @@ -380,6 +382,52 @@ async function requireActiveProject(session: ActiveProjectSession): Promise<{ pr }; } +function parseEndpoint(endpoint: string): { componentId: string; pinId: string } { + const [componentId, pinId] = String(endpoint || '').split(':'); + if (!componentId || !pinId) { + throw new Error(`Invalid endpoint format: ${endpoint}. Expected :.`); + } + return { componentId, pinId }; +} + +async function validateConnectionInProject(project: OpenHwProject, from: string, to: string): Promise<{ + from: string; + to: string; + valid: boolean; + issues: string[]; +}> { + const issues: string[] = []; + const endpoints = [from, to]; + + for (const endpoint of endpoints) { + let parsed: { componentId: string; pinId: string }; + try { + parsed = parseEndpoint(endpoint); + } catch (error) { + issues.push(String((error as Error).message || error)); + continue; + } + + const component = project.components.find((entry) => entry.id === parsed.componentId) || null; + if (!component) { + issues.push(`Component not found for endpoint: ${endpoint}`); + continue; + } + + const pins = await getPinsForType(component.type); + if (pins && pins.size > 0 && !pins.has(parsed.pinId)) { + issues.push(`Pin ${parsed.pinId} does not exist on component type ${component.type}.`); + } + } + + return { + from, + to, + valid: issues.length === 0, + issues, + }; +} + function buildInteractionEvent(event: unknown, value: unknown): unknown { if (event && typeof event === 'object' && !Array.isArray(event)) { return event; @@ -465,6 +513,142 @@ export async function runMcpServer(config: McpServerConfig): Promise { version: '0.1.0', }); + server.tool( + 'project_open', + 'Load an existing OpenHW project JSON and set it as active session project.', + { + file: z.string().min(1), + token: z.string().optional(), + }, + async ({ file, token }) => { + assertToken(config, token); + + const projectFile = resolveWorkspacePath(String(file)); + const project = await loadProject(projectFile); + session.projectFile = projectFile; + + return makeToolResult({ + ok: true, + action: 'project_open', + file: relToCwd(projectFile), + summary: summarizeProject(project), + }); + } + ); + + server.tool( + 'project_status', + 'Return current MCP active project session status.', + { + token: z.string().optional(), + }, + async ({ token }) => { + assertToken(config, token); + + if (!session.projectFile) { + return makeToolResult({ + ok: true, + action: 'project_status', + active: false, + file: null, + }); + } + + const project = await loadProject(session.projectFile); + return makeToolResult({ + ok: true, + action: 'project_status', + active: true, + file: relToCwd(session.projectFile), + summary: summarizeProject(project), + }); + } + ); + + server.tool( + 'project_validate', + 'Validate active project schema and references.', + { + token: z.string().optional(), + }, + async ({ token }) => { + assertToken(config, token); + + const { project, projectFile } = await requireActiveProject(session); + const validation = await validateProject(project); + return makeToolResult({ + ok: validation.valid, + action: 'project_validate', + file: relToCwd(projectFile), + ...validation, + }); + } + ); + + server.tool( + 'component_catalog', + 'List known component manifest capabilities (pins, group, onEvent, telemetry keys).', + { + token: z.string().optional(), + }, + async ({ token }) => { + assertToken(config, token); + const manifests = await listManifestInfos(); + return makeToolResult({ + ok: true, + action: 'component_catalog', + count: manifests.length, + components: manifests.map((entry) => ({ + type: entry.type, + label: entry.label, + group: entry.group, + pins: entry.pins.map((pin) => pin.id), + hasOnEvent: entry.hasOnEvent, + telemetry: entry.telemetry || null, + })), + }); + } + ); + + server.tool( + 'wiring_validate', + 'Validate one or more proposed wires against the active project without mutating it.', + { + from: z.string().optional(), + to: z.string().optional(), + wires: z.array(z.object({ from: z.string().min(1), to: z.string().min(1) })).optional(), + token: z.string().optional(), + }, + async ({ from, to, wires, token }) => { + assertToken(config, token); + + const { project, projectFile } = await requireActiveProject(session); + const plannedWires = Array.isArray(wires) && wires.length > 0 + ? wires.map((entry) => ({ from: String(entry.from), to: String(entry.to) })) + : [{ + from: String(from || ''), + to: String(to || ''), + }]; + + if (plannedWires.some((entry) => !entry.from || !entry.to)) { + throw new Error('wiring_validate requires either --from/--to or non-empty wires[].'); + } + + const diagnostics = await Promise.all( + plannedWires.map((entry) => validateConnectionInProject(project, entry.from, entry.to)) + ); + const valid = diagnostics.every((entry) => entry.valid); + + return makeToolResult({ + ok: valid, + action: 'wiring_validate', + file: relToCwd(projectFile), + valid, + diagnostics, + }); + } + ); + server.tool( 'project_init', 'Create a new OpenHW project JSON and set it as active session project.', From 653c21011d484c92923fc0b77dc749aba630842c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:29:02 +0000 Subject: [PATCH 06/15] fix: add dry-run scenario mode and sample wiring correction Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/1c6315b7-83b3-45c1-a76c-fff9e56a76c3 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- openhw-studio-cli-danish/README.md | 5 +- openhw-studio-cli-danish/package.json | 3 +- .../scenarios/pico-led-lifecycle.yaml | 2 +- .../scripts/mcp-scenario-runner.mjs | 65 +++++++++++++++---- 4 files changed, 60 insertions(+), 15 deletions(-) diff --git a/openhw-studio-cli-danish/README.md b/openhw-studio-cli-danish/README.md index c610250..bded52d 100644 --- a/openhw-studio-cli-danish/README.md +++ b/openhw-studio-cli-danish/README.md @@ -150,9 +150,12 @@ npm run cli -- lib sync-project temp/project.json --dry-run # MCP response contract coverage (positive + negative paths) npm run test:mcp:contracts -# Deterministic scenario runner (YAML/JSON manifest) with unified behavior report export +# Deterministic scenario runner dry-run (YAML/JSON manifest parse + wiring diagnostics + report export) npm run test:mcp:scenario +# Full runtime scenario execution (requires simulation dependencies/backend) +npm run test:mcp:scenario:full + # Override scenario/report paths node scripts/mcp-scenario-runner.mjs \ --scenario scenarios/pico-led-lifecycle.yaml \ diff --git a/openhw-studio-cli-danish/package.json b/openhw-studio-cli-danish/package.json index a3503e8..ceb4106 100644 --- a/openhw-studio-cli-danish/package.json +++ b/openhw-studio-cli-danish/package.json @@ -13,7 +13,8 @@ "test:mcp:pico-all-components": "node scripts/test-mcp-pico-all-components.mjs", "test:mcp:pico-components-individual": "node scripts/test-mcp-pico-components-individual.mjs", "test:mcp:contracts": "node scripts/test-mcp-contracts.mjs", - "test:mcp:scenario": "node scripts/mcp-scenario-runner.mjs --scenario scenarios/pico-led-lifecycle.yaml" + "test:mcp:scenario": "node scripts/mcp-scenario-runner.mjs --scenario scenarios/pico-led-lifecycle.yaml --dry-run", + "test:mcp:scenario:full": "node scripts/mcp-scenario-runner.mjs --scenario scenarios/pico-led-lifecycle.yaml" }, "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", diff --git a/openhw-studio-cli-danish/scenarios/pico-led-lifecycle.yaml b/openhw-studio-cli-danish/scenarios/pico-led-lifecycle.yaml index 35b4e1d..c2010d9 100644 --- a/openhw-studio-cli-danish/scenarios/pico-led-lifecycle.yaml +++ b/openhw-studio-cli-danish/scenarios/pico-led-lifecycle.yaml @@ -39,7 +39,7 @@ wires: - from: board1:GP2 to: led1:A - from: board1:GND - to: led1:C + to: led1:K inspect: - id: led1 ms: 1200 diff --git a/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs b/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs index fd5129b..e2fa35e 100644 --- a/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs +++ b/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs @@ -22,6 +22,7 @@ function parseArgs(argv) { outputJson: 'temp/mcp-scenario-report.json', outputMd: 'temp/mcp-scenario-report.md', baseline: '', + dryRun: false, }; for (let i = 2; i < argv.length; i += 1) { @@ -44,6 +45,8 @@ function parseArgs(argv) { } else if (arg === '--baseline') { out.baseline = String(argv[i + 1] || ''); i += 1; + } else if (arg === '--dry-run') { + out.dryRun = true; } } @@ -92,7 +95,23 @@ async function callTool(client, name, args) { } async function readScenario(filePath) { - const absolute = path.isAbsolute(filePath) ? filePath : path.resolve(workspaceRoot, filePath); + const absolute = await (async () => { + if (path.isAbsolute(filePath)) return filePath; + const candidates = [ + path.resolve(process.cwd(), filePath), + path.resolve(cliRoot, filePath), + path.resolve(workspaceRoot, filePath), + ]; + for (const candidate of candidates) { + try { + await fs.access(candidate); + return candidate; + } catch { + continue; + } + } + return candidates[0]; + })(); const raw = await fs.readFile(absolute, 'utf8'); const ext = path.extname(absolute).toLowerCase(); const parsed = ext === '.yaml' || ext === '.yml' ? YAML.parse(raw) : JSON.parse(raw); @@ -439,17 +458,35 @@ async function main() { } } - const executePayload = await callTool(client, 'sim_execute', { - ms: durationMs, - all_boards: true, - include_telemetry: true, - include_trace: true, - include_console: true, - include_state: true, - include_serial_text: true, - trace_event_types: traceEventTypes, - ...(args.token ? { token: args.token } : {}), - }); + const executePayload = args.dryRun + ? { + result: { + elapsedMs: 0, + faultCount: 0, + serialChars: 0, + }, + telemetry: { + components: [], + }, + trace: [], + traceSummary: { + droppedEvents: 0, + }, + console: { + length: 0, + }, + } + : await callTool(client, 'sim_execute', { + ms: durationMs, + all_boards: true, + include_telemetry: true, + include_trace: true, + include_console: true, + include_state: true, + include_serial_text: true, + trace_event_types: traceEventTypes, + ...(args.token ? { token: args.token } : {}), + }); const trace = Array.isArray(executePayload.trace) ? executePayload.trace : []; const telemetry = executePayload.telemetry || executePayload.result?.telemetry || { components: [] }; @@ -469,6 +506,7 @@ async function main() { const inspectResults = []; for (const inspect of inspections) { + if (args.dryRun) break; const inspectId = String(inspect.id || '').trim(); if (!inspectId) continue; @@ -565,6 +603,9 @@ async function main() { console.log(`[mcp-scenario-runner] json=${path.relative(workspaceRoot, outputJsonAbsolute).replace(/\\/g, '/')}`); console.log(`[mcp-scenario-runner] md=${path.relative(workspaceRoot, outputMdAbsolute).replace(/\\/g, '/')}`); console.log(`[mcp-scenario-runner] total=${report.totals.total} passed=${report.totals.passed} failed=${report.totals.failed}`); + if (args.dryRun) { + console.log('[mcp-scenario-runner] mode=dry-run (simulation execution skipped)'); + } if (report.behaviorDiff?.compared) { console.log(`[mcp-scenario-runner] behavior-diff changed=${report.behaviorDiff.changedCount}`); } From 6b5b6620ef41b6192b0d2e70f809c98c08073cf3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:31:31 +0000 Subject: [PATCH 07/15] chore: address validation feedback for MCP diagnostics updates Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/1c6315b7-83b3-45c1-a76c-fff9e56a76c3 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs | 6 ++++-- openhw-studio-cli-danish/src/mcp/server.ts | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs b/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs index e2fa35e..dc3e6fb 100644 --- a/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs +++ b/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs @@ -51,7 +51,7 @@ function parseArgs(argv) { } if (!out.scenario) { - throw new Error('Usage: node scripts/mcp-scenario-runner.mjs --scenario [--output-json ] [--output-md ] [--baseline ]'); + throw new Error('Usage: node scripts/mcp-scenario-runner.mjs --scenario [--output-json ] [--output-md ] [--baseline ] [--dry-run]'); } return out; @@ -264,7 +264,9 @@ function indexTrace(trace) { } function safeText(value) { - return String(value || '').replace(/\|/g, '\\|'); + return String(value || '') + .replace(/\\/g, '\\\\') + .replace(/\|/g, '\\|'); } function buildMarkdownReport(report) { diff --git a/openhw-studio-cli-danish/src/mcp/server.ts b/openhw-studio-cli-danish/src/mcp/server.ts index 5076f1b..80348f3 100644 --- a/openhw-studio-cli-danish/src/mcp/server.ts +++ b/openhw-studio-cli-danish/src/mcp/server.ts @@ -404,7 +404,11 @@ async function validateConnectionInProject(project: OpenHwProject, from: string, try { parsed = parseEndpoint(endpoint); } catch (error) { - issues.push(String((error as Error).message || error)); + const asError = error as { name?: string; code?: string; message?: string }; + const errorName = String(asError?.name || 'Error'); + const errorCode = String(asError?.code || '').trim(); + const errorMessage = String(asError?.message || error || 'Unknown error'); + issues.push(errorCode ? `${errorName}(${errorCode}): ${errorMessage}` : `${errorName}: ${errorMessage}`); continue; } @@ -631,7 +635,7 @@ export async function runMcpServer(config: McpServerConfig): Promise { }]; if (plannedWires.some((entry) => !entry.from || !entry.to)) { - throw new Error('wiring_validate requires either --from/--to or non-empty wires[].'); + throw new Error('wiring_validate requires either from/to fields or a non-empty wires[] array.'); } const diagnostics = await Promise.all( From 1e8ad57a9f1512ac569611aca2e390da224a06c9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:33:36 +0000 Subject: [PATCH 08/15] refactor: polish scenario diffing and endpoint parser naming Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/1c6315b7-83b3-45c1-a76c-fff9e56a76c3 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- .../scripts/mcp-scenario-runner.mjs | 19 +++++++++++++++++-- openhw-studio-cli-danish/src/mcp/server.ts | 4 ++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs b/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs index dc3e6fb..0d5a559 100644 --- a/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs +++ b/openhw-studio-cli-danish/scripts/mcp-scenario-runner.mjs @@ -9,6 +9,7 @@ const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const cliRoot = path.resolve(__dirname, '..'); const workspaceRoot = path.resolve(cliRoot, '..'); +const DEFAULT_INSPECT_MIN_MS = 800; function npmCommand() { return process.platform === 'win32' ? 'npm.cmd' : 'npm'; @@ -352,7 +353,21 @@ function computeBehaviorDiff(report, baseline) { serialEventCount: Number(before?.serial?.eventCount || 0), }; - if (JSON.stringify(compareFields) !== JSON.stringify(beforeFields)) { + const normalizeForCompare = (value) => { + if (Array.isArray(value)) { + return value.map((entry) => normalizeForCompare(entry)); + } + if (value && typeof value === 'object') { + const out = {}; + for (const key of Object.keys(value).sort((a, b) => a.localeCompare(b))) { + out[key] = normalizeForCompare(value[key]); + } + return out; + } + return value; + }; + + if (JSON.stringify(normalizeForCompare(compareFields)) !== JSON.stringify(normalizeForCompare(beforeFields))) { changedCases.push({ key, reason: 'behavior-diff' }); } } @@ -514,7 +529,7 @@ async function main() { const inspectArgs = { id: inspectId, - ms: Number(inspect.ms || Math.max(800, Math.floor(durationMs / 2))), + ms: Number(inspect.ms || Math.max(DEFAULT_INSPECT_MIN_MS, Math.floor(durationMs / 2))), all_boards: true, include_trace: !!inspect.includeTrace, include_console: !!inspect.includeConsole, diff --git a/openhw-studio-cli-danish/src/mcp/server.ts b/openhw-studio-cli-danish/src/mcp/server.ts index 80348f3..a23d33a 100644 --- a/openhw-studio-cli-danish/src/mcp/server.ts +++ b/openhw-studio-cli-danish/src/mcp/server.ts @@ -382,7 +382,7 @@ async function requireActiveProject(session: ActiveProjectSession): Promise<{ pr }; } -function parseEndpoint(endpoint: string): { componentId: string; pinId: string } { +function parseWireEndpoint(endpoint: string): { componentId: string; pinId: string } { const [componentId, pinId] = String(endpoint || '').split(':'); if (!componentId || !pinId) { throw new Error(`Invalid endpoint format: ${endpoint}. Expected :.`); @@ -402,7 +402,7 @@ async function validateConnectionInProject(project: OpenHwProject, from: string, for (const endpoint of endpoints) { let parsed: { componentId: string; pinId: string }; try { - parsed = parseEndpoint(endpoint); + parsed = parseWireEndpoint(endpoint); } catch (error) { const asError = error as { name?: string; code?: string; message?: string }; const errorName = String(asError?.name || 'Error'); From a720fe3d3fa2c50fa8c984e176144b06392e652f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Apr 2026 20:28:41 +0000 Subject: [PATCH 09/15] feat: add block-coding CLI commands and automated test Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/9d176861-fc3f-41fa-b649-0f6ab95ce5d3 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- openhw-studio-cli-danish/README.md | 7 ++ openhw-studio-cli-danish/package.json | 3 +- .../scripts/test-cli-block-coding.mjs | 111 ++++++++++++++++++ .../src/commands/project.ts | 82 +++++++++++++ openhw-studio-cli-danish/src/types.ts | 3 + openhw-studio-cli-danish/src/utils/project.ts | 8 ++ 6 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 openhw-studio-cli-danish/scripts/test-cli-block-coding.mjs diff --git a/openhw-studio-cli-danish/README.md b/openhw-studio-cli-danish/README.md index bded52d..f004afa 100644 --- a/openhw-studio-cli-danish/README.md +++ b/openhw-studio-cli-danish/README.md @@ -53,6 +53,10 @@ npm run cli -- project connect temp/project.json --from board1:GND --to led1:C # Update board code (inline or file) npm run cli -- project set-code temp/project.json --board-id board1 --code-file examples/blink.ino +# Set block-coding metadata (Blockly XML + generated code preference) +npm run cli -- project set-blockly temp/project.json --xml-file temp/workspace.xml --generated-code-file temp/generated.cpp --use-blockly-code true +npm run cli -- project block-summary temp/project.json + # Update project/library.txt from a text file npm run cli -- project set-library-file temp/project.json --input temp/libs.txt ``` @@ -150,6 +154,9 @@ npm run cli -- lib sync-project temp/project.json --dry-run # MCP response contract coverage (positive + negative paths) npm run test:mcp:contracts +# Validate block-coding project workflow through CLI +npm run test:cli:block-coding + # Deterministic scenario runner dry-run (YAML/JSON manifest parse + wiring diagnostics + report export) npm run test:mcp:scenario diff --git a/openhw-studio-cli-danish/package.json b/openhw-studio-cli-danish/package.json index ceb4106..db05686 100644 --- a/openhw-studio-cli-danish/package.json +++ b/openhw-studio-cli-danish/package.json @@ -14,7 +14,8 @@ "test:mcp:pico-components-individual": "node scripts/test-mcp-pico-components-individual.mjs", "test:mcp:contracts": "node scripts/test-mcp-contracts.mjs", "test:mcp:scenario": "node scripts/mcp-scenario-runner.mjs --scenario scenarios/pico-led-lifecycle.yaml --dry-run", - "test:mcp:scenario:full": "node scripts/mcp-scenario-runner.mjs --scenario scenarios/pico-led-lifecycle.yaml" + "test:mcp:scenario:full": "node scripts/mcp-scenario-runner.mjs --scenario scenarios/pico-led-lifecycle.yaml", + "test:cli:block-coding": "node scripts/test-cli-block-coding.mjs" }, "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", diff --git a/openhw-studio-cli-danish/scripts/test-cli-block-coding.mjs b/openhw-studio-cli-danish/scripts/test-cli-block-coding.mjs new file mode 100644 index 0000000..26b6571 --- /dev/null +++ b/openhw-studio-cli-danish/scripts/test-cli-block-coding.mjs @@ -0,0 +1,111 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawn } from 'node:child_process'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const cliRoot = path.resolve(__dirname, '..'); +const workspaceRoot = path.resolve(cliRoot, '..'); + +function tsxCommand() { + return process.platform === 'win32' + ? path.join(cliRoot, 'node_modules', '.bin', 'tsx.cmd') + : path.join(cliRoot, 'node_modules', '.bin', 'tsx'); +} + +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +function parseJsonOutput(output, context) { + try { + return JSON.parse(output.trim()); + } catch (error) { + throw new Error(`${context} did not return valid JSON. Output: ${output}\n${String(error)}`); + } +} + +async function runCli(args) { + return new Promise((resolve, reject) => { + const child = spawn(tsxCommand(), ['src/cli.ts', ...args], { + cwd: cliRoot, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (chunk) => { + stdout += String(chunk); + }); + + child.stderr.on('data', (chunk) => { + stderr += String(chunk); + }); + + child.on('error', reject); + + child.on('close', (code) => { + if (code !== 0) { + reject(new Error(`CLI command failed (exit=${code}): ${args.join(' ')}\nSTDOUT:\n${stdout}\nSTDERR:\n${stderr}`)); + return; + } + resolve({ stdout, stderr }); + }); + }); +} + +async function main() { + const tempDir = path.join(workspaceRoot, 'temp'); + await fs.mkdir(tempDir, { recursive: true }); + + const projectPath = path.join(tempDir, `block-cli-${Date.now()}.json`); + const xmlPath = path.join(tempDir, `block-cli-${Date.now()}-workspace.xml`); + + const blockXml = ''; + const generatedCode = 'void setup(){}\nvoid loop(){}\n'; + + await fs.writeFile(xmlPath, blockXml, 'utf8'); + + await runCli(['project', 'init', projectPath, '--name', 'block-cli', '--board', 'rp2040']); + + const setOutput = await runCli([ + 'project', + 'set-blockly', + projectPath, + '--xml-file', + xmlPath, + '--generated-code', + generatedCode, + '--use-blockly-code', + 'true', + ]); + + const setPayload = parseJsonOutput(setOutput.stdout, 'project set-blockly'); + assert(setPayload.ok === true, 'project set-blockly should return ok=true'); + assert(setPayload.blockly?.useBlocklyCode === true, 'useBlocklyCode should be true after set-blockly'); + assert(Number(setPayload.blockly?.xmlLength || 0) > 0, 'xmlLength should be > 0'); + + const summaryOutput = await runCli(['project', 'block-summary', projectPath]); + const summaryPayload = parseJsonOutput(summaryOutput.stdout, 'project block-summary'); + assert(summaryPayload.ok === true, 'project block-summary should return ok=true'); + assert(summaryPayload.blockly?.hasXml === true, 'block-summary should report hasXml=true'); + assert(summaryPayload.blockly?.hasGeneratedCode === true, 'block-summary should report hasGeneratedCode=true'); + + const exportOutput = await runCli(['project', 'export-json', projectPath]); + const projectPayload = parseJsonOutput(exportOutput.stdout, 'project export-json'); + + assert(projectPayload.blocklyXml === blockXml, 'exported project should preserve blocklyXml'); + assert(projectPayload.blocklyGeneratedCode === generatedCode, 'exported project should preserve blocklyGeneratedCode'); + assert(projectPayload.useBlocklyCode === true, 'exported project should preserve useBlocklyCode=true'); + + console.log('[cli-block-coding] PASS'); +} + +main().catch((error) => { + console.error(`[cli-block-coding] FAIL ${String(error?.message || error)}`); + process.exit(1); +}); diff --git a/openhw-studio-cli-danish/src/commands/project.ts b/openhw-studio-cli-danish/src/commands/project.ts index 3630cf0..57c7590 100644 --- a/openhw-studio-cli-danish/src/commands/project.ts +++ b/openhw-studio-cli-danish/src/commands/project.ts @@ -33,6 +33,15 @@ function parseAttrs(attrsJson?: string): Record { } } +function parseBooleanInput(value: unknown, fallback: boolean): boolean { + if (typeof value === 'boolean') return value; + const text = String(value ?? '').trim().toLowerCase(); + if (!text.length) return fallback; + if (['1', 'true', 'yes', 'y', 'on'].includes(text)) return true; + if (['0', 'false', 'no', 'n', 'off'].includes(text)) return false; + throw new Error(`Invalid boolean value: ${String(value)} (expected true/false)`); +} + async function resolveAttrsInput(options: { attrsJson?: string; attrsFile?: string; @@ -258,6 +267,79 @@ export function registerProjectCommands(program: Command): void { }); }); + project + .command('set-blockly ') + .description('Set Blockly XML/generated code metadata for the project') + .option('--xml ', 'Inline Blockly XML payload') + .option('--xml-file ', 'Read Blockly XML from file') + .option('--generated-code ', 'Inline generated code from block workflow') + .option('--generated-code-file ', 'Read generated code from file') + .option('--use-blockly-code ', 'Whether generated Blockly code should be preferred') + .option('-o, --output ', 'Write to different output file') + .action(async (projectFile: string, options: any) => { + const hasXmlInline = typeof options.xml === 'string'; + const hasXmlFile = typeof options.xmlFile === 'string'; + if (hasXmlInline && hasXmlFile) { + throw new Error('Use only one XML source: --xml or --xml-file.'); + } + + const hasCodeInline = typeof options.generatedCode === 'string'; + const hasCodeFile = typeof options.generatedCodeFile === 'string'; + if (hasCodeInline && hasCodeFile) { + throw new Error('Use only one generated code source: --generated-code or --generated-code-file.'); + } + + const projectData = await loadProject(projectFile); + + if (hasXmlInline || hasXmlFile) { + projectData.blocklyXml = hasXmlInline + ? String(options.xml) + : await fs.readFile(resolveWorkspacePath(String(options.xmlFile)), 'utf8'); + } + + if (hasCodeInline || hasCodeFile) { + projectData.blocklyGeneratedCode = hasCodeInline + ? String(options.generatedCode) + : await fs.readFile(resolveWorkspacePath(String(options.generatedCodeFile)), 'utf8'); + } + + if (options.useBlocklyCode !== undefined) { + projectData.useBlocklyCode = parseBooleanInput(options.useBlocklyCode, !!projectData.useBlocklyCode); + } + + const target = options.output || projectFile; + await saveProject(target, projectData); + printJson({ + ok: true, + action: 'project.set-blockly', + file: relToCwd(resolveWorkspacePath(target)), + blockly: { + useBlocklyCode: !!projectData.useBlocklyCode, + xmlLength: String(projectData.blocklyXml || '').length, + generatedCodeLength: String(projectData.blocklyGeneratedCode || '').length, + }, + }); + }); + + project + .command('block-summary ') + .description('Show block-coding metadata summary') + .action(async (projectFile: string) => { + const projectData = await loadProject(projectFile); + printJson({ + ok: true, + action: 'project.block-summary', + file: relToCwd(resolveWorkspacePath(projectFile)), + blockly: { + useBlocklyCode: !!projectData.useBlocklyCode, + hasXml: !!String(projectData.blocklyXml || '').trim(), + hasGeneratedCode: !!String(projectData.blocklyGeneratedCode || '').trim(), + xmlLength: String(projectData.blocklyXml || '').length, + generatedCodeLength: String(projectData.blocklyGeneratedCode || '').length, + }, + }); + }); + project .command('component-types') .description('List known component manifest types from emulator package') diff --git a/openhw-studio-cli-danish/src/types.ts b/openhw-studio-cli-danish/src/types.ts index 3f0262d..83297f3 100644 --- a/openhw-studio-cli-danish/src/types.ts +++ b/openhw-studio-cli-danish/src/types.ts @@ -40,6 +40,9 @@ export interface OpenHwProject { components: ComponentEntry[]; connections: WireEntry[]; code: string; + blocklyXml?: string; + blocklyGeneratedCode?: string; + useBlocklyCode?: boolean; projectFiles: ProjectFileEntry[]; openCodeTabs: string[]; activeCodeFileId: string; diff --git a/openhw-studio-cli-danish/src/utils/project.ts b/openhw-studio-cli-danish/src/utils/project.ts index ab266c9..b5d3c11 100644 --- a/openhw-studio-cli-danish/src/utils/project.ts +++ b/openhw-studio-cli-danish/src/utils/project.ts @@ -217,6 +217,9 @@ export function normalizeProject(rawProject: any): OpenHwProject { components, connections, code: asString(rawProject?.code, asString(rawProject?.userCode, '')), + blocklyXml: asString(rawProject?.blocklyXml, ''), + blocklyGeneratedCode: asString(rawProject?.blocklyGeneratedCode, ''), + useBlocklyCode: !!rawProject?.useBlocklyCode, projectFiles, openCodeTabs: Array.isArray(rawProject?.openCodeTabs) ? rawProject.openCodeTabs.map((id: any) => String(id)).filter(Boolean) @@ -280,6 +283,9 @@ export function createProject(name: string, board: string): OpenHwProject { ], connections: [], code: defaultMainCode(boardKind === 'unknown' ? 'arduino_uno' : boardKind, boardId), + blocklyXml: '', + blocklyGeneratedCode: '', + useBlocklyCode: false, projectFiles: [], openCodeTabs: [], activeCodeFileId: '', @@ -504,6 +510,8 @@ export function summarizeProject(project: OpenHwProject): Record ({ id: b.id, type: b.type })), files: project.projectFiles.length, + useBlocklyCode: !!project.useBlocklyCode, + hasBlocklyXml: !!String(project.blocklyXml || '').trim(), activeCodeFileId: project.activeCodeFileId, exportedAt: project.exportedAt, }; From 356a15e2bdbcc63824d3ecbddc2de899fe78f490 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:54:53 +0000 Subject: [PATCH 10/15] Add agent observability helpers and new CLI/MCP simulation workflows Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/acbe4ebc-9458-4884-b271-b9dd5b868ab4 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- openhw-studio-cli-danish/README.md | 17 +- openhw-studio-cli-danish/package.json | 2 + .../scenarios/pico-display-observability.yaml | 34 ++ .../scenarios/pico-sensor-observability.yaml | 44 ++ .../scripts/test-mcp-contracts.mjs | 17 + openhw-studio-cli-danish/src/commands/sim.ts | 315 +++++++++++++- openhw-studio-cli-danish/src/mcp/server.ts | 387 +++++++++++++++++- .../src/sim/agent-observability.ts | 378 +++++++++++++++++ 8 files changed, 1175 insertions(+), 19 deletions(-) create mode 100644 openhw-studio-cli-danish/scenarios/pico-display-observability.yaml create mode 100644 openhw-studio-cli-danish/scenarios/pico-sensor-observability.yaml create mode 100644 openhw-studio-cli-danish/src/sim/agent-observability.ts diff --git a/openhw-studio-cli-danish/README.md b/openhw-studio-cli-danish/README.md index f004afa..2596f6d 100644 --- a/openhw-studio-cli-danish/README.md +++ b/openhw-studio-cli-danish/README.md @@ -101,6 +101,15 @@ npm run cli -- sim screenshot temp/project.json --duration-ms 1200 --output out/ # Inspect input/output component automation templates npm run cli -- sim capabilities temp/project.json --json +# Probe one component with diffed before/after state + pin/display behavior +npm run cli -- sim probe temp/project.json --component-id ldr1 --event SET_ATTR --key lux --value 700 --assertions-file temp/assertions.yaml + +# Display-focused normalized state capture for AI agents +npm run cli -- sim display temp/project.json --duration-ms 1500 --output out/display.json + +# Multi-step scenario with timed inputs and assertions +npm run cli -- sim scenario temp/project.json --scenario scenarios/sensor-check.yaml --output out/scenario-report.json + # Print simulation-focused summary and validation npm run cli -- sim summary temp/project.json ``` @@ -129,13 +138,15 @@ npm run cli -- mcp serve npm run cli -- mcp serve --auth-token local-dev-token ``` -MCP tools now include `sim_execute`, `sim_trace`, and `sim_inspect` with support for debug/GDB trace capture (`debug_mode`, `include_trace`) and simulation console capture (`include_console`) without polluting stdio transport. +MCP tools now include `sim_execute`, `sim_trace`, `sim_inspect`, `simulation_step`, and `simulation_assert` with support for debug/GDB trace capture (`debug_mode`, `include_trace`) and simulation console capture (`include_console`) without polluting stdio transport. Additional MCP lifecycle and diagnostics tools: - `project_open` (set active session from an existing project file) - `project_status` (active session state + summary) - `project_validate` (schema/reference validation for active session) - `component_catalog` (discover component capabilities/pins/onEvent/telemetry metadata) +- `simulation_capabilities` (project-scoped observability and interaction affordances for AI agents) +- `component_input_schema` (event templates + sensor profiles per component instance) - `wiring_validate` (dry-run endpoint/wire validation without project mutation) ### Library management @@ -160,6 +171,10 @@ npm run test:cli:block-coding # Deterministic scenario runner dry-run (YAML/JSON manifest parse + wiring diagnostics + report export) npm run test:mcp:scenario +# Dry-run fixture coverage for display-heavy and sensor-heavy projects +npm run test:mcp:scenario:display +npm run test:mcp:scenario:sensor + # Full runtime scenario execution (requires simulation dependencies/backend) npm run test:mcp:scenario:full diff --git a/openhw-studio-cli-danish/package.json b/openhw-studio-cli-danish/package.json index db05686..916e9c9 100644 --- a/openhw-studio-cli-danish/package.json +++ b/openhw-studio-cli-danish/package.json @@ -15,6 +15,8 @@ "test:mcp:contracts": "node scripts/test-mcp-contracts.mjs", "test:mcp:scenario": "node scripts/mcp-scenario-runner.mjs --scenario scenarios/pico-led-lifecycle.yaml --dry-run", "test:mcp:scenario:full": "node scripts/mcp-scenario-runner.mjs --scenario scenarios/pico-led-lifecycle.yaml", + "test:mcp:scenario:display": "node scripts/mcp-scenario-runner.mjs --scenario scenarios/pico-display-observability.yaml --dry-run", + "test:mcp:scenario:sensor": "node scripts/mcp-scenario-runner.mjs --scenario scenarios/pico-sensor-observability.yaml --dry-run", "test:cli:block-coding": "node scripts/test-cli-block-coding.mjs" }, "dependencies": { diff --git a/openhw-studio-cli-danish/scenarios/pico-display-observability.yaml b/openhw-studio-cli-danish/scenarios/pico-display-observability.yaml new file mode 100644 index 0000000..da0d317 --- /dev/null +++ b/openhw-studio-cli-danish/scenarios/pico-display-observability.yaml @@ -0,0 +1,34 @@ +name: pico-display-observability +board: wokwi-raspberry-pi-pico +durationMs: 1800 +envs: + - key: native + boardEnv: native + ms: 1800 +components: + - type: wokwi-ssd1306 + id: oled1 + x: 300 + y: 120 + - type: wokwi-led + id: led1 + x: 360 + y: 220 +wires: + - from: board1:GP0 + to: oled1:SDA + - from: board1:GP1 + to: oled1:SCL + - from: board1:3V3 + to: oled1:VCC + - from: board1:GND + to: oled1:GND + - from: board1:GP2 + to: led1:A + - from: board1:GND + to: led1:C +inspect: + - id: oled1 + includeTrace: true + includeConsole: false +traceEventTypes: [state, serial, fault, debug] diff --git a/openhw-studio-cli-danish/scenarios/pico-sensor-observability.yaml b/openhw-studio-cli-danish/scenarios/pico-sensor-observability.yaml new file mode 100644 index 0000000..6b534d6 --- /dev/null +++ b/openhw-studio-cli-danish/scenarios/pico-sensor-observability.yaml @@ -0,0 +1,44 @@ +name: pico-sensor-observability +board: wokwi-raspberry-pi-pico +durationMs: 2000 +envs: + - key: native + boardEnv: native + ms: 2000 +components: + - type: wokwi-photoresistor-sensor + id: ldr1 + x: 300 + y: 120 + - type: wokwi-slide-potentiometer + id: pot1 + x: 300 + y: 240 +wires: + - from: board1:GP26 + to: ldr1:AO + - from: board1:3V3 + to: ldr1:VCC + - from: board1:GND + to: ldr1:GND + - from: board1:GP27 + to: pot1:SIG + - from: board1:3V3 + to: pot1:VCC + - from: board1:GND + to: pot1:GND +inspect: + - id: ldr1 + includeTrace: true + event: + type: SET_ATTR + key: lux + value: 650 + atMs: 300 + - id: pot1 + includeTrace: true + event: + type: input + value: 75 + atMs: 700 +traceEventTypes: [state, serial, fault, debug] diff --git a/openhw-studio-cli-danish/scripts/test-mcp-contracts.mjs b/openhw-studio-cli-danish/scripts/test-mcp-contracts.mjs index 0cbb7be..4a7e64c 100644 --- a/openhw-studio-cli-danish/scripts/test-mcp-contracts.mjs +++ b/openhw-studio-cli-danish/scripts/test-mcp-contracts.mjs @@ -70,10 +70,14 @@ async function main() { 'project_status', 'project_validate', 'component_catalog', + 'simulation_capabilities', + 'component_input_schema', 'wiring_validate', 'sim_execute', 'sim_trace', 'sim_inspect', + 'simulation_step', + 'simulation_assert', ]) { assert(toolNames.has(requiredTool), `Missing MCP tool ${requiredTool}`); } @@ -104,6 +108,19 @@ async function main() { assert(!catalog.isError, 'component_catalog returned error.'); assert(Array.isArray(catalog.parsed?.components), 'component_catalog components missing.'); + const capabilities = await callTool(client, 'simulation_capabilities', { + ...(token ? { token } : {}), + }); + assert(!capabilities.isError, 'simulation_capabilities returned error.'); + assert(capabilities.parsed?.ok === true, 'simulation_capabilities should return ok=true.'); + assert(Array.isArray(capabilities.parsed?.project?.interactiveComponents), 'simulation_capabilities interactiveComponents missing.'); + + const inputSchema = await callTool(client, 'component_input_schema', { + ...(token ? { token } : {}), + }); + assert(!inputSchema.isError, 'component_input_schema returned error.'); + assert(Array.isArray(inputSchema.parsed?.components), 'component_input_schema components missing.'); + const invalidWire = await callTool(client, 'wiring_validate', { from: 'board1:NOT_A_PIN', to: 'board1:GND', diff --git a/openhw-studio-cli-danish/src/commands/sim.ts b/openhw-studio-cli-danish/src/commands/sim.ts index a4189a6..56fb6d5 100644 --- a/openhw-studio-cli-danish/src/commands/sim.ts +++ b/openhw-studio-cli-danish/src/commands/sim.ts @@ -2,6 +2,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { Command } from 'commander'; +import YAML from 'yaml'; import type { OpenHwProject, SimulationRunOptions, @@ -18,6 +19,15 @@ import { loadProject, summarizeProject, validateProject } from '../utils/project import { BOARD_DEFAULT_BAUD, normalizeBoardKind } from '../utils/boards.js'; import { getManifestInfo } from '../utils/manifests.js'; import { FRONTEND_ROOT, relToCwd, resolveWorkspacePath } from '../utils/paths.js'; +import { + buildProfileEvents, + componentInputSchemaForProject, + diffBoardPins, + diffComponentStates, + evaluateAssertions, + extractDisplayStates, + normalizeBoardPinStates, +} from '../sim/agent-observability.js'; function printJson(data: unknown): void { process.stdout.write(`${JSON.stringify(data, null, 2)}\n`); @@ -477,6 +487,13 @@ async function writeOutputFile(targetPath: string, content: string): Promise { + const absolute = resolveWorkspacePath(inputPath); + const raw = await fs.readFile(absolute, 'utf8'); + const ext = path.extname(absolute).toLowerCase(); + return ext === '.yaml' || ext === '.yml' ? YAML.parse(raw) : JSON.parse(raw); +} + async function runForDuration( project: Awaited>, runOptions: SimulationRunOptions @@ -1172,6 +1189,280 @@ export function registerSimCommands(program: Command, getBackendUrl: () => strin } }); + sim + .command('probe ') + .description('Inject one input/profile and return before/after component+pin+display behavior diff') + .requiredOption('--component-id ', 'Target component id') + .option('--board-id ', 'Board component id to run') + .option('--all-boards', 'Run all boards in project') + .option('--duration-ms ', 'Total probe runtime', '1800') + .option('--at-ms ', 'When to inject event', '250') + .option('--event ', 'Event name (e.g. input, SET_ATTR, press)') + .option('--value ', 'Event value (number/string/json)') + .option('--key ', 'Event key for SET_ATTR payloads') + .option('--event-json ', 'Full event JSON payload') + .option('--event-file ', 'Path to JSON event payload file') + .option('--profile ', 'Sensor profile name from sim capabilities') + .option('--assertions-file ', 'JSON/YAML assertions file') + .option('--output ', 'Write full probe payload JSON') + .action(async (projectFile: string, options: any) => { + const project = await loadProject(projectFile); + const boardSummary = summarizeProject(project).boards as Array<{ id: string; type: string }>; + const selectedBoard = options.boardId + ? boardSummary.find((b) => b.id === options.boardId) + : boardSummary.length === 1 + ? boardSummary[0] + : undefined; + + const componentId = String(options.componentId || '').trim(); + const targetComponent = project.components.find((entry) => entry.id === componentId); + if (!targetComponent) { + throw new Error(`Component not found: ${componentId}`); + } + + const durationMs = Math.max(1, parsePositiveInt(options.durationMs, 1800)); + const eventAtMs = Math.min(durationMs, parseNonNegative(options.atMs, 250)); + + const runOptions: SimulationRunOptions = { + backendUrl: getBackendUrl(), + boardId: options.boardId, + allBoards: !!options.allBoards, + durationMs, + debugMode: 'off', + telemetryMode: 'off', + baudRate: resolveDefaultBaud(selectedBoard?.type || project.board, options.baud), + }; + + const controller = await startSimulation(project, runOptions, { suppressConsoleOutput: true }); + if (eventAtMs > 0) { + await sleep(eventAtMs); + } + + const beforeSnapshot = controller.getSnapshot(); + const beforeTelemetry = controller.getTelemetryReport(); + + const eventsToInject: Array<{ atMs: number; event: unknown }> = []; + if (options.profile) { + eventsToInject.push(...buildProfileEvents(String(options.profile), targetComponent.type, durationMs)); + } else { + eventsToInject.push({ + atMs: eventAtMs, + event: await resolveEventInput({ + eventJson: options.eventJson, + eventFile: options.eventFile, + event: options.event, + value: options.value, + key: options.key, + }), + }); + } + + let delivered = true; + let cursorMs = eventAtMs; + for (const entry of eventsToInject.sort((a, b) => a.atMs - b.atMs)) { + const waitMs = Math.max(0, entry.atMs - cursorMs); + if (waitMs > 0) await sleep(waitMs); + delivered = controller.sendComponentEvent(componentId, entry.event) && delivered; + cursorMs = entry.atMs; + } + + const remainingMs = Math.max(0, durationMs - cursorMs); + if (remainingMs > 0) await sleep(remainingMs); + controller.stop(); + + const afterSnapshot = controller.getSnapshot(); + const telemetry = controller.getTelemetryReport(); + const displays = extractDisplayStates(afterSnapshot, telemetry); + const payload: Record = { + ok: delivered, + action: 'sim.probe', + file: relToCwd(resolveWorkspacePath(projectFile)), + componentId, + delivered, + injected: eventsToInject, + pinState: normalizeBoardPinStates(afterSnapshot), + displays, + diff: { + boardPins: diffBoardPins(beforeSnapshot, afterSnapshot), + components: diffComponentStates(beforeSnapshot, afterSnapshot, componentId), + displays: { + before: extractDisplayStates(beforeSnapshot, beforeTelemetry), + after: displays, + }, + }, + targetTelemetry: telemetry.components.find((entry) => entry.id === componentId) || null, + }; + + if (options.assertionsFile) { + const checksRaw = await parseScenarioFile(String(options.assertionsFile)); + const checks = Array.isArray(checksRaw?.assertions) ? checksRaw.assertions : checksRaw; + payload.assertions = evaluateAssertions({ + checks: Array.isArray(checks) ? checks : [], + displays, + telemetry, + snapshot: afterSnapshot, + }); + if ((payload.assertions as { ok?: boolean }).ok === false) { + process.exitCode = 1; + } + } + + if (options.output) { + await writeOutputFile(options.output, `${JSON.stringify(payload, null, 2)}\n`); + } + + printJson({ + ...payload, + output: options.output ? relToCwd(resolveWorkspacePath(options.output)) : null, + }); + + if (!delivered) { + process.exitCode = 1; + } + }); + + sim + .command('display ') + .description('Run simulation and return normalized display-focused state snapshots') + .option('--board-id ', 'Board component id to run') + .option('--all-boards', 'Run all boards in project') + .option('--duration-ms ', 'Run duration before capture', '1400') + .option('--output ', 'Write display payload JSON') + .action(async (projectFile: string, options: any) => { + const project = await loadProject(projectFile); + const runOptions: SimulationRunOptions = { + backendUrl: getBackendUrl(), + boardId: options.boardId, + allBoards: !!options.allBoards, + durationMs: Math.max(1, parsePositiveInt(options.durationMs, 1400)), + debugMode: 'off', + telemetryMode: 'off', + }; + + const { telemetry, snapshot } = await runForDuration(project, runOptions); + const displays = extractDisplayStates(snapshot, telemetry); + const payload = { + ok: true, + action: 'sim.display', + file: relToCwd(resolveWorkspacePath(projectFile)), + count: displays.length, + displays, + }; + + if (options.output) { + await writeOutputFile(options.output, `${JSON.stringify(payload, null, 2)}\n`); + } + + printJson({ + ...payload, + output: options.output ? relToCwd(resolveWorkspacePath(options.output)) : null, + }); + }); + + sim + .command('scenario ') + .description('Run repeatable simulation scenario with timed inputs and assertions from JSON/YAML') + .requiredOption('--scenario ', 'Scenario JSON/YAML file path') + .option('--output ', 'Write scenario report JSON') + .action(async (projectFile: string, options: any) => { + const project = await loadProject(projectFile); + const scenario = await parseScenarioFile(String(options.scenario)); + const durationMs = Math.max(1, parsePositiveInt(String(scenario?.durationMs || '1800'), 1800)); + const runOptions: SimulationRunOptions = { + backendUrl: getBackendUrl(), + boardId: typeof scenario?.boardId === 'string' ? scenario.boardId : undefined, + allBoards: !!scenario?.allBoards, + durationMs, + debugMode: 'off', + telemetryMode: 'off', + }; + + const controller = await startSimulation(project, runOptions, { suppressConsoleOutput: true }); + const inputs = Array.isArray(scenario?.inputs) ? scenario.inputs : []; + let cursorMs = 0; + let deliveredAll = true; + + for (const input of inputs.sort((a: any, b: any) => Number(a?.atMs || 0) - Number(b?.atMs || 0))) { + const atMs = Math.max(0, Math.min(durationMs, Number(input?.atMs || 0))); + const waitMs = Math.max(0, atMs - cursorMs); + if (waitMs > 0) await sleep(waitMs); + cursorMs = atMs; + + const targetId = String(input?.componentId || '').trim(); + if (!targetId) continue; + let eventPayload: unknown = input?.event; + + if (String(input?.profile || '').trim()) { + const component = project.components.find((entry) => entry.id === targetId); + if (component) { + const profileEvents = buildProfileEvents(String(input.profile), component.type, durationMs); + for (const profileEvent of profileEvents) { + const profileWaitMs = Math.max(0, profileEvent.atMs - cursorMs); + if (profileWaitMs > 0) await sleep(profileWaitMs); + cursorMs = profileEvent.atMs; + deliveredAll = controller.sendComponentEvent(targetId, profileEvent.event) && deliveredAll; + } + continue; + } + } + + if (input && typeof input === 'object' && !eventPayload) { + eventPayload = { + type: input.eventName || 'input', + value: input.value, + ...(input.key ? { key: input.key } : {}), + }; + } + + deliveredAll = controller.sendComponentEvent(targetId, eventPayload) && deliveredAll; + } + + const remainingMs = Math.max(0, durationMs - cursorMs); + if (remainingMs > 0) await sleep(remainingMs); + controller.stop(); + + const snapshot = controller.getSnapshot(); + const telemetry = controller.getTelemetryReport(); + const displays = extractDisplayStates(snapshot, telemetry); + const checks = Array.isArray(scenario?.assertions) ? scenario.assertions : []; + const assertions = evaluateAssertions({ + checks, + displays, + telemetry, + snapshot, + }); + + const payload = { + ok: deliveredAll && assertions.ok, + action: 'sim.scenario', + file: relToCwd(resolveWorkspacePath(projectFile)), + scenario: relToCwd(resolveWorkspacePath(options.scenario)), + durationMs, + deliveredAll, + assertions, + pinState: normalizeBoardPinStates(snapshot), + displays, + telemetrySummary: { + faults: telemetry.faults, + serialChars: telemetry.serialChars, + nonOk: telemetry.components.filter((entry) => entry.status !== 'ok').map((entry) => entry.id), + }, + }; + + if (options.output) { + await writeOutputFile(options.output, `${JSON.stringify(payload, null, 2)}\n`); + } + + printJson({ + ...payload, + output: options.output ? relToCwd(resolveWorkspacePath(options.output)) : null, + }); + + if (!payload.ok) { + process.exitCode = 1; + } + }); + sim .command('screenshot ') .description('Export simulation screenshot SVG for running or non-running project state') @@ -1248,23 +1539,13 @@ export function registerSimCommands(program: Command, getBackendUrl: () => strin .option('--json', 'Print JSON output') .action(async (projectFile: string, options: { json?: boolean }) => { const project = await loadProject(projectFile); - const components = await Promise.all( - project.components.map(async (c) => { - const info = await getManifestInfo(c.type); - const group = info?.group || 'Other'; - const role = classifyRole(c.type, group); - const templates = interactionTemplatesForType(c.type); - return { - id: c.id, - type: c.type, - label: c.label || c.id, - group, - role, - interactive: !!info?.hasOnEvent || templates.length > 0, - templates, - }; - }) - ); + const manifestByType = new Map>>(); + for (const component of project.components) { + if (!manifestByType.has(component.type)) { + manifestByType.set(component.type, await getManifestInfo(component.type)); + } + } + const components = componentInputSchemaForProject(project, manifestByType); if (options.json) { printJson({ diff --git a/openhw-studio-cli-danish/src/mcp/server.ts b/openhw-studio-cli-danish/src/mcp/server.ts index a23d33a..c8d6580 100644 --- a/openhw-studio-cli-danish/src/mcp/server.ts +++ b/openhw-studio-cli-danish/src/mcp/server.ts @@ -14,7 +14,16 @@ import { } from '../utils/project.js'; import { relToCwd, resolveWorkspacePath } from '../utils/paths.js'; import { startSimulation } from '../sim/session.js'; -import { getPinsForType, listManifestInfos } from '../utils/manifests.js'; +import { getManifestInfo, getPinsForType, listManifestInfos } from '../utils/manifests.js'; +import { + buildProfileEvents, + componentInputSchemaForProject, + diffBoardPins, + diffComponentStates, + evaluateAssertions, + extractDisplayStates, + normalizeBoardPinStates, +} from '../sim/agent-observability.js'; export interface McpServerConfig { backendUrl: string; @@ -455,6 +464,55 @@ function buildInteractionEvent(event: unknown, value: unknown): unknown { }; } +function buildCorrelationId(prefix: string): string { + const stamp = Date.now().toString(36); + const rand = Math.random().toString(36).slice(2, 8); + return `${prefix}-${stamp}-${rand}`; +} + +function normalizeInputEventsForStep(options: { + project: OpenHwProject; + durationMs: number; + inputs: Array<{ + id?: string; + event?: unknown; + value?: unknown; + at_ms?: number; + profile?: string; + }>; +}): Array<{ atMs: number; id: string; event: unknown; profile?: string }> { + const events: Array<{ atMs: number; id: string; event: unknown; profile?: string }> = []; + + for (const input of options.inputs) { + const id = String(input?.id || '').trim(); + if (!id) continue; + const component = options.project.components.find((entry) => entry.id === id) || null; + if (!component) continue; + + const atMs = Math.max(0, Math.min(options.durationMs, Math.floor(Number(input?.at_ms || 0)))); + const profile = String(input?.profile || '').trim(); + if (profile) { + for (const profileEvent of buildProfileEvents(profile, component.type, options.durationMs)) { + events.push({ + atMs: profileEvent.atMs, + id, + event: profileEvent.event, + profile, + }); + } + continue; + } + + events.push({ + atMs, + id, + event: buildInteractionEvent(input?.event, input?.value), + }); + } + + return events.sort((a, b) => a.atMs - b.atMs); +} + async function runSimulationForDuration( project: OpenHwProject, options: SimulationRunOptions, @@ -653,6 +711,83 @@ export async function runMcpServer(config: McpServerConfig): Promise { } ); + server.tool( + 'simulation_capabilities', + 'Describe simulation observability/input/assertion capabilities for the active project.', + { + token: z.string().optional(), + }, + async ({ token }) => { + assertToken(config, token); + const { project, projectFile } = await requireActiveProject(session); + + const manifestByType = new Map>>(); + for (const component of project.components) { + if (!manifestByType.has(component.type)) { + manifestByType.set(component.type, await getManifestInfo(component.type)); + } + } + + const componentSchemas = componentInputSchemaForProject(project, manifestByType); + return makeToolResult({ + ok: true, + action: 'simulation_capabilities', + file: relToCwd(projectFile), + tools: { + observability: ['sim_execute', 'sim_trace', 'sim_inspect'], + interaction: ['component_interact', 'simulation_step'], + assertions: ['simulation_assert'], + metadata: ['component_catalog', 'component_input_schema'], + }, + project: { + boards: project.components.filter((entry) => /(arduino|esp32|stm32|rp2040|pico)/i.test(entry.type)).map((entry) => ({ + id: entry.id, + type: entry.type, + label: entry.label || entry.id, + })), + interactiveComponents: componentSchemas.filter((entry) => entry.interactive).map((entry) => ({ + id: entry.id, + type: entry.type, + role: entry.role, + profiles: entry.profiles.map((profile) => profile.name), + })), + }, + }); + } + ); + + server.tool( + 'component_input_schema', + 'Return supported input payload templates, profiles, and event hints per project component.', + { + id: z.string().optional(), + token: z.string().optional(), + }, + async ({ id, token }) => { + assertToken(config, token); + const { project, projectFile } = await requireActiveProject(session); + + const manifestByType = new Map>>(); + for (const component of project.components) { + if (!manifestByType.has(component.type)) { + manifestByType.set(component.type, await getManifestInfo(component.type)); + } + } + + const allSchemas = componentInputSchemaForProject(project, manifestByType); + const componentId = String(id || '').trim(); + const filtered = componentId ? allSchemas.filter((entry) => entry.id === componentId) : allSchemas; + + return makeToolResult({ + ok: filtered.length > 0 || !componentId, + action: 'component_input_schema', + file: relToCwd(projectFile), + count: filtered.length, + components: filtered, + }); + } + ); + server.tool( 'project_init', 'Create a new OpenHW project JSON and set it as active session project.', @@ -951,6 +1086,218 @@ export async function runMcpServer(config: McpServerConfig): Promise { } ); + server.tool( + 'simulation_step', + 'Run deterministic step-based simulation control with optional timed inputs and bounded trace capture.', + { + steps: z.number().int().positive().max(500).optional(), + step_ms: z.number().int().positive().max(60000).optional(), + board_id: z.string().optional(), + all_boards: z.boolean().optional(), + include_trace: z.boolean().optional(), + include_console: z.boolean().optional(), + include_state: z.boolean().optional(), + include_serial_text: z.boolean().optional(), + include_diff: z.boolean().optional(), + include_display: z.boolean().optional(), + include_pin_state: z.boolean().optional(), + max_events: z.number().int().positive().max(5000).optional(), + max_console_chars: z.number().int().positive().max(250000).optional(), + max_serial_chars: z.number().int().positive().max(10000).optional(), + inputs: z.array(z.object({ + id: z.string().optional(), + event: z.any().optional(), + value: z.any().optional(), + at_ms: z.number().int().nonnegative().optional(), + profile: z.string().optional(), + })).optional(), + token: z.string().optional(), + }, + async ({ + steps, + step_ms, + board_id, + all_boards, + include_trace, + include_console, + include_state, + include_serial_text, + include_diff, + include_display, + include_pin_state, + max_events, + max_console_chars, + max_serial_chars, + inputs, + token, + }) => { + assertToken(config, token); + const { project, projectFile } = await requireActiveProject(session); + const totalSteps = clampPositiveInt(steps, 8, 1, 500); + const stepMs = clampPositiveInt(step_ms, 120, 1, 60000); + const durationMs = totalSteps * stepMs; + const includeTrace = !!include_trace; + const includeConsole = !!include_console; + + const runOptions: SimulationRunOptions = { + backendUrl: config.backendUrl, + boardId: board_id, + allBoards: !!all_boards, + durationMs: 0, + debugMode: includeTrace ? 'json' : 'off', + telemetryMode: 'off', + }; + + const runtimeCapture = createRuntimeCapture({ + includeTrace, + includeConsole, + maxEvents: clampPositiveInt(max_events, 400, 1, 5000), + maxConsoleChars: clampPositiveInt(max_console_chars, 10000, 256, 250000), + traceEventTypes: [], + traceBuild: { + includeState: !!include_state, + includeSerialText: !!include_serial_text, + maxSerialChars: clampPositiveInt(max_serial_chars, 120, 16, 10000), + componentId: null, + }, + }); + + const scheduledInputs = normalizeInputEventsForStep({ + project, + durationMs, + inputs: Array.isArray(inputs) ? inputs : [], + }); + + const controller = await startSimulation(project, runOptions, { + suppressConsoleOutput: true, + onEvent: runtimeCapture.onEvent, + }); + + const snapshots: Array<{ step: number; tMs: number; boards: number; components: number }> = []; + let elapsedMs = 0; + let inputCursor = 0; + for (let step = 1; step <= totalSteps; step += 1) { + const nextElapsed = step * stepMs; + while (inputCursor < scheduledInputs.length && scheduledInputs[inputCursor].atMs <= nextElapsed) { + const inputEvent = scheduledInputs[inputCursor]; + controller.sendComponentEvent(inputEvent.id, inputEvent.event); + inputCursor += 1; + } + await sleep(stepMs); + elapsedMs = nextElapsed; + const snapshot = controller.getSnapshot(); + snapshots.push({ + step, + tMs: elapsedMs, + boards: snapshot.boards.length, + components: snapshot.components.length, + }); + } + + controller.stop(); + + const result = controller.getResult(); + const telemetry = controller.getTelemetryReport(); + const snapshot = controller.getSnapshot(); + const captured = runtimeCapture.flush(); + const displays = extractDisplayStates(snapshot, telemetry); + + return makeToolResult({ + ok: true, + action: 'simulation_step', + file: relToCwd(projectFile), + run: { + steps: totalSteps, + stepMs, + elapsedMs, + boardId: board_id || null, + allBoards: !!all_boards, + }, + inputs: scheduledInputs, + snapshots, + pinState: normalizeBoardPinStates(snapshot), + displays, + result, + telemetry, + trace: includeTrace ? captured.trace : undefined, + traceSummary: includeTrace + ? { + capturedEvents: captured.trace.length, + droppedEvents: captured.droppedTraceEvents, + } + : undefined, + console: includeConsole + ? { + text: captured.consoleText, + length: captured.consoleText.length, + } + : undefined, + }); + } + ); + + server.tool( + 'simulation_assert', + 'Run a short simulation and evaluate assertion checks against telemetry, display state, and pin values.', + { + ms: z.number().int().positive().optional(), + board_id: z.string().optional(), + all_boards: z.boolean().optional(), + assertions: z.array(z.object({ + type: z.string(), + component_id: z.string().optional(), + board_id: z.string().optional(), + pin: z.string().optional(), + text: z.string().optional(), + status: z.enum(['ok', 'warn', 'error']).optional(), + high: z.boolean().optional(), + })), + token: z.string().optional(), + }, + async ({ ms, board_id, all_boards, assertions, token }) => { + assertToken(config, token); + const { project, projectFile } = await requireActiveProject(session); + const durationMs = clampPositiveInt(ms, 1200, 1, 1200000); + + const runOptions: SimulationRunOptions = { + backendUrl: config.backendUrl, + boardId: board_id, + allBoards: !!all_boards, + durationMs, + debugMode: 'off', + telemetryMode: 'off', + }; + + const { result, telemetry } = await runSimulationForDuration(project, runOptions, true); + const controller = await startSimulation(project, { + ...runOptions, + durationMs: 1, + }, { + suppressConsoleOutput: true, + }); + await sleep(1); + controller.stop(); + const snapshot = controller.getSnapshot(); + + const displays = extractDisplayStates(snapshot, telemetry); + const assertionResult = evaluateAssertions({ + checks: assertions as any, + displays, + telemetry, + snapshot, + }); + + return makeToolResult({ + ok: assertionResult.ok, + action: 'simulation_assert', + file: relToCwd(projectFile), + durationMs, + result, + assertions: assertionResult, + }); + } + ); + server.tool( 'sim_inspect', 'Inspect runtime board/component telemetry for the active project with optional event injection.', @@ -966,6 +1313,9 @@ export async function runMcpServer(config: McpServerConfig): Promise { include_console: z.boolean().optional(), include_state: z.boolean().optional(), include_serial_text: z.boolean().optional(), + include_diff: z.boolean().optional(), + include_display: z.boolean().optional(), + include_pin_state: z.boolean().optional(), max_events: z.number().int().positive().max(5000).optional(), max_console_chars: z.number().int().positive().max(250000).optional(), max_serial_chars: z.number().int().positive().max(10000).optional(), @@ -984,6 +1334,9 @@ export async function runMcpServer(config: McpServerConfig): Promise { include_console, include_state, include_serial_text, + include_diff, + include_display, + include_pin_state, max_events, max_console_chars, max_serial_chars, @@ -1028,6 +1381,9 @@ export async function runMcpServer(config: McpServerConfig): Promise { let delivered = false; let eventPayload: unknown = null; let eventAtMs = 0; + let beforeSnapshot = controller.getSnapshot(); + let beforeTelemetry = controller.getTelemetryReport(); + let correlationId = ''; if (injectEvent) { const targetComponentId = componentId; @@ -1042,6 +1398,9 @@ export async function runMcpServer(config: McpServerConfig): Promise { await sleep(eventAtMs); } + beforeSnapshot = controller.getSnapshot(); + beforeTelemetry = controller.getTelemetryReport(); + correlationId = buildCorrelationId('inspect'); delivered = controller.sendComponentEvent(targetComponentId, eventPayload); } @@ -1056,6 +1415,7 @@ export async function runMcpServer(config: McpServerConfig): Promise { const telemetry = controller.getTelemetryReport(); const snapshot = controller.getSnapshot(); const captured = runtimeCapture.flush(); + const displays = include_display === false ? [] : extractDisplayStates(snapshot, telemetry); const componentTelemetry = componentId ? telemetry.components.find((component) => component.id === componentId) || null @@ -1093,11 +1453,36 @@ export async function runMcpServer(config: McpServerConfig): Promise { event: eventPayload, atMs: eventAtMs, delivered, + correlationId: correlationId || null, + }; + } + + if (include_pin_state !== false) { + payload.pinState = normalizeBoardPinStates(snapshot); + } + + if (include_display !== false) { + payload.displays = displays; + } + + if (injectEvent && include_diff !== false) { + payload.diff = { + boardPins: diffBoardPins(beforeSnapshot, snapshot), + components: diffComponentStates(beforeSnapshot, snapshot, componentId || null), + displays: { + before: extractDisplayStates(beforeSnapshot, beforeTelemetry), + after: displays, + }, }; } if (include_trace) { payload.trace = captured.trace; + if (correlationId) { + payload.correlatedTrace = captured.trace + .filter((entry) => entry.tMs >= eventAtMs) + .map((entry) => ({ ...entry, correlationId })); + } payload.traceSummary = { capturedEvents: captured.trace.length, droppedEvents: captured.droppedTraceEvents, diff --git a/openhw-studio-cli-danish/src/sim/agent-observability.ts b/openhw-studio-cli-danish/src/sim/agent-observability.ts new file mode 100644 index 0000000..f04818a --- /dev/null +++ b/openhw-studio-cli-danish/src/sim/agent-observability.ts @@ -0,0 +1,378 @@ +import type { OpenHwProject, SimulationSnapshot, SimulationTelemetryReport } from '../types.js'; +import type { ManifestInfo } from '../utils/manifests.js'; + +export type ComponentInputTemplate = string | Record; + +export type ComponentInputSchema = { + id: string; + type: string; + label: string; + group: string; + role: 'board' | 'input' | 'output' | 'other'; + interactive: boolean; + hasOnEvent: boolean; + templates: ComponentInputTemplate[]; + profiles: Array<{ + name: string; + description: string; + defaultDurationMs: number; + example: Array<{ atMs: number; event: unknown }>; + }>; +}; + +export type DisplayStateView = { + id: string; + type: string; + label: string; + text: string | null; + value: number | string | null; + segments: unknown; + pixels: unknown; + rawState: Record; +}; + +export type BoardPinState = { + boardId: string; + type: string; + pins: Array<{ pin: string; high: boolean }>; +}; + +function asRecord(value: unknown): Record | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + return value as Record; +} + +export function classifyRole(type: string, group: string): 'board' | 'input' | 'output' | 'other' { + if (/(arduino|esp32|stm32|rp2040|pico)/i.test(type)) return 'board'; + const g = String(group || '').toLowerCase(); + if (/(output|display|actuator|memory)/.test(g)) return 'output'; + if (/(sensor|input|basic|communication|logic)/.test(g)) return 'input'; + return 'other'; +} + +export function interactionTemplatesForType(type: string): ComponentInputTemplate[] { + const t = String(type || '').toLowerCase(); + + if (t.includes('pushbutton')) return ['press', 'release']; + if (t.includes('potentiometer') || t.includes('slide-potentiometer')) { + return [{ type: 'input', value: 0 }, { type: 'input', value: 50 }, { type: 'input', value: 100 }]; + } + if (t.includes('ldr')) { + return [ + { type: 'SET_ATTR', key: 'lux', value: 100 }, + { type: 'SET_ATTR', key: 'lux', value: 800 }, + { type: 'SET_ATTR', key: 'threshold', value: 500 }, + ]; + } + if (t.includes('max30102')) { + return [{ type: 'SET_ATTR', key: 'heartRate', value: 75 }, { type: 'SET_ATTR', key: 'spo2', value: 98 }]; + } + if (t.includes('dht')) { + return [ + { type: 'SET_ATTR', key: 'temperature', value: 24 }, + { type: 'SET_ATTR', key: 'humidity', value: 60 }, + ]; + } + return [{ type: 'input', value: 50 }, { type: 'SET_ATTR', key: 'value', value: 50 }]; +} + +export function sensorProfilesForType(type: string): ComponentInputSchema['profiles'] { + const t = String(type || '').toLowerCase(); + + if (t.includes('pushbutton')) { + return [ + { + name: 'button_bounce', + description: 'Press/release burst that simulates switch bounce.', + defaultDurationMs: 250, + example: [ + { atMs: 10, event: 'press' }, + { atMs: 40, event: 'release' }, + { atMs: 65, event: 'press' }, + { atMs: 95, event: 'release' }, + { atMs: 130, event: 'press' }, + ], + }, + ]; + } + + if (t.includes('potentiometer')) { + return [ + { + name: 'pot_ramp', + description: 'Sweep potentiometer input from low to high.', + defaultDurationMs: 900, + example: [ + { atMs: 50, event: { type: 'input', value: 0 } }, + { atMs: 350, event: { type: 'input', value: 50 } }, + { atMs: 700, event: { type: 'input', value: 100 } }, + ], + }, + ]; + } + + if (t.includes('ldr') || t.includes('light')) { + return [ + { + name: 'light_sweep', + description: 'Change light intensity from dark to bright.', + defaultDurationMs: 1000, + example: [ + { atMs: 50, event: { type: 'SET_ATTR', key: 'lux', value: 80 } }, + { atMs: 450, event: { type: 'SET_ATTR', key: 'lux', value: 450 } }, + { atMs: 850, event: { type: 'SET_ATTR', key: 'lux', value: 900 } }, + ], + }, + { + name: 'light_noise', + description: 'Inject noisy light readings around a threshold.', + defaultDurationMs: 1000, + example: [ + { atMs: 100, event: { type: 'SET_ATTR', key: 'lux', value: 480 } }, + { atMs: 240, event: { type: 'SET_ATTR', key: 'lux', value: 510 } }, + { atMs: 420, event: { type: 'SET_ATTR', key: 'lux', value: 495 } }, + { atMs: 610, event: { type: 'SET_ATTR', key: 'lux', value: 520 } }, + ], + }, + ]; + } + + if (t.includes('max30102')) { + return [ + { + name: 'heart_rate_ramp', + description: 'Ramp heart rate and SpO2 readings.', + defaultDurationMs: 1200, + example: [ + { atMs: 100, event: { type: 'SET_ATTR', key: 'heartRate', value: 68 } }, + { atMs: 500, event: { type: 'SET_ATTR', key: 'heartRate', value: 88 } }, + { atMs: 900, event: { type: 'SET_ATTR', key: 'spo2', value: 97 } }, + ], + }, + ]; + } + + return []; +} + +export function componentInputSchemaForProject( + project: OpenHwProject, + manifestByType: Map +): ComponentInputSchema[] { + return project.components.map((component) => { + const manifest = manifestByType.get(component.type) || null; + const group = manifest?.group || 'Other'; + const role = classifyRole(component.type, group); + const templates = interactionTemplatesForType(component.type); + const profiles = sensorProfilesForType(component.type); + + return { + id: component.id, + type: component.type, + label: String(component.label || component.id), + group, + role, + interactive: !!manifest?.hasOnEvent || templates.length > 0, + hasOnEvent: !!manifest?.hasOnEvent, + templates, + profiles, + }; + }); +} + +function looksLikeDisplay(type: string, state: Record): boolean { + const t = String(type || '').toLowerCase(); + if (/(display|oled|lcd|ssd1306|ili9341|max7219|7segment|segment|matrix|tm1637)/.test(t)) return true; + return ['text', 'segments', 'pixels', 'chars', 'buffer'].some((key) => Object.prototype.hasOwnProperty.call(state, key)); +} + +export function extractDisplayStates( + snapshot: SimulationSnapshot, + telemetry?: SimulationTelemetryReport | null +): DisplayStateView[] { + const telemetryById = new Map( + Array.isArray(telemetry?.components) + ? telemetry!.components.map((entry) => [String(entry.id), entry]) + : [] + ); + + return snapshot.components + .filter((component) => looksLikeDisplay(component.type, component.state || {})) + .map((component) => { + const state = asRecord(component.state) || {}; + const telemetryEntry = telemetryById.get(component.id); + const telemetryData = asRecord(telemetryEntry?.telemetryData); + const text = typeof state.text === 'string' + ? state.text + : typeof state.value === 'string' + ? state.value + : (typeof telemetryEntry?.outputSummary === 'string' ? telemetryEntry.outputSummary : null); + + const numeric = typeof state.value === 'number' + ? state.value + : (typeof state.number === 'number' ? state.number : null); + + return { + id: component.id, + type: component.type, + label: component.label, + text, + value: numeric ?? (typeof state.value === 'string' ? state.value : null), + segments: state.segments ?? telemetryData?.segments ?? null, + pixels: state.pixels ?? telemetryData?.pixels ?? state.buffer ?? null, + rawState: state, + }; + }); +} + +export function normalizeBoardPinStates(snapshot: SimulationSnapshot): BoardPinState[] { + return snapshot.boards.map((board) => { + const pins = snapshot.pinsByBoard[board.id] || {}; + const normalizedPins = Object.keys(pins) + .sort((a, b) => a.localeCompare(b)) + .map((pin) => ({ pin, high: !!pins[pin] })); + + return { + boardId: board.id, + type: board.type, + pins: normalizedPins, + }; + }); +} + +export function diffBoardPins( + before: SimulationSnapshot, + after: SimulationSnapshot +): Array<{ boardId: string; changedPins: Array<{ pin: string; before: boolean; after: boolean }> }> { + const boardIds = new Set([ + ...Object.keys(before.pinsByBoard || {}), + ...Object.keys(after.pinsByBoard || {}), + ]); + + const diffs: Array<{ boardId: string; changedPins: Array<{ pin: string; before: boolean; after: boolean }> }> = []; + for (const boardId of boardIds) { + const beforePins = before.pinsByBoard[boardId] || {}; + const afterPins = after.pinsByBoard[boardId] || {}; + const pinIds = new Set([...Object.keys(beforePins), ...Object.keys(afterPins)]); + const changedPins: Array<{ pin: string; before: boolean; after: boolean }> = []; + + for (const pinId of pinIds) { + const a = !!beforePins[pinId]; + const b = !!afterPins[pinId]; + if (a !== b) { + changedPins.push({ pin: pinId, before: a, after: b }); + } + } + + if (changedPins.length > 0) { + changedPins.sort((x, y) => x.pin.localeCompare(y.pin)); + diffs.push({ boardId, changedPins }); + } + } + + return diffs.sort((a, b) => a.boardId.localeCompare(b.boardId)); +} + +export function diffComponentStates( + before: SimulationSnapshot, + after: SimulationSnapshot, + componentId?: string | null +): Array<{ id: string; changedKeys: string[]; before: Record; after: Record }> { + const beforeById = new Map(before.components.map((entry) => [entry.id, asRecord(entry.state) || {}])); + const afterById = new Map(after.components.map((entry) => [entry.id, asRecord(entry.state) || {}])); + + const ids = componentId + ? [componentId] + : [...new Set([...beforeById.keys(), ...afterById.keys()])].sort((a, b) => a.localeCompare(b)); + + const diffs: Array<{ id: string; changedKeys: string[]; before: Record; after: Record }> = []; + + for (const id of ids) { + const b = beforeById.get(id) || {}; + const a = afterById.get(id) || {}; + const keys = new Set([...Object.keys(b), ...Object.keys(a)]); + const changed = [...keys] + .filter((key) => JSON.stringify((b as Record)[key]) !== JSON.stringify((a as Record)[key])) + .sort((x, y) => x.localeCompare(y)); + if (changed.length > 0) { + diffs.push({ id, changedKeys: changed, before: b, after: a }); + } + } + + return diffs; +} + +export function buildProfileEvents( + profile: string, + componentType: string, + durationMs: number +): Array<{ atMs: number; event: unknown }> { + const profileSpec = sensorProfilesForType(componentType).find((entry) => entry.name === profile); + if (!profileSpec) return []; + const base = Math.max(1, profileSpec.defaultDurationMs); + const scale = Math.max(0.05, durationMs / base); + return profileSpec.example.map((entry) => ({ + atMs: Math.max(0, Math.min(durationMs, Math.floor(entry.atMs * scale))), + event: entry.event, + })); +} + +export type AssertionCheck = + | { type: 'display_contains'; component_id?: string; text: string } + | { type: 'component_status'; component_id: string; status: 'ok' | 'warn' | 'error' } + | { type: 'pin_state'; board_id: string; pin: string; high: boolean }; + +export function evaluateAssertions(input: { + checks: AssertionCheck[]; + displays: DisplayStateView[]; + telemetry: SimulationTelemetryReport | null; + snapshot: SimulationSnapshot; +}): { + ok: boolean; + total: number; + passed: number; + results: Array<{ check: AssertionCheck; ok: boolean; actual: unknown }>; +} { + const results = input.checks.map((check) => { + if (check.type === 'display_contains') { + const displays = check.component_id + ? input.displays.filter((entry) => entry.id === check.component_id) + : input.displays; + const haystack = displays.map((entry) => String(entry.text || '')).join('\n'); + const ok = haystack.toLowerCase().includes(String(check.text || '').toLowerCase()); + return { + check, + ok, + actual: { + componentIds: displays.map((entry) => entry.id), + text: haystack, + }, + }; + } + + if (check.type === 'component_status') { + const component = input.telemetry?.components.find((entry) => entry.id === check.component_id) || null; + const actual = component?.status || 'missing'; + return { + check, + ok: actual === check.status, + actual, + }; + } + + const pinValue = !!(input.snapshot.pinsByBoard?.[check.board_id]?.[check.pin]); + return { + check, + ok: pinValue === !!check.high, + actual: pinValue, + }; + }); + + const passed = results.filter((entry) => entry.ok).length; + return { + ok: passed === results.length, + total: results.length, + passed, + results, + }; +} From 8dc85100b54abdd33372d82d05e74a12f3216c73 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:57:02 +0000 Subject: [PATCH 11/15] Fix new display and sensor scenario fixtures for valid component types/pins Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/acbe4ebc-9458-4884-b271-b9dd5b868ab4 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- .../scenarios/pico-display-observability.yaml | 4 ++-- .../scenarios/pico-sensor-observability.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openhw-studio-cli-danish/scenarios/pico-display-observability.yaml b/openhw-studio-cli-danish/scenarios/pico-display-observability.yaml index da0d317..0219994 100644 --- a/openhw-studio-cli-danish/scenarios/pico-display-observability.yaml +++ b/openhw-studio-cli-danish/scenarios/pico-display-observability.yaml @@ -6,7 +6,7 @@ envs: boardEnv: native ms: 1800 components: - - type: wokwi-ssd1306 + - type: wokwi-ssd1306-oled id: oled1 x: 300 y: 120 @@ -26,7 +26,7 @@ wires: - from: board1:GP2 to: led1:A - from: board1:GND - to: led1:C + to: led1:K inspect: - id: oled1 includeTrace: true diff --git a/openhw-studio-cli-danish/scenarios/pico-sensor-observability.yaml b/openhw-studio-cli-danish/scenarios/pico-sensor-observability.yaml index 6b534d6..4f3d277 100644 --- a/openhw-studio-cli-danish/scenarios/pico-sensor-observability.yaml +++ b/openhw-studio-cli-danish/scenarios/pico-sensor-observability.yaml @@ -6,7 +6,7 @@ envs: boardEnv: native ms: 2000 components: - - type: wokwi-photoresistor-sensor + - type: wokwi-ldr-module id: ldr1 x: 300 y: 120 From c1ef99bf0a0c80a8c7dc2e6fdfef535a95d76751 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:59:41 +0000 Subject: [PATCH 12/15] Address review feedback and tighten typing in observability features Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/acbe4ebc-9458-4884-b271-b9dd5b868ab4 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- openhw-studio-cli-danish/src/commands/sim.ts | 11 +++-- openhw-studio-cli-danish/src/mcp/server.ts | 47 ++++++++++++++++++- .../src/sim/agent-observability.ts | 9 +++- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/openhw-studio-cli-danish/src/commands/sim.ts b/openhw-studio-cli-danish/src/commands/sim.ts index 56fb6d5..39c0e34 100644 --- a/openhw-studio-cli-danish/src/commands/sim.ts +++ b/openhw-studio-cli-danish/src/commands/sim.ts @@ -487,7 +487,7 @@ async function writeOutputFile(targetPath: string, content: string): Promise { +async function parseScenarioFile(inputPath: string): Promise { const absolute = resolveWorkspacePath(inputPath); const raw = await fs.readFile(absolute, 'utf8'); const ext = path.extname(absolute).toLowerCase(); @@ -1295,7 +1295,8 @@ export function registerSimCommands(program: Command, getBackendUrl: () => strin if (options.assertionsFile) { const checksRaw = await parseScenarioFile(String(options.assertionsFile)); - const checks = Array.isArray(checksRaw?.assertions) ? checksRaw.assertions : checksRaw; + const checksRoot = (checksRaw && typeof checksRaw === 'object') ? (checksRaw as Record) : {}; + const checks = Array.isArray(checksRoot.assertions) ? checksRoot.assertions : checksRaw; payload.assertions = evaluateAssertions({ checks: Array.isArray(checks) ? checks : [], displays, @@ -1366,7 +1367,7 @@ export function registerSimCommands(program: Command, getBackendUrl: () => strin .option('--output ', 'Write scenario report JSON') .action(async (projectFile: string, options: any) => { const project = await loadProject(projectFile); - const scenario = await parseScenarioFile(String(options.scenario)); + const scenario = (await parseScenarioFile(String(options.scenario))) as Record; const durationMs = Math.max(1, parsePositiveInt(String(scenario?.durationMs || '1800'), 1800)); const runOptions: SimulationRunOptions = { backendUrl: getBackendUrl(), @@ -1378,11 +1379,11 @@ export function registerSimCommands(program: Command, getBackendUrl: () => strin }; const controller = await startSimulation(project, runOptions, { suppressConsoleOutput: true }); - const inputs = Array.isArray(scenario?.inputs) ? scenario.inputs : []; + const inputs = Array.isArray(scenario?.inputs) ? (scenario.inputs as Array>) : []; let cursorMs = 0; let deliveredAll = true; - for (const input of inputs.sort((a: any, b: any) => Number(a?.atMs || 0) - Number(b?.atMs || 0))) { + for (const input of inputs.sort((a, b) => Number(a?.atMs || 0) - Number(b?.atMs || 0))) { const atMs = Math.max(0, Math.min(durationMs, Number(input?.atMs || 0))); const waitMs = Math.max(0, atMs - cursorMs); if (waitMs > 0) await sleep(waitMs); diff --git a/openhw-studio-cli-danish/src/mcp/server.ts b/openhw-studio-cli-danish/src/mcp/server.ts index c8d6580..bf9c330 100644 --- a/openhw-studio-cli-danish/src/mcp/server.ts +++ b/openhw-studio-cli-danish/src/mcp/server.ts @@ -16,6 +16,7 @@ import { relToCwd, resolveWorkspacePath } from '../utils/paths.js'; import { startSimulation } from '../sim/session.js'; import { getManifestInfo, getPinsForType, listManifestInfos } from '../utils/manifests.js'; import { + type AssertionCheck, buildProfileEvents, componentInputSchemaForProject, diffBoardPins, @@ -513,6 +514,49 @@ function normalizeInputEventsForStep(options: { return events.sort((a, b) => a.atMs - b.atMs); } +function normalizeAssertionChecks(assertions: Array>): AssertionCheck[] { + const checks: AssertionCheck[] = []; + for (const entry of assertions) { + const type = String(entry?.type || '').trim(); + if (type === 'display_contains') { + const text = String(entry?.text || '').trim(); + if (!text) continue; + checks.push({ + type, + component_id: String(entry?.component_id || '').trim() || undefined, + text, + }); + continue; + } + + if (type === 'component_status') { + const componentId = String(entry?.component_id || '').trim(); + const status = String(entry?.status || '').trim(); + if (!componentId) continue; + if (status !== 'ok' && status !== 'warn' && status !== 'error') continue; + checks.push({ + type, + component_id: componentId, + status, + }); + continue; + } + + if (type === 'pin_state') { + const boardId = String(entry?.board_id || '').trim(); + const pin = String(entry?.pin || '').trim(); + if (!boardId || !pin) continue; + checks.push({ + type, + board_id: boardId, + pin, + high: !!entry?.high, + }); + } + } + return checks; +} + async function runSimulationForDuration( project: OpenHwProject, options: SimulationRunOptions, @@ -1280,8 +1324,9 @@ export async function runMcpServer(config: McpServerConfig): Promise { const snapshot = controller.getSnapshot(); const displays = extractDisplayStates(snapshot, telemetry); + const checks = normalizeAssertionChecks(assertions as Array>); const assertionResult = evaluateAssertions({ - checks: assertions as any, + checks, displays, telemetry, snapshot, diff --git a/openhw-studio-cli-danish/src/sim/agent-observability.ts b/openhw-studio-cli-danish/src/sim/agent-observability.ts index f04818a..28fa4d4 100644 --- a/openhw-studio-cli-danish/src/sim/agent-observability.ts +++ b/openhw-studio-cli-danish/src/sim/agent-observability.ts @@ -286,13 +286,20 @@ export function diffComponentStates( : [...new Set([...beforeById.keys(), ...afterById.keys()])].sort((a, b) => a.localeCompare(b)); const diffs: Array<{ id: string; changedKeys: string[]; before: Record; after: Record }> = []; + const equals = (left: unknown, right: unknown): boolean => { + if (left === right) return true; + const leftIsObject = left !== null && typeof left === 'object'; + const rightIsObject = right !== null && typeof right === 'object'; + if (!leftIsObject || !rightIsObject) return false; + return JSON.stringify(left) === JSON.stringify(right); + }; for (const id of ids) { const b = beforeById.get(id) || {}; const a = afterById.get(id) || {}; const keys = new Set([...Object.keys(b), ...Object.keys(a)]); const changed = [...keys] - .filter((key) => JSON.stringify((b as Record)[key]) !== JSON.stringify((a as Record)[key])) + .filter((key) => !equals((b as Record)[key], (a as Record)[key])) .sort((x, y) => x.localeCompare(y)); if (changed.length > 0) { diffs.push({ id, changedKeys: changed, before: b, after: a }); From 6a565e63b9a99c7ff2d6fec06038785eaf8022bb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Apr 2026 19:01:43 +0000 Subject: [PATCH 13/15] Refine display extraction readability in agent observability helper Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/acbe4ebc-9458-4884-b271-b9dd5b868ab4 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- .../src/sim/agent-observability.ts | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/openhw-studio-cli-danish/src/sim/agent-observability.ts b/openhw-studio-cli-danish/src/sim/agent-observability.ts index 28fa4d4..23d4771 100644 --- a/openhw-studio-cli-danish/src/sim/agent-observability.ts +++ b/openhw-studio-cli-danish/src/sim/agent-observability.ts @@ -202,15 +202,21 @@ export function extractDisplayStates( const state = asRecord(component.state) || {}; const telemetryEntry = telemetryById.get(component.id); const telemetryData = asRecord(telemetryEntry?.telemetryData); - const text = typeof state.text === 'string' - ? state.text - : typeof state.value === 'string' - ? state.value - : (typeof telemetryEntry?.outputSummary === 'string' ? telemetryEntry.outputSummary : null); - - const numeric = typeof state.value === 'number' - ? state.value - : (typeof state.number === 'number' ? state.number : null); + let text: string | null = null; + if (typeof state.text === 'string') { + text = state.text; + } else if (typeof state.value === 'string') { + text = state.value; + } else if (typeof telemetryEntry?.outputSummary === 'string') { + text = telemetryEntry.outputSummary; + } + + let numeric: number | null = null; + if (typeof state.value === 'number') { + numeric = state.value; + } else if (typeof state.number === 'number') { + numeric = state.number; + } return { id: component.id, From c04109803fedbba385e068935e0e949df69383e8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 23 Apr 2026 03:08:56 +0000 Subject: [PATCH 14/15] Unify component input contracts with internal logic/context-menu metadata Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/2e285869-755c-4bb0-8635-05c294f59e49 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- .../src/sim/agent-observability.ts | 103 ++++++++++- .../src/utils/manifests.ts | 168 ++++++++++++++++++ 2 files changed, 263 insertions(+), 8 deletions(-) diff --git a/openhw-studio-cli-danish/src/sim/agent-observability.ts b/openhw-studio-cli-danish/src/sim/agent-observability.ts index 23d4771..9fdab5e 100644 --- a/openhw-studio-cli-danish/src/sim/agent-observability.ts +++ b/openhw-studio-cli-danish/src/sim/agent-observability.ts @@ -11,6 +11,12 @@ export type ComponentInputSchema = { role: 'board' | 'input' | 'output' | 'other'; interactive: boolean; hasOnEvent: boolean; + contract: { + eventTypes: string[]; + controlKeys: string[]; + contextMenuDuringRun: boolean; + contextMenuOnlyDuringRun: boolean; + }; templates: ComponentInputTemplate[]; profiles: Array<{ name: string; @@ -50,7 +56,70 @@ export function classifyRole(type: string, group: string): 'board' | 'input' | ' return 'other'; } -export function interactionTemplatesForType(type: string): ComponentInputTemplate[] { +function camelizeLowerUnderscore(value: string): string { + const normalized = String(value || '').trim().toLowerCase(); + if (!normalized) return ''; + const chunks = normalized.split(/[_\s-]+/).filter(Boolean); + if (chunks.length === 0) return ''; + return `${chunks[0]}${chunks.slice(1).map((entry) => entry.charAt(0).toUpperCase() + entry.slice(1)).join('')}`; +} + +function templatesFromManifestContract(manifest: ManifestInfo | null): ComponentInputTemplate[] { + if (!manifest) return []; + const templates: ComponentInputTemplate[] = []; + const eventTypes = manifest.interaction?.eventTypes || []; + const controlKeys = manifest.interaction?.controlKeys || []; + + for (const template of manifest.interaction?.uiEventTemplates || []) { + templates.push(template); + } + + if (eventTypes.includes('SET_ATTR')) { + const keys = controlKeys.length > 0 ? controlKeys : Object.keys(manifest.attrs || {}); + for (const key of keys) { + templates.push({ + type: 'SET_ATTR', + key, + value: manifest.attrs?.[key]?.default ?? 0, + }); + } + } + + for (const eventType of eventTypes) { + if (!eventType || eventType === 'SET_ATTR') continue; + if (eventType === 'press' || eventType === 'release') { + templates.push(eventType); + continue; + } + + if (/^SET_[A-Z0-9_]+$/.test(eventType)) { + const normalized = camelizeLowerUnderscore(eventType.replace(/^SET_/, '')); + const attrKey = Object.keys(manifest.attrs || {}).find((key) => key.toLowerCase() === normalized.toLowerCase()); + templates.push({ + type: eventType, + value: attrKey ? (manifest.attrs?.[attrKey]?.default ?? 0) : 0, + }); + continue; + } + + templates.push({ type: eventType }); + } + + const dedup = new Set(); + return templates.filter((entry) => { + const key = JSON.stringify(entry); + if (dedup.has(key)) return false; + dedup.add(key); + return true; + }); +} + +export function interactionTemplatesForType(type: string, manifest?: ManifestInfo | null): ComponentInputTemplate[] { + const manifestTemplates = templatesFromManifestContract(manifest || null); + if (manifestTemplates.length > 0) { + return manifestTemplates; + } + const t = String(type || '').toLowerCase(); if (t.includes('pushbutton')) return ['press', 'release']; @@ -65,7 +134,7 @@ export function interactionTemplatesForType(type: string): ComponentInputTemplat ]; } if (t.includes('max30102')) { - return [{ type: 'SET_ATTR', key: 'heartRate', value: 75 }, { type: 'SET_ATTR', key: 'spo2', value: 98 }]; + return [{ type: 'SET_RED_LED', value: 24 }, { type: 'SET_IR_LED', value: 24 }]; } if (t.includes('dht')) { return [ @@ -139,14 +208,24 @@ export function sensorProfilesForType(type: string): ComponentInputSchema['profi if (t.includes('max30102')) { return [ + { + name: 'max30102_led_sweep', + description: 'Sweep MAX30102 red/IR LED drive currents via component onEvent controls.', + defaultDurationMs: 1200, + example: [ + { atMs: 100, event: { type: 'SET_RED_LED', value: 32 } }, + { atMs: 500, event: { type: 'SET_IR_LED', value: 40 } }, + { atMs: 900, event: { type: 'SET_RED_LED', value: 64 } }, + ], + }, { name: 'heart_rate_ramp', - description: 'Ramp heart rate and SpO2 readings.', + description: 'Backward-compatible alias of max30102_led_sweep.', defaultDurationMs: 1200, example: [ - { atMs: 100, event: { type: 'SET_ATTR', key: 'heartRate', value: 68 } }, - { atMs: 500, event: { type: 'SET_ATTR', key: 'heartRate', value: 88 } }, - { atMs: 900, event: { type: 'SET_ATTR', key: 'spo2', value: 97 } }, + { atMs: 100, event: { type: 'SET_RED_LED', value: 32 } }, + { atMs: 500, event: { type: 'SET_IR_LED', value: 40 } }, + { atMs: 900, event: { type: 'SET_RED_LED', value: 64 } }, ], }, ]; @@ -163,8 +242,10 @@ export function componentInputSchemaForProject( const manifest = manifestByType.get(component.type) || null; const group = manifest?.group || 'Other'; const role = classifyRole(component.type, group); - const templates = interactionTemplatesForType(component.type); + const templates = interactionTemplatesForType(component.type, manifest); const profiles = sensorProfilesForType(component.type); + const eventTypes = manifest?.interaction?.eventTypes || []; + const controlKeys = manifest?.interaction?.controlKeys || []; return { id: component.id, @@ -172,8 +253,14 @@ export function componentInputSchemaForProject( label: String(component.label || component.id), group, role, - interactive: !!manifest?.hasOnEvent || templates.length > 0, + interactive: !!manifest?.hasOnEvent || templates.length > 0 || eventTypes.length > 0, hasOnEvent: !!manifest?.hasOnEvent, + contract: { + eventTypes, + controlKeys, + contextMenuDuringRun: !!manifest?.interaction?.contextMenuDuringRun, + contextMenuOnlyDuringRun: !!manifest?.interaction?.contextMenuOnlyDuringRun, + }, templates, profiles, }; diff --git a/openhw-studio-cli-danish/src/utils/manifests.ts b/openhw-studio-cli-danish/src/utils/manifests.ts index f9e2acd..b318b8a 100644 --- a/openhw-studio-cli-danish/src/utils/manifests.ts +++ b/openhw-studio-cli-danish/src/utils/manifests.ts @@ -15,19 +15,112 @@ export interface ManifestTelemetryInfo { criticalKeys: string[]; } +export interface ManifestAttrInfo { + label?: string; + type?: string; + default?: unknown; + min?: number; + max?: number; + step?: number; +} + +export interface ManifestInteractionInfo { + eventTypes: string[]; + contextMenuDuringRun: boolean; + contextMenuOnlyDuringRun: boolean; + controlKeys: string[]; + uiEventTemplates: Array>; +} + export interface ManifestInfo { type: string; label: string; group: string; pins: ManifestPinInfo[]; pinIds: Set; + attrs: Record; hasOnEvent: boolean; + interaction?: ManifestInteractionInfo; telemetry?: ManifestTelemetryInfo; } const MANIFEST_CACHE = new Map(); let loaded = false; +function parseLogicEventTypes(logicRaw: string): string[] { + const events = new Set(); + + const caseRegex = /case\s+['"`]([^'"`]+)['"`]\s*:/g; + let match: RegExpExecArray | null; + while ((match = caseRegex.exec(logicRaw))) { + const eventType = String(match[1] || '').trim(); + if (eventType) events.add(eventType); + } + + const equalsRegex = /event(?:\??\.type)?\s*===\s*['"`]([^'"`]+)['"`]/g; + while ((match = equalsRegex.exec(logicRaw))) { + const eventType = String(match[1] || '').trim(); + if (eventType) events.add(eventType); + } + + return [...events].sort((a, b) => a.localeCompare(b)); +} + +function parseControlKeysFromUi(uiRaw: string): string[] { + const keys = new Set(); + const keyRegex = /(?:handleSlider|onUpdate)\(\s*['"`]([^'"`]+)['"`]\s*,/g; + let match: RegExpExecArray | null; + while ((match = keyRegex.exec(uiRaw))) { + const key = String(match[1] || '').trim(); + if (key) keys.add(key); + } + return [...keys].sort((a, b) => a.localeCompare(b)); +} + +function parseUiEventTemplates( + uiRaw: string, + attrs: Record +): Array> { + const templates: Array> = []; + const onInteractRegex = /onInteract(?:\?\.|)\s*\(\s*\{([\s\S]*?)\}\s*\)/g; + let match: RegExpExecArray | null; + while ((match = onInteractRegex.exec(uiRaw))) { + const body = String(match[1] || ''); + const typeMatch = /type:\s*['"`]([^'"`]+)['"`]/.exec(body); + const type = String(typeMatch?.[1] || '').trim(); + if (!type) continue; + + const keyMatch = /key:\s*['"`]([^'"`]+)['"`]/.exec(body); + const key = String(keyMatch?.[1] || '').trim(); + const template: Record = { type }; + + if (key) { + template.key = key; + template.value = attrs[key]?.default ?? 0; + } else { + template.value = 0; + } + + templates.push(template); + } + + const dedup = new Set(); + return templates.filter((entry) => { + const key = JSON.stringify(entry); + if (dedup.has(key)) return false; + dedup.add(key); + return true; + }); +} + +function parseContextFlag(raw: string, key: 'contextMenuDuringRun' | 'contextMenuOnlyDuringRun'): boolean | null { + const trueRegex = new RegExp(`${key}\\s*:\\s*true`); + if (trueRegex.test(raw)) return true; + const falseRegex = new RegExp(`${key}\\s*:\\s*false`); + if (falseRegex.test(raw)) return false; + return null; +} + async function walk(dirPath: string, output: string[]): Promise { const entries = await fs.readdir(dirPath, { withFileTypes: true }); for (const entry of entries) { @@ -56,6 +149,9 @@ async function loadAllManifests(): Promise { type?: string; label?: string; group?: string; + attrs?: Record; + contextMenuDuringRun?: unknown; + contextMenuOnlyDuringRun?: unknown; pins?: Array<{ id?: string; x?: number; y?: number; type?: string; description?: string }>; telemetry?: { template?: unknown; @@ -81,14 +177,76 @@ async function loadAllManifests(): Promise { } let hasOnEvent = false; + let eventTypes: string[] = []; const logicPath = path.join(path.dirname(manifestPath), 'logic.ts'); try { const logicRaw = await fs.readFile(logicPath, 'utf8'); hasOnEvent = /\bonEvent\s*\(/.test(logicRaw); + if (hasOnEvent) { + eventTypes = parseLogicEventTypes(logicRaw); + } } catch { hasOnEvent = false; + eventTypes = []; + } + + const attrsRaw = parsed.attrs && typeof parsed.attrs === 'object' && !Array.isArray(parsed.attrs) + ? (parsed.attrs as Record) + : {}; + const attrs: Record = {}; + for (const [attrKey, attrValue] of Object.entries(attrsRaw)) { + if (!attrKey) continue; + if (attrValue && typeof attrValue === 'object' && !Array.isArray(attrValue)) { + const src = attrValue as Record; + attrs[attrKey] = { + label: typeof src.label === 'string' ? src.label : undefined, + type: typeof src.type === 'string' ? src.type : undefined, + default: src.default, + min: Number.isFinite(Number(src.min)) ? Number(src.min) : undefined, + max: Number.isFinite(Number(src.max)) ? Number(src.max) : undefined, + step: Number.isFinite(Number(src.step)) ? Number(src.step) : undefined, + }; + continue; + } + attrs[attrKey] = { default: attrValue }; } + let indexRaw = ''; + try { + indexRaw = await fs.readFile(path.join(path.dirname(manifestPath), 'index.ts'), 'utf8'); + } catch { + indexRaw = ''; + } + + let uiRaw = ''; + for (const fileName of ['ui.tsx', 'ui.ts']) { + try { + uiRaw = await fs.readFile(path.join(path.dirname(manifestPath), fileName), 'utf8'); + break; + } catch { + // Try next UI filename. + } + } + + const uiEventTemplates = uiRaw ? parseUiEventTemplates(uiRaw, attrs) : []; + const controlKeys = uiRaw ? parseControlKeysFromUi(uiRaw) : []; + const interactionEventTypes = [...new Set([ + ...eventTypes, + ...uiEventTemplates + .map((entry) => (entry && typeof entry === 'object' ? String((entry as Record).type || '').trim() : '')) + .filter(Boolean), + ])].sort((a, b) => a.localeCompare(b)); + + const manifestContextMenuDuringRun = + typeof parsed.contextMenuDuringRun === 'boolean' ? parsed.contextMenuDuringRun : null; + const manifestContextMenuOnlyDuringRun = + typeof parsed.contextMenuOnlyDuringRun === 'boolean' ? parsed.contextMenuOnlyDuringRun : null; + const indexContextMenuDuringRun = indexRaw ? parseContextFlag(indexRaw, 'contextMenuDuringRun') : null; + const indexContextMenuOnlyDuringRun = indexRaw ? parseContextFlag(indexRaw, 'contextMenuOnlyDuringRun') : null; + + const contextMenuDuringRun = manifestContextMenuDuringRun ?? indexContextMenuDuringRun ?? false; + const contextMenuOnlyDuringRun = manifestContextMenuOnlyDuringRun ?? indexContextMenuOnlyDuringRun ?? false; + const telemetryRaw = parsed.telemetry; const telemetry = telemetryRaw && typeof telemetryRaw === 'object' ? { @@ -105,7 +263,17 @@ async function loadAllManifests(): Promise { group: String(parsed.group || 'Other'), pins, pinIds, + attrs, hasOnEvent, + interaction: (hasOnEvent || uiEventTemplates.length > 0 || controlKeys.length > 0 || contextMenuDuringRun || contextMenuOnlyDuringRun) + ? { + eventTypes: interactionEventTypes, + contextMenuDuringRun, + contextMenuOnlyDuringRun, + controlKeys, + uiEventTemplates, + } + : undefined, telemetry, }); } catch { From 7f970c2f3a1e60608d6dc52c471e29bafe619836 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 23 Apr 2026 03:23:38 +0000 Subject: [PATCH 15/15] Add real onEvent templates for more interactive components Agent-Logs-Url: https://github.com/danish9661/Arduino-simulator/sessions/a123dc4a-45fb-405a-8436-ab7dd7bf2343 Co-authored-by: danish9661 <110881758+danish9661@users.noreply.github.com> --- .../src/sim/agent-observability.ts | 105 +++++++++++++++--- 1 file changed, 89 insertions(+), 16 deletions(-) diff --git a/openhw-studio-cli-danish/src/sim/agent-observability.ts b/openhw-studio-cli-danish/src/sim/agent-observability.ts index 9fdab5e..425fb7c 100644 --- a/openhw-studio-cli-danish/src/sim/agent-observability.ts +++ b/openhw-studio-cli-danish/src/sim/agent-observability.ts @@ -64,6 +64,91 @@ function camelizeLowerUnderscore(value: string): string { return `${chunks[0]}${chunks.slice(1).map((entry) => entry.charAt(0).toUpperCase() + entry.slice(1)).join('')}`; } +function hasTemplateType(templates: ComponentInputTemplate[], type: string): boolean { + const wanted = String(type || '').trim(); + if (!wanted) return false; + return templates.some((entry) => { + if (typeof entry === 'string') return entry === wanted; + if (!entry || typeof entry !== 'object') return false; + return String((entry as Record).type || '').trim() === wanted; + }); +} + +function synthesizeTemplateForEventType(eventType: string, manifest: ManifestInfo): ComponentInputTemplate { + if (eventType === 'press' || eventType === 'release' || eventType === 'rotate-cw' || eventType === 'rotate-ccw') { + return eventType; + } + + if (eventType === 'move') { + return { type: 'move', x: 0.5, y: 0.5 }; + } + + if (eventType === 'input') { + return { type: 'input', value: manifest.attrs?.value?.default ?? 50 }; + } + + if (eventType === 'SD_MOUNT' || eventType === 'MOUNT') { + return { type: eventType }; + } + if (eventType === 'SD_UNMOUNT' || eventType === 'UNMOUNT' || eventType === 'EJECT') { + return { type: eventType }; + } + if (eventType === 'SD_FORMAT' || eventType === 'FORMAT') { + return { type: eventType }; + } + if (eventType === 'SD_WRITE_FILE' || eventType === 'WRITE_FILE') { + return { type: eventType, path: '/LOG.TXT', data: '' }; + } + if (eventType === 'SD_READ_FILE' || eventType === 'READ_FILE') { + return { type: eventType, path: '/README.TXT' }; + } + if (eventType === 'SD_DELETE_FILE' || eventType === 'DELETE_FILE') { + return { type: eventType, path: '/README.TXT' }; + } + + if (/^SET_[A-Z0-9_]+$/.test(eventType)) { + const normalized = camelizeLowerUnderscore(eventType.replace(/^SET_/, '')); + const attrKey = Object.keys(manifest.attrs || {}).find((key) => key.toLowerCase() === normalized.toLowerCase()); + return { + type: eventType, + value: attrKey ? (manifest.attrs?.[attrKey]?.default ?? 0) : 0, + }; + } + + return { type: eventType }; +} + +function appendComponentSpecificTemplates(templates: ComponentInputTemplate[], manifest: ManifestInfo): void { + const type = String(manifest.type || '').toLowerCase(); + + if (type.includes('membrane-keypad')) { + const keypadKeys = ['1', '2', '3', 'A', '4', '5', '6', 'B', '7', '8', '9', 'C', '*', '0', '#', 'D']; + for (const key of keypadKeys) { + const eventName = `press:${key}`; + if (!templates.includes(eventName)) templates.push(eventName); + } + if (!templates.includes('release')) templates.push('release'); + return; + } + + if (type.includes('wokwi-sd-card')) { + const sdTemplates: ComponentInputTemplate[] = [ + { type: 'SD_MOUNT' }, + { type: 'SD_UNMOUNT' }, + { type: 'SD_FORMAT' }, + { type: 'SD_WRITE_FILE', path: '/LOG.TXT', data: '' }, + { type: 'SD_READ_FILE', path: '/README.TXT' }, + { type: 'SD_DELETE_FILE', path: '/README.TXT' }, + ]; + for (const entry of sdTemplates) { + const key = JSON.stringify(entry); + if (!templates.some((existing) => JSON.stringify(existing) === key)) { + templates.push(entry); + } + } + } +} + function templatesFromManifestContract(manifest: ManifestInfo | null): ComponentInputTemplate[] { if (!manifest) return []; const templates: ComponentInputTemplate[] = []; @@ -87,24 +172,12 @@ function templatesFromManifestContract(manifest: ManifestInfo | null): Component for (const eventType of eventTypes) { if (!eventType || eventType === 'SET_ATTR') continue; - if (eventType === 'press' || eventType === 'release') { - templates.push(eventType); - continue; - } - - if (/^SET_[A-Z0-9_]+$/.test(eventType)) { - const normalized = camelizeLowerUnderscore(eventType.replace(/^SET_/, '')); - const attrKey = Object.keys(manifest.attrs || {}).find((key) => key.toLowerCase() === normalized.toLowerCase()); - templates.push({ - type: eventType, - value: attrKey ? (manifest.attrs?.[attrKey]?.default ?? 0) : 0, - }); - continue; - } - - templates.push({ type: eventType }); + if (hasTemplateType(templates, eventType)) continue; + templates.push(synthesizeTemplateForEventType(eventType, manifest)); } + appendComponentSpecificTemplates(templates, manifest); + const dedup = new Set(); return templates.filter((entry) => { const key = JSON.stringify(entry);