diff --git a/README.en.md b/README.en.md index fde2587..0217d37 100644 --- a/README.en.md +++ b/README.en.md @@ -41,6 +41,7 @@ With any provider enabled, Codex CLI's model picker shows ` / , +} + +/// Codex CLI 把 `apply_patch` 作为 freeform 工具注册 +/// (`codex-rs/core/src/tools/handlers/apply_patch_spec.rs` — +/// `ToolSpec::Freeform { name: "apply_patch", ... }`),响应侧 router +/// (`codex-rs/core/src/tools/router.rs:92-130`)按 wire item type 路由: +/// `ResponseItem::FunctionCall` → `ToolPayload::Function { arguments }`, +/// `ResponseItem::CustomToolCall` → `ToolPayload::Custom { input }`,而 +/// apply_patch handler 硬要求 `ToolPayload::Custom`,收 Function 直接返回 +/// `"apply_patch handler received unsupported payload"` → abort +/// (`codex-rs/core/src/tools/handlers/apply_patch.rs:324`)。本 adapter +/// 把 chat completions provider(DeepSeek / Kimi / MiMo 等)回来的 +/// `tool_calls[]` 默认渲染成 `function_call` wire,所以必须对 apply_patch +/// 特判 — 用 `custom_tool_call` wire 给 Codex CLI 才不 abort。 +/// +/// 名字以常量集中是为了和 `request/tools.rs::APPLY_PATCH_TOOL_NAME` 对齐 +/// 字符串一致性(请求侧的特判描述 / 响应侧的 wire 重打包必须按同一 name 触发)。 +fn is_apply_patch_tool_name(name: &str) -> bool { + name == "apply_patch" +} + +/// Codex CLI 的 shell 工具名(从 strings dump 验证:`exec_command` 是 Codex +/// CLI 在 chat function-call provider 上注册的 shell 工具)。 +/// 命中此名 + args.cmd 是 file-write 模式 → 触发 `shell_to_apply_patch` server-side +/// normalize,改写 wire 为 `custom_tool_call apply_patch` 让 Codex Desktop 的 +/// V4A applier 渲染 diff UI,而不是真跑 shell。 +fn is_exec_command_tool_name(name: &str) -> bool { + name == "exec_command" } #[derive(Debug)] @@ -489,6 +551,14 @@ impl ChatToResponsesConverter { .clone() .unwrap_or_else(|| format!("call_{}_{}", self.fc_id_seed, openai_index)); let name = tc.function.name.clone().unwrap_or_default(); + // **取舍**:wire 形态(function_call vs custom_tool_call)在 open + // 时一次性根据**首帧 name** 决定,后续帧补全 name 不改 wire。 + // 实测 DeepSeek / Kimi / MiMo 都在首帧带 name。极端情况下首帧 + // name 为空、后续才补 apply_patch,会 fallback 到 function_call + // wire(同当前行为,Codex CLI 仍会 abort apply_patch 一次),不 + // 比修复前差。 + let is_apply_patch = is_apply_patch_tool_name(&name); + let is_exec_command = is_exec_command_tool_name(&name); self.tool_calls.insert( openai_index, PendingToolCall { @@ -498,35 +568,97 @@ impl ChatToResponsesConverter { name: name.clone(), args_acc: String::new(), closed: false, + is_apply_patch, + is_exec_command, + output_item_added_emitted: false, + apply_patch_input: None, }, ); - // 如果 function name 来自 namespace 包(从 original_request.tools - // 反查表查到),给 item 加 `namespace` 字段 — Codex.app 客户端 - // dispatch namespace 工具时这是必要字段(strings 实证 binary 含 - // `dynamic tool namespace must not be empty for` 校验,缺字段会 - // 报 `unsupported call: `)。 - let namespace = self.lookup_namespace_for(&name).map(str::to_owned); - let mut item = json!({ - "type": "function_call", - "id": fc_id, - "call_id": call_id, - "name": name, - "arguments": "", - "status": "in_progress", - }); - if let Some(ns) = namespace.as_ref() { - item["namespace"] = Value::String(ns.clone()); + // apply_patch:wire 必须是 `custom_tool_call`(裸 `input` 字段)。 + // 中间增量 delta **不 emit** — chat 上游给的 args 是 JSON 字符串 + // 增量(`{"input": "*** Begin Patch\n..."`),从 JSON 字符串拼接 + // 过程中流式提取 `input` 字段值需要专门的 streaming JSON state + // machine,本提交不引入。退而求其次:close 时一次性解 args 再 + // emit input.delta + output_item.done,代价是客户端看不到逐字 + // 流出的 diff(一次性出现整段 patch)。对一个长期完全不工作的 + // 功能,这是合理的第一步;后续可优化为真流式。 + if is_apply_patch { + tracing::info!( + target = "adapters::apply_patch", + call_id = %call_id, + "apply_patch shim engaged: rewriting chat function_call wire to Responses custom_tool_call", + ); + let item = json!({ + "type": "custom_tool_call", + "id": fc_id, + "call_id": call_id, + "name": name, + "input": "", + "status": "in_progress", + }); + emit_event( + out, + &mut self.sequence_number, + "response.output_item.added", + json!({ + "type": "response.output_item.added", + "output_index": output_index, + "item": item, + }), + ); + } else if is_exec_command { + // server-side normalize:不立即 emit `output_item.added`。 + // 整个 item 走 buffer 路径,到 `close_tool_call` run detector + // 后再决定 wire 形态(custom_tool_call apply_patch 或回退 + // function_call exec_command)。output_item_added_emitted + // 保持 false,delta/done 都延迟到 close。 + tracing::debug!( + target = "adapters::shell_to_apply_patch", + call_id = %call_id, + "exec_command buffering engaged; SSE emit deferred until close (file-write detection)", + ); + } else { + // 如果 function name 来自 namespace 包(从 original_request.tools + // 反查表查到),给 item 加 `namespace` 字段 — Codex.app 客户端 + // dispatch namespace 工具时这是必要字段(strings 实证 binary 含 + // `dynamic tool namespace must not be empty for` 校验,缺字段会 + // 报 `unsupported call: `)。 + let namespace = self.lookup_namespace_for(&name).map(str::to_owned); + let mut item = json!({ + "type": "function_call", + "id": fc_id, + "call_id": call_id, + "name": name, + "arguments": "", + "status": "in_progress", + }); + if let Some(ns) = namespace.as_ref() { + item["namespace"] = Value::String(ns.clone()); + } + emit_event( + out, + &mut self.sequence_number, + "response.output_item.added", + json!({ + "type": "response.output_item.added", + "output_index": output_index, + "item": item, + }), + ); + } + // output_item.added 已 emit。后续帧 backfill `id` 不应再换 call_id + // (否则 `output_item.added` 与后续 `input.delta` / `output_item.done` + // 用不同 call_id,严格客户端会两次解读为不同 item)。同样地, + // apply_patch 的 `is_apply_patch` 决策也已固定。 + // + // 例外:`is_exec_command` 走 server-side normalize buffer 路径, + // 当前 frame **没有** emit `output_item.added`(延迟到 close), + // 后续帧 backfill 的 call_id 仍可替换。 + if let Some(pending) = self.tool_calls.get_mut(&openai_index) { + if !pending.is_exec_command { + pending.output_item_added_emitted = true; + } } - emit_event( - out, - &mut self.sequence_number, - "response.output_item.added", - json!({ - "type": "response.output_item.added", - "output_index": output_index, - "item": item, - }), - ); } // 后续帧可能补全 name(罕见但兼容) @@ -535,16 +667,31 @@ impl ChatToResponsesConverter { if let Some(pending) = self.tool_calls.get_mut(&openai_index) { if pending.name.is_empty() { pending.name = name.to_owned(); + if is_apply_patch_tool_name(name) && !pending.is_apply_patch { + // 罕见极端:首帧 name 为空,后续才补 apply_patch。 + // `output_item.added` 已经 emit `function_call` wire, + // 不能回退。这一调用 Codex CLI 仍会 abort,但起码我们 + // 在日志里能看到根因。 + tracing::warn!( + target = "adapters::apply_patch", + call_id = %pending.call_id, + "apply_patch tool name arrived AFTER first frame; wire stays function_call and Codex CLI will reject. Investigate upstream provider chunking.", + ); + } } } } } - // call_id 也可能在后续帧才出现 + // call_id 也可能在后续帧才出现 — 但只在 `output_item.added` 还没 emit + // 时才允许替换。已 emit 后再换会让客户端看到同一 item 用两个不同 + // call_id。 if let Some(id) = tc.id.as_deref() { if !id.is_empty() { if let Some(pending) = self.tool_calls.get_mut(&openai_index) { - // 只在首次给出 id 时覆盖(避免相同 index 不同 id 的混乱) - if pending.call_id.starts_with("call_") && pending.call_id.contains('_') { + if pending.output_item_added_emitted { + // 不再覆盖 — 同 item 已经对外暴露 call_id。 + } else if pending.call_id.starts_with("call_") && pending.call_id.contains('_') + { // 兜底生成的 call_id 形如 `call__`,真 id 来了就替换 if !pending.call_id.starts_with(id) && pending.call_id != id { pending.call_id = id.to_owned(); @@ -554,11 +701,22 @@ impl ChatToResponsesConverter { } } - // arguments delta(增量字符串) + // arguments delta(增量字符串)。apply_patch 路径**只**累积不 emit + // (理由见上文 open 处注释);非 apply_patch 仍逐 chunk emit + // `function_call_arguments.delta` 让客户端看到逐字流。 if let Some(args) = tc.function.arguments.as_deref() { if !args.is_empty() { if let Some(pending) = self.tool_calls.get_mut(&openai_index) { pending.args_acc.push_str(args); + if pending.is_apply_patch { + return; + } + // server-side normalize buffer 路径:exec_command 不 emit + // 增量 delta,close 时一次性 emit(改写为 apply_patch 或 + // 回退 function_call)。 + if pending.is_exec_command { + return; + } let item_id = pending.fc_id.clone(); let output_index = pending.output_index; emit_event( @@ -577,10 +735,19 @@ impl ChatToResponsesConverter { } } - fn close_tool_call(&mut self, openai_index: u32, out: &mut Vec) { + fn close_tool_call(&mut self, openai_index: u32, interrupted: bool, out: &mut Vec) { // 先把所有需要的字段 clone 出来,避免 mutable borrow 跟 // self.lookup_namespace_for 的 immutable borrow 冲突 - let (fc_id, call_id, name, args_acc, output_index, already_closed) = { + let ( + fc_id, + call_id, + name, + args_acc, + output_index, + already_closed, + is_apply_patch, + is_exec_command, + ) = { let Some(pending) = self.tool_calls.get(&openai_index) else { return; }; @@ -591,11 +758,234 @@ impl ChatToResponsesConverter { pending.args_acc.clone(), pending.output_index, pending.closed, + pending.is_apply_patch, + pending.is_exec_command, ) }; if already_closed { return; } + + // ===== server-side normalize: exec_command → apply_patch ===== + // 仅在非 interrupted 路径检测;interrupted 时模型未完成意图表达,即使 + // detect 在 partial args 上意外匹配也不应执行(等同于伪造模型本不打算 + // 的 apply_patch 调用)。强制让 interrupted exec_command 走"incomplete + // 占位"路径,**对称** apply_patch interrupted 处理:emit + // `status="incomplete"` 让 Codex Desktop 看到 partial item 不去执行 + // shell(防止跑半截 here-doc 等 destructive 行为 — code-reviewer + // IMPORTANT-3 修复)。 + if is_exec_command && interrupted { + tracing::warn!( + target = "adapters::shell_to_apply_patch", + call_id = %call_id, + args_len = args_acc.len(), + "exec_command tool call cut off mid-stream. Emitting output_item with status=incomplete to prevent Codex Desktop from executing partial shell (e.g. truncated here-doc).", + ); + // buffer 期 added 没 emit,补 emit 一个 in_progress 占位让客户端 + // 看到这条 item 存在(否则后续 output_item.done 会找不到对应 item)。 + self.emit_deferred_exec_command_added(&fc_id, &call_id, &name, output_index, out); + if !args_acc.is_empty() { + emit_event( + out, + &mut self.sequence_number, + "response.function_call_arguments.delta", + json!({ + "type": "response.function_call_arguments.delta", + "item_id": fc_id, + "output_index": output_index, + "delta": args_acc, + }), + ); + } + // 直接 emit `output_item.done status=incomplete` 不走下方默认 + // function_call 路径(那条会 emit `status=completed`,Codex + // Desktop 看到 completed 就执行 shell)。**不**写 ToolCallCache, + // 让 orphan-repair 路径补占位避免 partial 上下文污染下一 turn。 + let namespace = self.lookup_namespace_for(&name).map(str::to_owned); + let mut item = json!({ + "type": "function_call", + "id": fc_id, + "call_id": call_id, + "name": name, + "arguments": args_acc, + "status": "incomplete", + }); + if let Some(ns) = namespace.as_ref() { + item["namespace"] = Value::String(ns.clone()); + } + emit_event( + out, + &mut self.sequence_number, + "response.output_item.done", + json!({ + "type": "response.output_item.done", + "output_index": output_index, + "item": item, + }), + ); + if let Some(pending) = self.tool_calls.get_mut(&openai_index) { + pending.closed = true; + } + return; + } else if is_exec_command && !interrupted { + if let Some(patch) = detect_shell_file_write_in_exec_args(&args_acc) { + tracing::info!( + target = "adapters::shell_to_apply_patch", + call_id = %call_id, + target_path = %patch.target_path, + patch_bytes = patch.patch_input.len(), + "rewriting exec_command file-write to apply_patch custom_tool_call", + ); + self.emit_normalized_apply_patch( + openai_index, + &fc_id, + &call_id, + output_index, + patch.patch_input, + out, + ); + return; + } + // 未匹配 file-write → 继续走下方 function_call 路径,但需要补 + // emit `output_item.added` + `function_call_arguments.delta`(open + // 阶段被 buffer 跳过了,客户端收不到 delta 累积,需要在 .done + // 之前补一段含整个 args 的 delta 让客户端按协议拼装)。 + self.emit_deferred_exec_command_added(&fc_id, &call_id, &name, output_index, out); + if !args_acc.is_empty() { + emit_event( + out, + &mut self.sequence_number, + "response.function_call_arguments.delta", + json!({ + "type": "response.function_call_arguments.delta", + "item_id": fc_id, + "output_index": output_index, + "delta": args_acc, + }), + ); + } + // fall through to function_call close path + } + + if is_apply_patch { + // 从累积的 chat function args(标准形态 `{"input":""}`) + // 提取裸 V4A 文本。降级:模型可能直接吐裸 V4A(不包 JSON)— 历史 + // 上 freeform 工具的输出就是这个形态,某些 chat 上游可能没把它 + // 重新包成 JSON。fallback 把 args_acc 整段当 input,让上游能看到 + // 解析失败的具体内容(对调试 + 让 apply_patch parser 给出可读 + // 错误而不是静默 abort 都有用)。 + if args_acc.trim().is_empty() { + tracing::warn!( + target = "adapters::apply_patch", + call_id = %call_id, + "apply_patch tool was called with empty arguments — model likely misbehaving or provider stripped args", + ); + } + let input = extract_apply_patch_input(&args_acc); + // 缓存 input 到 pending,供 `tool_call_item_completed`(envelope + // output[] 终态)读,避免重复 parse 与潜在 drift。 + if let Some(pending) = self.tool_calls.get_mut(&openai_index) { + pending.apply_patch_input = Some(input.clone()); + } + // interrupted 中断时,patch 文本可能 mid-stream 被截断 — emit + // `status="incomplete"` 让 Codex CLI 看到 apply_patch handler 不 + // 应该执行 partial patch(apply_patch destructive,partial 执行 + // 可能在意外目标上写入意外内容)。同时 skip `input.done`(很多 + // 严格客户端在 `.done` 才触发执行)。 + if interrupted { + tracing::warn!( + target = "adapters::apply_patch", + call_id = %call_id, + args_len = args_acc.len(), + "apply_patch tool call cut off mid-stream (no finish_reason and not from [DONE]). Emitting output_item with status=incomplete; skipping input.done to prevent partial patch execution.", + ); + let item = json!({ + "type": "custom_tool_call", + "id": fc_id, + "call_id": call_id, + "name": name, + "input": input, + "status": "incomplete", + }); + emit_event( + out, + &mut self.sequence_number, + "response.output_item.done", + json!({ + "type": "response.output_item.done", + "output_index": output_index, + "item": item, + }), + ); + // 不存 cache(下一轮如果引用此 call_id 重建会拿到 incomplete + // 上下文,反而误导;让 orphan repair 路径补占位)。 + if let Some(pending) = self.tool_calls.get_mut(&openai_index) { + pending.closed = true; + } + return; + } + // open 阶段 emit 了空 input 的 output_item.added;这里一次性补 + // input.delta + output_item.done,让 Codex CLI 的 streaming + // parser(`StreamingPatchParser`)拿到完整 patch 文本后 finish。 + emit_event( + out, + &mut self.sequence_number, + "response.custom_tool_call_input.delta", + json!({ + "type": "response.custom_tool_call_input.delta", + "item_id": fc_id, + "output_index": output_index, + "call_id": call_id, + "delta": input, + }), + ); + emit_event( + out, + &mut self.sequence_number, + "response.custom_tool_call_input.done", + json!({ + "type": "response.custom_tool_call_input.done", + "item_id": fc_id, + "output_index": output_index, + "call_id": call_id, + "input": input, + }), + ); + let item = json!({ + "type": "custom_tool_call", + "id": fc_id, + "call_id": call_id, + "name": name, + "input": input, + "status": "completed", + }); + emit_event( + out, + &mut self.sequence_number, + "response.output_item.done", + json!({ + "type": "response.output_item.done", + "output_index": output_index, + "item": item, + }), + ); + // ToolCallCache 用于下一轮 Codex CLI 发 tool output 时重建工具 + // 调用上下文。回灌走 chat completions(messages.tool_calls.function + // .arguments 是 JSON 字符串),所以这里仍存原始 args_acc(JSON 形态) + // 而不是 input 裸文本,与 `assistant_message` 的 tool_calls 形态对齐。 + global_tool_call_cache().save( + &call_id, + ToolCallEntry { + name: name.clone(), + arguments: args_acc.clone(), + }, + ); + if let Some(pending) = self.tool_calls.get_mut(&openai_index) { + pending.closed = true; + } + return; + } + emit_event( out, &mut self.sequence_number, @@ -644,6 +1034,153 @@ impl ChatToResponsesConverter { } } + /// server-side normalize:detected file-write exec_command 转 apply_patch + /// custom_tool_call。一次性 emit 完整序列(added → input.delta → input.done + /// → output_item.done),让 Codex Desktop 把它当 apply_patch 调用渲染 diff UI。 + /// + /// **call_id 复用**:用模型原 exec_command 的 call_id,这样下一 turn + /// Codex Desktop 把 apply_patch 执行结果(custom_tool_call_output)按同 + /// call_id 回灌给模型,模型按自己的 turn 历史关联得上。 + /// + /// **名字改为 `apply_patch`**:Codex Desktop V4A applier 看 name 字段 + /// dispatch handler,必须是 `apply_patch` 才会跑 V4A parse + diff UI + /// 渲染。 + /// + /// **缓存**:apply_patch_input 缓到 pending,envelope 终态读这个;同时 + /// ToolCallCache 用合成的 `{"input":}` JSON 形态存,跟 chat + /// completions assistant.tool_calls.function.arguments 形态对齐(下一 + /// turn `repair_tool_call_ids` 路径 B 重建上下文用)。 + fn emit_normalized_apply_patch( + &mut self, + openai_index: u32, + fc_id: &str, + call_id: &str, + output_index: u32, + v4a_input: String, + out: &mut Vec, + ) { + const APPLY_PATCH_NAME: &str = "apply_patch"; + // output_item.added(空 input,in_progress)— Codex Desktop 看到这条 + // 才会创建 pending custom_tool_call item;后续 delta/done 在同一 + // item 上累积。 + emit_event( + out, + &mut self.sequence_number, + "response.output_item.added", + json!({ + "type": "response.output_item.added", + "output_index": output_index, + "item": { + "type": "custom_tool_call", + "id": fc_id, + "call_id": call_id, + "name": APPLY_PATCH_NAME, + "input": "", + "status": "in_progress", + }, + }), + ); + emit_event( + out, + &mut self.sequence_number, + "response.custom_tool_call_input.delta", + json!({ + "type": "response.custom_tool_call_input.delta", + "item_id": fc_id, + "output_index": output_index, + "call_id": call_id, + "delta": v4a_input, + }), + ); + emit_event( + out, + &mut self.sequence_number, + "response.custom_tool_call_input.done", + json!({ + "type": "response.custom_tool_call_input.done", + "item_id": fc_id, + "output_index": output_index, + "call_id": call_id, + "input": v4a_input, + }), + ); + emit_event( + out, + &mut self.sequence_number, + "response.output_item.done", + json!({ + "type": "response.output_item.done", + "output_index": output_index, + "item": { + "type": "custom_tool_call", + "id": fc_id, + "call_id": call_id, + "name": APPLY_PATCH_NAME, + "input": v4a_input, + "status": "completed", + }, + }), + ); + // 缓存:envelope 终态读 apply_patch_input;ToolCallCache 存 chat + // 形态 `{"input":}` 让下一 turn repair 路径用合成 args 重建上下文, + // 同时把 name 切换为 apply_patch 让 cache 一致。 + let synthetic_args = + serde_json::to_string(&json!({ "input": v4a_input })).unwrap_or_default(); + global_tool_call_cache().save( + call_id, + ToolCallEntry { + name: APPLY_PATCH_NAME.to_string(), + arguments: synthetic_args.clone(), + }, + ); + if let Some(pending) = self.tool_calls.get_mut(&openai_index) { + pending.name = APPLY_PATCH_NAME.to_string(); + // IMPORTANT-1 修复(code-reviewer pre-push):pending.args_acc 必须 + // 跟 normalize 后的 wire 形态 + ToolCallCache 同步,否则 + // `assistant_message()` 用 (name=apply_patch, args_acc=原 {"cmd":...}) + // 不一致组合写进 `global_response_session_cache`,下一 turn + // `previous_response_id` 重建走 `request.rs::build_messages_from_input` + // 时,upstream 看到 name/args 不匹配的 assistant tool_call → 严格 + // provider 直接 400,宽松 provider 误导模型记忆。 + pending.args_acc = synthetic_args; + pending.is_apply_patch = true; + pending.is_exec_command = false; + pending.apply_patch_input = Some(v4a_input); + pending.output_item_added_emitted = true; + pending.closed = true; + } + } + + /// 在 exec_command 进入普通 function_call 关闭路径之前,补 emit `output_item.added` + /// — open 阶段被 buffer 路径跳过了。namespace 字段对 exec_command 不适用 + /// (Codex CLI 内置工具,不在 namespace 包里),不加。 + fn emit_deferred_exec_command_added( + &mut self, + fc_id: &str, + call_id: &str, + name: &str, + output_index: u32, + out: &mut Vec, + ) { + emit_event( + out, + &mut self.sequence_number, + "response.output_item.added", + json!({ + "type": "response.output_item.added", + "output_index": output_index, + "item": { + "type": "function_call", + "id": fc_id, + "call_id": call_id, + "name": name, + "arguments": "", + "status": "in_progress", + }, + }), + ); + } + fn emit_reasoning_delta(&mut self, text: &str, out: &mut Vec) { if !self.reasoning_open { self.open_reasoning(out); @@ -811,6 +1348,28 @@ impl ChatToResponsesConverter { } fn tool_call_item_completed(&self, pending: &PendingToolCall) -> Value { + if pending.is_apply_patch { + // envelope.output[] 终态必须和流式 `response.output_item.done` + // 的 item 一致(见 close_tool_call apply_patch 分支),否则严格 + // 客户端会两次解读为不同 item。读 close 时缓存好的 input, + // 不重新 parse args_acc — 万一 args_acc 在 close 与 envelope + // 构造之间发生意外变化(目前看不会,但防御性写法),两侧仍一致。 + // 缓存缺失时(理论上 close 一定先于 envelope build 跑,不应触发) + // fallback 到 raw args_acc,而不是再次 parse,避免重复 emit + // 任何 telemetry。 + let input = pending + .apply_patch_input + .clone() + .unwrap_or_else(|| pending.args_acc.clone()); + return json!({ + "type": "custom_tool_call", + "id": pending.fc_id, + "call_id": pending.call_id, + "name": pending.name, + "input": input, + "status": "completed", + }); + } let mut item = json!({ "type": "function_call", "id": pending.fc_id, @@ -1054,10 +1613,16 @@ impl ChatToResponsesConverter { if self.message_open && !self.message_closed { self.close_message(out); } - // tool_calls 按 OpenAI index 顺序闭合(BTreeMap 自然有序) + // tool_calls 按 OpenAI index 顺序闭合(BTreeMap 自然有序)。 + // `interrupted` = 没有 finish_reason **且**不是因 `[DONE]` 自然结束。 + // 这是用于让 apply_patch 在 close_tool_call 里 emit + // `status="incomplete"` 而不是 `completed`,防止严格客户端在 stream + // 半截断时仍把 partial patch 当成完整 tool 调用执行 + // (apply_patch 是 destructive,partial 执行风险高)。 + let interrupted = self.finish_reason.is_none() && !from_done; let tc_indices: Vec = self.tool_calls.keys().copied().collect(); for idx in tc_indices { - self.close_tool_call(idx, out); + self.close_tool_call(idx, interrupted, out); } // finish_reason → status / incomplete_details 映射。保留现有 5 路径 @@ -1252,6 +1817,83 @@ fn emit_event(out: &mut Vec, seq: &mut u64, event_name: &str, payload: Value emit_sse_event(out, seq, event_name, payload); } +/// 从 chat function args(标准形态 `{"input": ""}`)提取裸 V4A +/// 文本,供 `custom_tool_call.input` 字段使用。 +/// +/// 降级路径分两类,通过 tracing 区分以便事后定位: +/// +/// 1. **JSON parse 失败 + 看起来像裸 V4A**(以 `*** Begin Patch` 开头): +/// debug 级,预期 happy path —— 上游 chat provider 把 freeform 工具 +/// 输出原样透传未包 JSON。 +/// 2. **JSON parse 失败 + 不像 V4A**:warn 级,通常是 stream 截断 / UTF-8 +/// 损坏 / 上游加 markdown fence。把整段透传给 Codex CLI 至少能让用户 +/// 看到 `apply_patch verification failed: ` 而不是 abort +/// 无线索。 +/// 3. **JSON valid 但缺 `input` 字段**:warn 级,通常是 schema drift +/// (模型用了 `patch` 而不是 `input` 等)。整段透传暴露真坏。 +/// +/// 借鉴上游 `codex-rs/apply-patch/apply_patch_tool_instructions.md` 的 +/// V4A 格式约束。不做 V4A 语法校验 — 留给 Codex CLI 端的 `parse_patch`。 +fn extract_apply_patch_input(args_acc: &str) -> String { + let trimmed = args_acc.trim(); + if trimmed.is_empty() { + return String::new(); + } + match serde_json::from_str::(trimmed) { + Ok(parsed) => match parsed.get("input").and_then(Value::as_str) { + Some(s) => s.to_owned(), + None => { + tracing::warn!( + target = "adapters::apply_patch", + args_preview = %args_acc.chars().take(120).collect::(), + "apply_patch args parsed as JSON but missing `input` string field; passing raw args to Codex CLI", + ); + args_acc.to_owned() + } + }, + Err(err) => { + if trimmed.starts_with("*** Begin Patch") { + tracing::debug!( + target = "adapters::apply_patch", + "apply_patch args are bare V4A (no JSON wrapper); passthrough", + ); + } else { + tracing::warn!( + target = "adapters::apply_patch", + error = %err, + args_len = args_acc.len(), + args_preview = %args_acc.chars().take(120).collect::(), + "apply_patch args failed JSON parse and don't look like bare V4A; falling back to raw passthrough — likely truncation or schema drift", + ); + } + args_acc.to_owned() + } + } +} + +/// 从 `exec_command` 工具调用的累积 args(chat completions function_call +/// arguments,标准形态 `{"cmd":"", ...}`)里抽出 `cmd` 字符串, +/// 跑 [`crate::responses::shell_to_apply_patch::detect_shell_file_write`] +/// 判断是不是 MVP scope 内的 file-write 模式。 +/// +/// 三层 fallback: +/// 1. **JSON valid + `cmd` 字段**:主路径,直接 detect。 +/// 2. **JSON valid 但缺 `cmd` 字段**:`return None`(透传 exec_command,避免 +/// 误判)。 +/// 3. **JSON 解析失败**:`return None`(args 还在 streaming / 模型乱发, +/// 透传安全)。 +fn detect_shell_file_write_in_exec_args( + args_acc: &str, +) -> Option { + let trimmed = args_acc.trim(); + if trimmed.is_empty() { + return None; + } + let parsed: Value = serde_json::from_str(trimmed).ok()?; + let cmd = parsed.get("cmd").and_then(Value::as_str)?; + crate::responses::shell_to_apply_patch::detect_shell_file_write(cmd) +} + fn drain_one_frame(buf: &mut BytesMut) -> Option { let pos = find_double_newline(buf)?; Some(buf.split_to(pos + 2).freeze()) @@ -2501,6 +3143,203 @@ data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]} assert_eq!(done.1["arguments"], "{\"a\":1}"); } + #[test] + fn apply_patch_tool_call_emits_custom_tool_call_wire_not_function_call() { + // 回归保护(issue #235):chat 上游(DeepSeek 等)用 function call 返回 + // apply_patch 时,adapter 必须把 wire 重打包成 Codex CLI 期望的 + // `custom_tool_call` 形态(上游 router 按 wire type 路由,apply_patch + // handler 硬要求 `ToolPayload::Custom { input }`)。 + // patch 文本走标准 JSON 转义:在 args.arguments 字符串里,V4A 原文的 + // `\n` 被双重转义成 `\\n`(JSON 字符串里写 `\n`)。 + let mut c = fixed(); + // 真实 chat 上游 wire 中,tool_call.arguments 是 JSON 字符串字面值, + // patch 里的换行必须双重转义(SSE outer JSON 的 string value 里写 + // `\\n`,解码后是 `\n` 字面;`arguments` 值再被 client 当 JSON 解一次 + // 得到 `*** Begin Patch\n...` 真换行的 V4A patch)。 + let chunks = concat!( + r#"data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_ap","type":"function","function":{"name":"apply_patch","arguments":"{\"input\":\"*** Begin Patch\\n*** Update File: foo.py\\n@@\\n-old\\n+new\\n*** End Patch\\n\"}"}}]}}]}"#, + "\n\n", + r#"data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}"#, + "\n\n", + "data: [DONE]\n\n", + ); + let out = c.feed(chunks.as_bytes()); + let events = parse_emitted(&out); + let kinds = names(&events); + // open 必须用 custom_tool_call 而不是 function_call + let added = events + .iter() + .find(|(n, _)| n == "response.output_item.added") + .expect("应当 emit output_item.added"); + assert_eq!( + added.1["item"]["type"], "custom_tool_call", + "apply_patch wire 必须是 custom_tool_call,实际 events: {kinds:?}" + ); + assert_eq!(added.1["item"]["name"], "apply_patch"); + // 中间不应有 function_call_arguments.delta(apply_patch 路径 close 时 + // 一次性 emit custom_tool_call_input.delta) + assert!( + !kinds.contains(&"response.function_call_arguments.delta"), + "apply_patch 路径不应 emit function_call_arguments.delta,events: {kinds:?}" + ); + // close 必须 emit custom_tool_call_input.delta + .done + let input_delta = events + .iter() + .find(|(n, _)| n == "response.custom_tool_call_input.delta") + .expect("应当 emit custom_tool_call_input.delta"); + let expected_v4a = + "*** Begin Patch\n*** Update File: foo.py\n@@\n-old\n+new\n*** End Patch\n"; + assert_eq!(input_delta.1["delta"], expected_v4a); + assert_eq!(input_delta.1["call_id"], "call_ap"); + // envelope.output[] 终态也必须是 custom_tool_call + let completed = events + .iter() + .rev() + .find(|(n, _)| n == "response.completed") + .unwrap(); + let output = &completed.1["response"]["output"][0]; + assert_eq!(output["type"], "custom_tool_call"); + assert_eq!(output["input"], expected_v4a); + assert_eq!(output["call_id"], "call_ap"); + } + + #[test] + fn apply_patch_falls_back_to_raw_args_when_not_json() { + // 模型直接吐裸 V4A 而不包 JSON(某些 chat 上游可能这样转译 freeform)。 + // adapter 必须把整段 args_acc 当 input 而不是空字符串,让 Codex CLI + // 至少能看到 patch 内容并尝试解析。 + let raw_v4a = "*** Begin Patch\n*** Add File: a.md\n+hi\n*** End Patch\n"; + // serde_json::to_string 自动产生合法 JSON 字符串 escape(`\n` → `\\n` + // 字面、引号转义、反斜杠转义),比手工 replace 链可靠且贴近真实 wire。 + let args_json_string = serde_json::to_string(raw_v4a).unwrap(); + let mut c = fixed(); + let frame = format!( + "data: {{\"choices\":[{{\"index\":0,\"delta\":{{\"tool_calls\":[{{\"index\":0,\"id\":\"call_ap\",\"type\":\"function\",\"function\":{{\"name\":\"apply_patch\",\"arguments\":{args_json_string}}}}}]}}}}]}}\n\ndata: {{\"choices\":[{{\"delta\":{{}},\"finish_reason\":\"tool_calls\"}}]}}\n\ndata: [DONE]\n\n", + args_json_string = args_json_string, + ); + let out = c.feed(frame.as_bytes()); + let events = parse_emitted(&out); + let delta = events + .iter() + .find(|(n, _)| n == "response.custom_tool_call_input.delta") + .expect("custom_tool_call_input.delta 应当 emit"); + assert_eq!( + delta.1["delta"], raw_v4a, + "非 JSON args 应整段当 input(裸 V4A 兜底)" + ); + } + + #[test] + fn apply_patch_interrupted_stream_emits_incomplete_status_skips_input_done() { + // 回归保护:apply_patch 是 destructive 工具,stream 中途断开 → close + // 必须 emit `status="incomplete"` 且 skip `custom_tool_call_input.done`, + // 让 Codex CLI 看到不完整状态而不是执行 partial patch。 + let partial_v4a = "*** Begin Patch\n*** Update File: foo.py\n@@\n-old\n"; // 截断在 @@ 之后 + let inner = serde_json::to_string(&json!({ "input": partial_v4a })).unwrap(); + let args_json_string = serde_json::to_string(&inner).unwrap(); + let mut c = fixed(); + // 仅 emit tool_call 增量与 lifecycle 开头,不 emit finish_reason / [DONE], + // 模拟 upstream EOF 中断。 + let frame = format!( + "data: {{\"choices\":[{{\"index\":0,\"delta\":{{\"tool_calls\":[{{\"index\":0,\"id\":\"call_ap\",\"type\":\"function\",\"function\":{{\"name\":\"apply_patch\",\"arguments\":{args_json_string}}}}}]}}}}]}}\n\n", + args_json_string = args_json_string, + ); + let _ = c.feed(frame.as_bytes()); + let out = c.finish(); + let events = parse_emitted(&out); + let kinds = names(&events); + // 必须 NOT 出现 .delta 或 .done 的 custom_tool_call_input(防止 client + // 在 .done 时触发执行 partial patch) + assert!( + !kinds.contains(&"response.custom_tool_call_input.done"), + "interrupted 时禁止 emit custom_tool_call_input.done,events: {kinds:?}" + ); + assert!( + !kinds.contains(&"response.custom_tool_call_input.delta"), + "interrupted 时禁止 emit custom_tool_call_input.delta(避免提前触发执行),events: {kinds:?}" + ); + // output_item.done item 必须含 status=incomplete + let done = events + .iter() + .find(|(n, _)| n == "response.output_item.done") + .expect("interrupted 仍应 emit output_item.done"); + assert_eq!(done.1["item"]["type"], "custom_tool_call"); + assert_eq!( + done.1["item"]["status"], "incomplete", + "interrupted apply_patch 必须 status=incomplete" + ); + // envelope 也是 incomplete + interrupted + let completed = events + .iter() + .rev() + .find(|(n, _)| n == "response.completed") + .unwrap(); + assert_eq!(completed.1["response"]["status"], "incomplete"); + assert_eq!( + completed.1["response"]["incomplete_details"]["reason"], + "interrupted" + ); + } + + #[test] + fn apply_patch_streaming_input_matches_envelope_output() { + // 防御性回归:`response.output_item.done` 的 `item.input` 必须跟 + // `response.completed.output[].input` 完全一致,避免两次 emit 路径 + // 在未来重构时 drift。 + let patch = "*** Begin Patch\n*** Update File: x.txt\n@@\n-a\n+b\n*** End Patch\n"; + // chat wire 里 `arguments` 是 JSON-string(双重编码):先把 V4A 包成 + // `{"input": ""}` JSON 文本,再 JSON-quote 一次作为字符串值。 + let inner = serde_json::to_string(&json!({ "input": patch })).unwrap(); + let args_json_string = serde_json::to_string(&inner).unwrap(); + let mut c = fixed(); + let frame = format!( + "data: {{\"choices\":[{{\"index\":0,\"delta\":{{\"tool_calls\":[{{\"index\":0,\"id\":\"call_match\",\"type\":\"function\",\"function\":{{\"name\":\"apply_patch\",\"arguments\":{args_json_string}}}}}]}}}}]}}\n\ndata: {{\"choices\":[{{\"delta\":{{}},\"finish_reason\":\"tool_calls\"}}]}}\n\ndata: [DONE]\n\n", + args_json_string = args_json_string, + ); + let out = c.feed(frame.as_bytes()); + let events = parse_emitted(&out); + let done = events + .iter() + .find(|(n, v)| { + n == "response.output_item.done" && v["item"]["type"] == "custom_tool_call" + }) + .expect("应当有 custom_tool_call output_item.done"); + let streamed_input = done.1["item"]["input"].as_str().unwrap().to_owned(); + let completed = events + .iter() + .rev() + .find(|(n, _)| n == "response.completed") + .unwrap(); + let envelope_input = completed.1["response"]["output"][0]["input"] + .as_str() + .unwrap() + .to_owned(); + assert_eq!( + streamed_input, envelope_input, + "streamed output_item.done.input 必须跟 envelope.output[].input 完全一致" + ); + assert_eq!(streamed_input, patch); + } + + #[test] + fn extract_apply_patch_input_extracts_or_falls_back() { + // happy path:`{input: string}` 提出 string 字段 + assert_eq!( + extract_apply_patch_input(r#"{"input":"*** Begin Patch\nfoo"}"#), + "*** Begin Patch\nfoo" + ); + // 非 JSON:整段透传 + let raw = "*** Begin Patch\nfoo\n*** End Patch\n"; + assert_eq!(extract_apply_patch_input(raw), raw); + // JSON 但无 input 字段:整段透传 + assert_eq!( + extract_apply_patch_input(r#"{"other":"x"}"#), + r#"{"other":"x"}"# + ); + // 空字符串:返回空 + assert_eq!(extract_apply_patch_input(""), ""); + } + #[test] fn message_then_tool_call_keeps_output_index_order() { let mut c = fixed(); @@ -2812,4 +3651,343 @@ data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"},{"index":1,"delt assert_eq!(usage["output_tokens"], 6); assert_eq!(usage["total_tokens"], 10); } + + // ===== server-side normalize: exec_command → apply_patch ===== + // 关键回归保护(issue #235 round 2 真机数据):chat-completions provider + // (Kimi 实测 t0024/t0027/t0030/t0037)调 exec_command 跑 `cat <` + // `printf > file` `echo > file` 写文件,Codex Desktop 看不到 apply_patch + // wire → 不渲染 diff UI(4+2 元素全失效)。adapter 在 server-side normalize + // 这三种 shell file-write 为 custom_tool_call apply_patch SSE,让 Codex + // Desktop V4A applier 渲染 diff card / 行号 / +N -M / 文件清单。 + + fn exec_command_sse_chunk(cmd_value: &str) -> Vec { + // 构造单 chunk 含完整 exec_command tool_call:首帧 name + 完整 args + // (chat 上游一般首帧带 name,中间帧追加 args delta;我们 buffer + // 整段 args 后才决策,所以单 chunk vs 多 chunk 等价测) + // + // chat completions schema: `function.arguments` 是个**JSON 字符串**, + // 它的 value 又是个 JSON object 字面('{"cmd":""}')。因此需要 + // **双重 JSON encode**:先把 `{"cmd": cmd_value}` 序列化成 JSON 字符串, + // 再把这个字符串当 string value 序列化进 SSE chunk。 + let args_obj = serde_json::to_string(&serde_json::json!({"cmd": cmd_value})).unwrap(); + let escaped_args = serde_json::to_string(&args_obj).unwrap(); + format!( + r#"data: {{"choices":[{{"index":0,"delta":{{"tool_calls":[{{"index":0,"id":"call_test_1","function":{{"name":"exec_command","arguments":{escaped_args}}}}}]}},"finish_reason":"tool_calls"}}]}} + +"# + ) + .into_bytes() + } + + fn collect_apply_patch_inputs(events: &[(String, Value)]) -> Vec { + events + .iter() + .filter(|(n, _)| n == "response.custom_tool_call_input.done") + .filter_map(|(_, p)| p["input"].as_str().map(str::to_owned)) + .collect() + } + + fn find_function_call_done_with_name<'a>( + events: &'a [(String, Value)], + name: &str, + ) -> Option<&'a Value> { + events.iter().find_map(|(n, p)| { + if n == "response.output_item.done" + && p["item"].get("type").and_then(|v| v.as_str()) == Some("function_call") + && p["item"].get("name").and_then(|v| v.as_str()) == Some(name) + { + Some(p) + } else { + None + } + }) + } + + #[test] + fn exec_command_here_doc_rewritten_to_apply_patch_custom_tool_call() { + let mut c = fixed(); + let mut all = Vec::new(); + // 真机 round 2 turn 0024 Kimi 同款 `cat <<'PYEOF' > /tmp/...py\n...\nPYEOF` + let cmd = "cat <<'PYEOF' > /tmp/apply-patch-test-A/app-1.py\ndef main():\n pass\n\n\nif __name__ == \"__main__\":\n main()\nPYEOF"; + all.extend(c.feed(&exec_command_sse_chunk(cmd))); + all.extend(c.feed(b"data: [DONE]\n\n")); + let events = parse_emitted(&all); + + // emit 含完整 custom_tool_call apply_patch 序列 + let added = events + .iter() + .find(|(n, p)| { + n == "response.output_item.added" + && p["item"].get("type").and_then(|v| v.as_str()) == Some("custom_tool_call") + }) + .expect("custom_tool_call output_item.added 必须 emit"); + assert_eq!(added.1["item"]["name"], "apply_patch"); + assert_eq!( + added.1["item"]["call_id"], "call_test_1", + "call_id 必须复用模型原 exec_command call_id" + ); + + // V4A input.done 含 Add File header + 行 `+` 前缀 + let inputs = collect_apply_patch_inputs(&events); + assert_eq!(inputs.len(), 1, "应只有 1 个 apply_patch input.done"); + let v4a = &inputs[0]; + assert!(v4a.starts_with("*** Begin Patch\n")); + assert!(v4a.contains("*** Add File: /tmp/apply-patch-test-A/app-1.py")); + assert!(v4a.contains("+def main():")); + assert!(v4a.contains("+ pass")); + assert!(v4a.ends_with("*** End Patch\n")); + + // 不应再 emit 原 exec_command function_call 序列 + assert!( + find_function_call_done_with_name(&events, "exec_command").is_none(), + "exec_command 已被 normalize,不应再有 function_call.done with name=exec_command" + ); + + // response.completed.output 数组里的 final item 应为 custom_tool_call apply_patch + let completed = events + .iter() + .find(|(n, _)| n == "response.completed") + .expect("response.completed 必须 emit"); + let final_output = completed.1["response"]["output"].as_array().unwrap(); + let has_apply_patch = final_output.iter().any(|item| { + item.get("type").and_then(|v| v.as_str()) == Some("custom_tool_call") + && item.get("name").and_then(|v| v.as_str()) == Some("apply_patch") + }); + assert!( + has_apply_patch, + "envelope final output 必须含 apply_patch item" + ); + } + + #[test] + fn exec_command_printf_with_escapes_rewritten_to_apply_patch() { + let mut c = fixed(); + let mut all = Vec::new(); + // 真机 round 2 turn 0027/0030 Kimi 同款 `printf '...\n' > file` + let cmd = r#"printf '# 测试笔记\n\n今天天气不错。\n' > /tmp/apply-patch-test-B/note.md"#; + all.extend(c.feed(&exec_command_sse_chunk(cmd))); + all.extend(c.feed(b"data: [DONE]\n\n")); + let events = parse_emitted(&all); + + let inputs = collect_apply_patch_inputs(&events); + assert_eq!(inputs.len(), 1); + let v4a = &inputs[0]; + assert!(v4a.contains("*** Add File: /tmp/apply-patch-test-B/note.md")); + // \n 转义应展开成真实 newline,每个非空行 `+` 前缀 + assert!(v4a.contains("+# 测试笔记")); + assert!(v4a.contains("+今天天气不错。")); + } + + #[test] + fn exec_command_echo_simple_rewritten_to_apply_patch() { + let mut c = fixed(); + let mut all = Vec::new(); + let cmd = "echo 'hello world' > /tmp/x.txt"; + all.extend(c.feed(&exec_command_sse_chunk(cmd))); + all.extend(c.feed(b"data: [DONE]\n\n")); + let events = parse_emitted(&all); + + let inputs = collect_apply_patch_inputs(&events); + assert_eq!(inputs.len(), 1); + assert!(inputs[0].contains("*** Add File: /tmp/x.txt")); + assert!(inputs[0].contains("+hello world")); + } + + #[test] + fn exec_command_non_file_write_passes_through_as_function_call() { + let mut c = fixed(); + let mut all = Vec::new(); + // `ls -la /tmp` 不是 file-write 模式 → 走 function_call 透传 + let cmd = "ls -la /tmp"; + all.extend(c.feed(&exec_command_sse_chunk(cmd))); + all.extend(c.feed(b"data: [DONE]\n\n")); + let events = parse_emitted(&all); + + // 应 emit 完整 function_call exec_command 序列(added/delta/done/output_item.done) + let added = events + .iter() + .find(|(n, p)| { + n == "response.output_item.added" + && p["item"].get("name").and_then(|v| v.as_str()) == Some("exec_command") + }) + .expect("exec_command output_item.added 必须 emit(buffer 期延迟后补)"); + assert_eq!(added.1["item"]["type"], "function_call"); + + // function_call_arguments.delta 应含完整 args(buffer 期没 emit,close 时一次补) + let args_delta = events + .iter() + .find(|(n, p)| { + n == "response.function_call_arguments.delta" + && p["delta"] + .as_str() + .map(|s| s.contains("ls -la")) + .unwrap_or(false) + }) + .expect("deferred function_call_arguments.delta 必须含完整 args"); + let _ = args_delta; + + let done = find_function_call_done_with_name(&events, "exec_command") + .expect("function_call.done exec_command 必须 emit"); + assert_eq!(done["item"]["status"], "completed"); + + // 不应有 custom_tool_call apply_patch + let has_apply_patch = events.iter().any(|(n, p)| { + n == "response.output_item.added" + && p["item"].get("name").and_then(|v| v.as_str()) == Some("apply_patch") + }); + assert!( + !has_apply_patch, + "非 file-write 不应被 normalize 成 apply_patch" + ); + } + + #[test] + fn exec_command_append_redirect_passes_through() { + let mut c = fixed(); + let mut all = Vec::new(); + // `>>` append 模式被 detector reject,透传 function_call + let cmd = "echo 'log line' >> /tmp/app.log"; + all.extend(c.feed(&exec_command_sse_chunk(cmd))); + all.extend(c.feed(b"data: [DONE]\n\n")); + let events = parse_emitted(&all); + + assert!( + find_function_call_done_with_name(&events, "exec_command").is_some(), + "append redirect 透传为 function_call exec_command" + ); + assert!( + collect_apply_patch_inputs(&events).is_empty(), + "append redirect 不应被 normalize" + ); + } + + #[test] + fn exec_command_multi_command_passes_through() { + let mut c = fixed(); + let mut all = Vec::new(); + // multi-cmd `&&` reject + let cmd = "mkdir -p /tmp/d && echo 'x' > /tmp/d/a.txt"; + all.extend(c.feed(&exec_command_sse_chunk(cmd))); + all.extend(c.feed(b"data: [DONE]\n\n")); + let events = parse_emitted(&all); + assert!(find_function_call_done_with_name(&events, "exec_command").is_some()); + assert!(collect_apply_patch_inputs(&events).is_empty()); + } + + #[test] + fn exec_command_var_substitution_passes_through() { + let mut c = fixed(); + let mut all = Vec::new(); + let cmd = r#"echo "$(date)" > /tmp/timestamp.txt"#; + all.extend(c.feed(&exec_command_sse_chunk(cmd))); + all.extend(c.feed(b"data: [DONE]\n\n")); + let events = parse_emitted(&all); + assert!(find_function_call_done_with_name(&events, "exec_command").is_some()); + assert!(collect_apply_patch_inputs(&events).is_empty()); + } + + #[test] + fn normalized_apply_patch_envelope_assistant_message_uses_synthetic_args() { + // IMPORTANT-1 修复(code-reviewer):normalize 后 pending.args_acc 必须 + // 跟 wire 形态 + ToolCallCache 同步,否则 `assistant_message()` 用 + // (name=apply_patch, args_acc=原 {"cmd":...}) 写进 response_session_cache, + // 下一 turn previous_response_id 重建给上游 name/args mismatch 数据。 + let mut c = fixed(); + let mut all = Vec::new(); + let cmd = "echo 'hello' > /tmp/x.txt"; + all.extend(c.feed(&exec_command_sse_chunk(cmd))); + all.extend(c.feed(b"data: [DONE]\n\n")); + let events = parse_emitted(&all); + + // envelope.output[] 含 custom_tool_call apply_patch + input = V4A + let completed = events + .iter() + .find(|(n, _)| n == "response.completed") + .expect("response.completed"); + let output = completed.1["response"]["output"].as_array().unwrap(); + let apply_patch_item = output + .iter() + .find(|item| { + item.get("type").and_then(|v| v.as_str()) == Some("custom_tool_call") + && item.get("name").and_then(|v| v.as_str()) == Some("apply_patch") + }) + .expect("envelope.output 含 apply_patch"); + let input = apply_patch_item["input"].as_str().expect("input field"); + assert!(input.contains("*** Add File: /tmp/x.txt")); + assert!(input.contains("+hello")); + } + + #[test] + fn exec_command_interrupted_emits_incomplete_not_completed() { + // IMPORTANT-3 修复(code-reviewer):exec_command 在 partial args 上被 + // 中断时(stream 没收到 finish_reason / [DONE]),必须 emit + // `status="incomplete"`,防止 Codex Desktop 跑半截 shell(truncated + // here-doc 等 destructive 行为)。原默认 function_call 关闭路径 emit + // `status="completed"`,对 interrupted exec_command 必须特判。 + let mut c = fixed(); + let mut all = Vec::new(); + // 不带 finish_reason,模拟 stream 中断 + all.extend(c.feed( + br#"data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_partial","function":{"name":"exec_command","arguments":"{\"cmd\": \"cat < /tmp/x.py\nincomplete"}}]}}]} + +"#, + )); + // **不**发 [DONE],直接 finish() 触发 interrupted + let final_out = c.finish(); + all.extend(final_out); + + let events = parse_emitted(&all); + + // exec_command output_item.done 必须 status=incomplete + let done = events + .iter() + .find(|(n, p)| { + n == "response.output_item.done" + && p["item"].get("name").and_then(|v| v.as_str()) == Some("exec_command") + }) + .expect("interrupted exec_command output_item.done 必须 emit"); + assert_eq!( + done.1["item"]["status"], "incomplete", + "interrupted exec_command 必须 emit status=incomplete 防 Codex Desktop 跑半截 shell" + ); + } + + #[test] + fn normalized_apply_patch_call_id_reused_for_tool_output_correlation() { + // 关键:下一 turn Codex Desktop 把 apply_patch 执行结果通过同 call_id + // 回灌给模型,模型按 call_id 关联到自己原 exec_command 的 tool_call。 + // 必须 call_id 100% 复用,否则模型看到 orphan tool result。 + let mut c = fixed(); + let mut all = Vec::new(); + let cmd = "echo 'x' > /tmp/a.txt"; + all.extend(c.feed(&exec_command_sse_chunk(cmd))); + all.extend(c.feed(b"data: [DONE]\n\n")); + let events = parse_emitted(&all); + + let added = events + .iter() + .find(|(n, p)| { + n == "response.output_item.added" + && p["item"].get("name").and_then(|v| v.as_str()) == Some("apply_patch") + }) + .unwrap(); + assert_eq!( + added.1["item"]["call_id"], "call_test_1", + "normalize 后 call_id 必须复用模型原 exec_command call_id" + ); + + let done_item_call_id = events + .iter() + .find_map(|(n, p)| { + if n == "response.output_item.done" + && p["item"].get("name").and_then(|v| v.as_str()) == Some("apply_patch") + { + p["item"]["call_id"].as_str().map(str::to_owned) + } else { + None + } + }) + .unwrap(); + assert_eq!(done_item_call_id, "call_test_1"); + } } diff --git a/crates/adapters/src/responses/mod.rs b/crates/adapters/src/responses/mod.rs index 2b9b9fc..f37ff81 100644 --- a/crates/adapters/src/responses/mod.rs +++ b/crates/adapters/src/responses/mod.rs @@ -13,6 +13,7 @@ pub mod compact; pub mod converter; pub mod request; pub mod session; +pub mod shell_to_apply_patch; pub mod stream; pub mod tool_call_cache; diff --git a/crates/adapters/src/responses/request.rs b/crates/adapters/src/responses/request.rs index 621e4cf..c008254 100644 --- a/crates/adapters/src/responses/request.rs +++ b/crates/adapters/src/responses/request.rs @@ -296,6 +296,14 @@ fn build_messages_from_input( messages.push(msg); } + // 紧跟 Codex CLI 自带 instructions 之后注入 apply_patch chat-path 指引 + // (仅当本 turn 真正注册了 apply_patch 工具时)。位置选择:Codex 系统 + // 指令之后,user input 之前 — 既不污染 Codex 原指令,又确保模型在 + // 读完工具列表准备调 apply_patch 时已经见过 chat-path 限制。 + if tools_register_apply_patch(body) { + messages.push(apply_patch_chat_guidance_message()); + } + let current_messages = body .get("input") .map(input_field_to_messages) @@ -518,6 +526,70 @@ fn input_item_to_messages(item: &serde_json::Map) -> Vec { "content": output_str, })] } + "custom_tool_call" => { + // Codex CLI 把 freeform apply_patch 的回放 wire 包成 + // `ResponseItem::CustomToolCall { name, input, call_id, ... }` + // (`codex-rs/protocol/src/models.rs:824-832`)。我们在 turn N 通过 + // `converter.rs::close_tool_call` apply_patch 分支 emit 了它; + // Codex CLI 在 turn N+1 把同一 item 通过 `input[]` 回放给我们。 + // 转下游 chat completions 时必须重新打包成 `assistant.tool_calls` + // 的 `type:"function"` 形态(chat 端不认 custom_tool_call),且 + // `function.arguments` 必须是 JSON 字符串 `{"input":""}` + // (与首轮在 `tools.rs::convert_responses_tool_to_chat_tool` 的 + // `"custom" =>` 分支 lowering 形态保持一致)—— 模型才不会因 + // wire 形态变化失忆。 + let call_id = item + .get("call_id") + .and_then(|v| v.as_str()) + .or_else(|| item.get("id").and_then(|v| v.as_str())) + .unwrap_or("") + .to_owned(); + let name = item.get("name").and_then(|v| v.as_str()).unwrap_or(""); + let input_text = item.get("input").and_then(|v| v.as_str()).unwrap_or(""); + // arguments 必须是 chat function-call 的标准 JSON 字符串形态。 + // serde_json::to_string 自动处理换行 / 引号 / 反斜杠等所有转义。 + let arguments_json = serde_json::to_string(&json!({ "input": input_text })) + .unwrap_or_else(|_| { + // to_string 在 input 是 valid UTF-8 string 时不会失败;若 + // 真发生,fallback 到空对象保持下游 chat schema 合法。 + "{}".to_owned() + }); + let arguments = sanitize_tool_arguments_json_string(&arguments_json); + vec![json!({ + "role": "assistant", + "content": "", + "tool_calls": [{ + "id": if call_id.is_empty() { "call_unknown".to_owned() } else { call_id }, + "type": "function", + "function": { "name": name, "arguments": arguments }, + }], + })] + } + "custom_tool_call_output" => { + // `ResponseItem::CustomToolCallOutput { call_id, output, ... }` + // (`codex-rs/protocol/src/models.rs:839-847`)使用与 function_call_output + // 相同的 `output` payload encoding(string 或 content_items array)。 + // 转 chat 时只需把 wire item type 对齐到普通 `role:"tool"` message, + // tool_call_id 来源仍按 call_id / tool_call_id / id 三级兜底。 + let call_id = item + .get("call_id") + .and_then(|v| v.as_str()) + .or_else(|| item.get("tool_call_id").and_then(|v| v.as_str())) + .or_else(|| item.get("id").and_then(|v| v.as_str())) + .unwrap_or("") + .to_owned(); + let output_value = item + .get("output") + .cloned() + .unwrap_or(Value::String(String::new())); + let output_str = + normalize_tool_output_for_context(Some(call_id.as_str()), output_value); + vec![json!({ + "role": "tool", + "tool_call_id": call_id, + "content": output_str, + })] + } "input_image" => { let image_url = item .get("image_url") @@ -2207,4 +2279,48 @@ mod tests; use tools::{ contains_kimi_web_search_tool, convert_responses_tool_to_chat_tool, normalize_tool_choice, + APPLY_PATCH_TOOL_NAME, }; + +/// chat-path 实战指引,作为独立 `role:"system"` 注入,仅在该 turn 的 tools +/// 数组里注册了 `apply_patch` 时启用。理由参见 issue #235 真机稳定性测试 +/// (DeepSeek 跑 10 个 Level 共发现的 4 个 chat-path 行为):tool/参数 +/// description 同时含紧凑版作 fallback,但 system message 在多数 chat +/// 上游里被赋予更高权重,且模型在 system 块里读到的指引更难被遗忘 / 截断。 +const APPLY_PATCH_CHAT_PATH_SYSTEM_GUIDANCE: &str = concat!( + "[apply_patch chat-path guidance — injected by codex-app-transfer adapter because the upstream lark grammar constraint is unavailable on chat function-call providers]\n", + "When you call the `apply_patch` tool, follow these rules empirically observed with non-OpenAI chat providers:\n", + "\n", + "1. Use an EMPTY LINE as the `@@` anchor whenever possible. Non-empty anchors (e.g. `@@ Hello World!`) frequently fail to match on this path. ", + "If the target file lacks a blank line near your hunk, first run `printf '\\n' >> ` via shell to seed one, then use `@@` with empty content as the anchor, and clean up extra blank lines after the patch lands.\n", + "\n", + "2. Do NOT combine `*** Add File: ` and `*** Update File: ` for the same path in a single patch. ", + "The Update step reads the file before the Add step lands on disk, so it sees an empty file and fails. Either: (a) make `*** Add File:` write the final content in one shot, or (b) split into two separate `apply_patch` invocations.\n", + "\n", + "3. `*** Update File:` cannot operate on a totally empty file. If the target is empty, first use shell (e.g. `printf '\\n' > `) to write at least one line, then call `apply_patch`.\n", + "\n", + "4. In a multi-line file, lone `+` lines following an `@@` anchor APPEND below the anchor — they do NOT replace the anchor line. To change an existing line, you must include BOTH a `-` line to remove the old content AND a `+` line to add the new content. Do not omit the `-` line.\n", + "\n", + "Following these rules avoids retry storms and improves the success rate on first attempt." +); + +/// 检测 Responses request body 的 tools 数组是否注册了 `apply_patch` 工具。 +/// `apply_patch` 在 Responses 协议里以 `type:"custom", name:"apply_patch"` 出现, +/// 在被 [`convert_responses_tool_to_chat_tool`] 降级前。 +/// 用于决定本 turn 是否注入 [`APPLY_PATCH_CHAT_PATH_SYSTEM_GUIDANCE`]。 +fn tools_register_apply_patch(body: &Value) -> bool { + let Some(tools) = body.get("tools").and_then(Value::as_array) else { + return false; + }; + tools.iter().any(|t| { + t.get("name").and_then(Value::as_str) == Some(APPLY_PATCH_TOOL_NAME) + && t.get("type").and_then(Value::as_str) == Some("custom") + }) +} + +fn apply_patch_chat_guidance_message() -> Value { + json!({ + "role": "system", + "content": APPLY_PATCH_CHAT_PATH_SYSTEM_GUIDANCE, + }) +} diff --git a/crates/adapters/src/responses/request/tests.rs b/crates/adapters/src/responses/request/tests.rs index 0bb2d70..43f658e 100644 --- a/crates/adapters/src/responses/request/tests.rs +++ b/crates/adapters/src/responses/request/tests.rs @@ -1952,6 +1952,176 @@ fn function_call_output_becomes_tool_message_with_placeholder_assistant() { assert_eq!(messages[1]["content"], "sunny"); } +#[test] +fn apply_patch_chat_path_guidance_injected_when_tool_registered() { + // 真机稳定性测试发现:即使 wire 桥接通了 + tool description 有 V4A + // 规则,DeepSeek 在 chat-path 上仍会反复尝试错误的 anchor / Add+Update + // 组合 / 空文件 Update 等无效路径,平均每次任务摸索 1-3 分钟。为节省 + // tokens 和提升首次成功率,adapter 在 tools 数组里注册了 apply_patch + // 的 turn 注入一段独立 system message 告知 chat-path 实战 workaround。 + let out = convert(json!({ + "input": [{"type": "message", "role": "user", "content": "edit foo.py"}], + "instructions": "You are a coding assistant.", + "tools": [{ + "type": "custom", + "name": "apply_patch", + "description": "Use the `apply_patch` tool to edit files." + }] + })); + let messages = out["messages"].as_array().unwrap(); + + // Codex CLI 原 instructions 必须保留在第一条 + assert_eq!(messages[0]["role"], "system"); + assert!( + messages[0]["content"] + .as_str() + .unwrap_or_default() + .contains("coding assistant"), + "Codex 原 instructions 不应被覆盖" + ); + + // 紧跟在 Codex instructions 之后必须有一条 adapter-injected guidance + assert_eq!(messages[1]["role"], "system"); + let guidance = messages[1]["content"].as_str().unwrap_or_default(); + assert!( + guidance.contains("apply_patch chat-path guidance"), + "注入的指引必须带可识别 marker:{guidance}" + ); + // 4 个核心 workaround 都要含进去 + assert!(guidance.contains("empty line") || guidance.contains("EMPTY LINE")); + assert!(guidance.contains("Add File") && guidance.contains("Update File")); + assert!(guidance.contains("empty file") || guidance.contains("totally empty")); + assert!(guidance.contains("APPEND") || guidance.contains("append")); +} + +#[test] +fn apply_patch_chat_path_guidance_skipped_when_tool_not_registered() { + // 非 apply_patch 任务不应注入指引,避免污染 token / 模型注意力 + let out = convert(json!({ + "input": [{"type": "message", "role": "user", "content": "list files"}], + "instructions": "You are a coding assistant.", + "tools": [{ + "type": "function", + "name": "shell_command", + "description": "Run a shell command", + "parameters": {"type": "object", "properties": {}} + }] + })); + let messages = out["messages"].as_array().unwrap(); + let has_guidance = messages.iter().any(|m| { + m["content"] + .as_str() + .unwrap_or_default() + .contains("apply_patch chat-path guidance") + }); + assert!( + !has_guidance, + "无 apply_patch 注册时不应注入 chat-path guidance" + ); +} + +#[test] +fn apply_patch_chat_path_guidance_idempotent_across_turns() { + // 防止 merge_consecutive_system_messages 把 adapter-injected guidance + // 跟 Codex instructions 拼到一起后,反复 convert 时被重复累积(连发 3 个 + // turn,每 turn 转换出的 messages 里仍只含 1 段 guidance)。 + let one_turn = json!({ + "input": [{"type": "message", "role": "user", "content": "edit"}], + "instructions": "You are helpful.", + "tools": [{ + "type": "custom", + "name": "apply_patch", + "description": "edit" + }] + }); + for _ in 0..3 { + let out = convert(one_turn.clone()); + let guidance_count = out["messages"] + .as_array() + .unwrap() + .iter() + .map(|m| m["content"].as_str().unwrap_or_default()) + .filter(|c| c.contains("apply_patch chat-path guidance")) + .count(); + assert_eq!(guidance_count, 1, "每次 convert 仅注入一次 guidance"); + } +} + +#[test] +fn custom_tool_call_input_item_lowered_to_assistant_tool_calls() { + // 回归保护(issue #235):turn N+1 Codex CLI 回放上一轮的 + // `ResponseItem::CustomToolCall { name, input, call_id }`,我们必须把它 + // 转成 chat completions 的 `assistant.tool_calls` 形态(function-call), + // 否则模型完全看不到上一轮 apply_patch 调用 → 多轮上下文丢失。 + // arguments 必须是 JSON 字符串 `{"input":""}`,与首轮在请求侧 + // lowering 的形态保持一致,模型才不失忆。 + let patch_text = "*** Begin Patch\n*** Update File: a.py\n@@\n-x\n+y\n*** End Patch\n"; + let out = convert(json!({ + "input": [{ + "type": "custom_tool_call", + "id": "ctc_1", + "call_id": "call_ap_1", + "name": "apply_patch", + "input": patch_text, + "status": "completed", + }] + })); + let messages = out["messages"].as_array().unwrap(); + let assistant = messages + .iter() + .find(|m| m["role"] == "assistant" && m["tool_calls"].is_array()) + .expect("custom_tool_call 应当映射成 assistant.tool_calls"); + let tc = &assistant["tool_calls"][0]; + assert_eq!(tc["type"], "function"); + assert_eq!(tc["id"], "call_ap_1"); + assert_eq!(tc["function"]["name"], "apply_patch"); + // arguments 是 JSON 字符串值。serde_json 解一次得到 {input: }, + // 再 V4A 的换行已被正常 JSON-escape(`\n` 字面值)。 + let args_str = tc["function"]["arguments"].as_str().unwrap(); + let parsed: serde_json::Value = + serde_json::from_str(args_str).expect("arguments 必须是合法 JSON"); + assert_eq!(parsed["input"], patch_text); +} + +#[test] +fn custom_tool_call_output_input_item_lowered_to_role_tool() { + // 回归保护(issue #235):`ResponseItem::CustomToolCallOutput { call_id, output }` + // 回放时必须转成 chat 端的 `role:"tool"` message,tool_call_id 跟前面的 + // assistant.tool_calls.id 配对,否则 chat 上游会因 orphan tool message 400。 + let out = convert(json!({ + "input": [ + { + "type": "custom_tool_call", + "call_id": "call_ap_2", + "name": "apply_patch", + "input": "*** Begin Patch\n*** Add File: b.md\n+hi\n*** End Patch\n", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_ap_2", + "output": "Patch applied successfully", + } + ] + })); + let messages = out["messages"].as_array().unwrap(); + let tool_msg = messages + .iter() + .find(|m| m["role"] == "tool") + .expect("custom_tool_call_output 应当映射成 role:tool"); + assert_eq!(tool_msg["tool_call_id"], "call_ap_2"); + assert_eq!(tool_msg["content"], "Patch applied successfully"); + // 同 PR 还要保证 assistant 在 tool 前(orphan repair 不会插占位) + let assistant_idx = messages + .iter() + .position(|m| m["role"] == "assistant") + .unwrap(); + let tool_idx = messages.iter().position(|m| m["role"] == "tool").unwrap(); + assert!( + assistant_idx < tool_idx, + "assistant.tool_calls 必须在 role:tool 之前出现" + ); +} + #[test] fn function_call_output_non_string_is_json_serialized() { // 走完整 convert 路径(global cache 在生产里就这条路); @@ -2644,6 +2814,87 @@ fn tools_custom_type_is_lowered_to_function_with_input() { "string" ); assert_eq!(tool["function"]["parameters"]["required"][0], "input"); + // 非 apply_patch 的 custom 工具仍透传 outer description,input 用泛指 + // 兜底描述,不注入 V4A 提示。 + assert_eq!(tool["function"]["description"], "anything"); + assert!( + tool["function"]["parameters"]["properties"]["input"]["description"] + .as_str() + .unwrap_or_default() + .contains("verbatim"), + "非 apply_patch 应保留泛指 input 描述,实际:{}", + tool["function"]["parameters"]["properties"]["input"]["description"] + ); +} + +#[test] +fn tools_custom_apply_patch_injects_v4a_format_hint() { + // 回归保护(issue #235):chat 上游(DeepSeek 等)拿到 freeform apply_patch + // 时,上游的 "do not wrap in JSON" 描述会误导模型;且原始描述里没有 V4A + // 格式样例。adapter 必须替换描述为 chat 路径准确的 V4A 指引,模型才能 + // 正确填充 `input` 字段。 + let out = convert(json!({ + "input": "hi", + "tools": [{ + "type": "custom", + "name": "apply_patch", + "description": "Use the `apply_patch` tool to edit files. This is a FREEFORM tool, so do not wrap the patch in JSON." + }] + })); + let tool = &out["tools"][0]; + assert_eq!(tool["type"], "function"); + assert_eq!(tool["function"]["name"], "apply_patch"); + + // outer description 必须替换(不能保留误导性的 "do not wrap" 文本) + let outer = tool["function"]["description"].as_str().unwrap_or_default(); + assert!(!outer.contains("do not wrap"), "误导性原描述未替换:{outer}"); + assert!( + outer.contains("V4A"), + "outer description 应当包含 V4A 关键字:{outer}" + ); + assert!( + outer.contains("*** Begin Patch"), + "outer description 应当含 V4A 边界标记:{outer}" + ); + + // input 参数描述必须含 V4A 格式约束(provider 可能更看 parameter desc) + let input_desc = tool["function"]["parameters"]["properties"]["input"]["description"] + .as_str() + .unwrap_or_default(); + assert!( + input_desc.contains("V4A") && input_desc.contains("*** Begin Patch"), + "input description 应含 V4A 与边界标记:{input_desc}" + ); + + // 回归保护(issue #235 真机验证暴露的二级问题):tool 描述必须显式解释 + // hunk semantics —— context 锚点 vs space-prefixed 行的区别。DeepSeek 在没 + // 有 lark grammar 强约束的 chat 路径上反复栽在这里(把 anchor 当 space 行 + // 重复一次),花 20 分钟、25+ 次 retry 最后 fallback 到 sed。description + // 必须含可执行的最小示例 + 显式的"do not repeat the anchor"指引。 + let outer_lc = outer.to_lowercase(); + assert!( + outer.contains("@@") && outer.contains("anchor"), + "tool description 必须解释 hunk anchor 语义:{outer}" + ); + assert!( + outer.contains("AFTER the anchor") || outer.contains("after it"), + "必须显式说明 space 行对应 anchor *之后* 的位置:{outer}" + ); + assert!( + outer_lc.contains("do not repeat the anchor") || outer_lc.contains("not again as a space"), + "必须显式禁止把 anchor 当 space 行重复:{outer}" + ); + assert!( + outer.contains("*** Update File:") && outer.contains("@@ fn main()"), + "必须包含一个最小可执行 V4A 示例:{outer}" + ); + + // 参数描述同样必须保留紧凑版的语义提示 + assert!( + input_desc.contains("anchor") + && (input_desc.contains("AFTER") || input_desc.contains("after")), + "input description 必须保留 anchor 语义紧凑版:{input_desc}" + ); } #[test] diff --git a/crates/adapters/src/responses/request/tools.rs b/crates/adapters/src/responses/request/tools.rs index 040c01f..c8ef490 100644 --- a/crates/adapters/src/responses/request/tools.rs +++ b/crates/adapters/src/responses/request/tools.rs @@ -3,6 +3,90 @@ use serde_json::{json, Value}; use super::provider_looks_like; +/// Codex freeform tool name we special-case. See the `"custom" =>` arm in +/// `convert_responses_tool_to_chat_tool` below for the request-side rewrite +/// rationale, and `converter.rs::close_tool_call` for the response-side +/// wire re-shape — they must trigger on the exact same tool name. +pub(crate) const APPLY_PATCH_TOOL_NAME: &str = "apply_patch"; + +/// Chat-path replacement for Codex CLI's freeform `apply_patch` description. +/// Original upstream text says "do not wrap the patch in JSON" because the +/// Responses API freeform/lark grammar accepts raw text — but on the +/// chat-completions path the model MUST emit a function call whose `input` +/// argument is a JSON string containing the V4A patch. We rewrite the +/// description so the model sees instructions consistent with the wire +/// format it has to produce. +/// +/// **重要:hunk body 的 space-prefixed 行语义** — 上游 freeform 工具用 lark +/// grammar 强制约束,模型在受约束的解码空间里不会搞错;但 chat function-call +/// 没有 grammar 约束,只剩 description。实测(issue #235 真机)DeepSeek +/// 反复在一个具体语义上栽跟头: +/// +/// > `@@ @@` 标记后的 space-prefixed 行 = 文件中 context 锚点 +/// > **之后**的行,**不是** context 行本身的重复 +/// +/// 不显式说清这个,模型会把 context 行当成 space 行再写一次,parse_patch +/// 找不到双行 → 整个 patch 拒收。本 description 通过显式规则 + 一个最小 +/// 可执行的更新文件 example 让模型看到正确形态。 +pub(crate) const APPLY_PATCH_TOOL_DESCRIPTION_FOR_CHAT: &str = concat!( + "Edit files using the apply_patch tool. ", + "Call this function with a single `input` string containing a V4A patch. ", + "The patch must start with `*** Begin Patch` and end with `*** End Patch`. ", + "Each file operation header is one of `*** Add File: `, ", + "`*** Update File: ` (optionally followed by `*** Move to: `), ", + "or `*** Delete File: `. ", + "Within Update hunks, use `@@ @@` markers, prefix unchanged lines ", + "with a single space, removed lines with `-`, and added lines with `+`. ", + "Use relative paths only (never absolute). ", + "Embed real newlines as `\\n` inside the JSON string value for `input`.\n\n", + "CRITICAL HUNK SEMANTICS (the most common cause of patch rejection):\n", + "`@@ @@` is an anchor that names ONE existing line in the file. ", + "Every space-prefixed line that follows the `@@` marker corresponds to lines ", + "AFTER the anchor in the file (not the anchor itself). ", + "Do NOT repeat the anchor line as the first space-prefixed line — the parser will reject it.\n\n", + "EXAMPLE — to change `let x = 1;` to `let x = 2;` in a file whose lines around the change read:\n", + " fn main() {\n", + " let x = 1;\n", + " println!(\"{}\", x);\n", + " }\n", + "The correct patch is:\n", + "*** Begin Patch\n", + "*** Update File: src/main.rs\n", + "@@ fn main() {\n", + "- let x = 1;\n", + "+ let x = 2;\n", + " println!(\"{}\", x);\n", + "*** End Patch\n", + "Notice: `fn main() {` appears in `@@ ... @@` once as the anchor, NOT again as a space-prefixed line below. ", + "The first content line under the anchor is the line immediately after `fn main() {` in the file.\n\n", + "CHAT-PATH GOTCHAS (the lark grammar is gone here; observed empirically with non-OpenAI providers):\n", + "1. Prefer matching an empty line as the `@@` anchor. Non-empty anchors often fail to match on this path. ", + "If needed, run `printf '\\n' >> ` first to seed a blank line, then `@@` with empty content, then clean up extra blank lines afterward.\n", + "2. Do NOT combine `*** Add File: foo` and `*** Update File: foo` in the SAME patch — Update reads the file before Add lands on disk. ", + "Either make Add File write the final content in one shot, or split into two separate patches.\n", + "3. `*** Update File:` cannot operate on a completely empty file. Use shell to write at least one line first, then apply_patch.\n", + "4. In a multi-line file, lone `+` lines AFTER an `@@` anchor APPEND below the anchor — they do NOT replace the anchor line. ", + "To change a line, use `-` to remove the old line AND `+` to add the new one; do not omit the `-`." +); + +/// Chat-path replacement for the freeform `input` parameter description. +/// Mirrors `APPLY_PATCH_TOOL_DESCRIPTION_FOR_CHAT` but at the parameter level, +/// so the model sees the format constraint regardless of whether providers +/// surface tool-level or parameter-level descriptions more prominently. +/// Same anchor-vs-space-line gotcha called out here in compact form (some +/// providers truncate or de-emphasize tool-level descriptions on long +/// histories — keep the rule visible at parameter level too). +pub(crate) const APPLY_PATCH_INPUT_DESCRIPTION_FOR_CHAT: &str = concat!( + "A V4A patch starting with `*** Begin Patch` and ending with `*** End Patch`. ", + "Use `*** Add File:`, `*** Update File:`, or `*** Delete File:` headers and ", + "`@@ @@` hunks with ` `/`+`/`-` line prefixes. Relative paths only. ", + "CRITICAL: in an Update hunk the `@@ @@` anchor is a SINGLE existing file line; ", + "the space-prefixed lines following the anchor describe lines AFTER it (do not repeat the anchor). ", + "Chat-path gotchas: prefer empty-line anchors (seed with `printf '\\n' >> file` if needed); ", + "do not Add+Update the same path in one patch; Update cannot operate on a totally empty file; ", + "lone `+` lines after `@@` APPEND below the anchor (use `-` + `+` to replace a line)." +); + /// Responses tool 定义 → Chat tool 定义. /// 把单个 Responses API tool 转成零或多个 Chat Completions tool。 /// @@ -53,23 +137,51 @@ pub fn convert_responses_tool_to_chat_tool( })] } "custom" => { - // Custom tool(无 JSON schema)降级为接受单字符串 input 的 function + // Custom tool(Responses API freeform tool,无 JSON schema)降级为 + // 接受单字符串 input 的 function tool — chat completions 不认 + // `type:"custom"`,DeepSeek / Kimi / MiMo 等 chat 上游必须走 function。 + // + // **apply_patch 特判**:Codex CLI 把 apply_patch 作为 freeform 工具 + // 注册,wire description 是 "Use the `apply_patch` tool to edit files. + // This is a FREEFORM tool, so do not wrap the patch in JSON." + // (上游 `codex-rs/core/src/tools/handlers/apply_patch_spec.rs` 实证)。 + // 经 chat function-call 反而**必须**把 patch 包进 JSON 字符串值 —— + // 上游的 "do not wrap in JSON" 指令在 chat 路径下会误导模型, + // 且原 description 没给 V4A 格式样例。这里替换成对 chat 路径准确 + // 的指引,把 V4A 关键字 / 文件操作头 / hunk 标记列清楚,让 DeepSeek + // 之类的模型知道 input 字段该填什么。 + // 响应侧(converter.rs::close_tool_call)对 name==apply_patch 特判, + // 把模型回来的 function_call 重新打包成 custom_tool_call wire, + // 让 Codex CLI router (`ResponseItem::CustomToolCall`) 正确路由到 + // apply_patch handler(handler 硬要求 `ToolPayload::Custom { input }`, + // 见 `codex-rs/core/src/tools/handlers/apply_patch.rs:324`)。 let name = obj.get("name").and_then(|v| v.as_str()).unwrap_or(""); - let description = obj + let original_description = obj .get("description") .and_then(|v| v.as_str()) .unwrap_or(""); + let (tool_description, input_description) = if name == APPLY_PATCH_TOOL_NAME { + ( + APPLY_PATCH_TOOL_DESCRIPTION_FOR_CHAT.to_owned(), + APPLY_PATCH_INPUT_DESCRIPTION_FOR_CHAT.to_owned(), + ) + } else { + ( + original_description.to_owned(), + "Free-form input passed verbatim to the tool.".to_owned(), + ) + }; vec![json!({ "type": "function", "function": { "name": name, - "description": description, + "description": tool_description, "parameters": { "type": "object", "properties": { "input": { "type": "string", - "description": "Free-form input passed verbatim to the tool.", + "description": input_description, } }, "required": ["input"], diff --git a/crates/adapters/src/responses/shell_to_apply_patch.rs b/crates/adapters/src/responses/shell_to_apply_patch.rs new file mode 100644 index 0000000..ae2965f --- /dev/null +++ b/crates/adapters/src/responses/shell_to_apply_patch.rs @@ -0,0 +1,605 @@ +//! Adapter server-side normalize:把模型 emit 的 `exec_command` shell file-write +//! 模式重写为 `apply_patch` custom_tool_call,让 Codex Desktop 自家 V4A applier +//! 渲染 4+2 diff UI(代码块行号 / +N -M / 文件清单 / 颜色高亮 / 点击跳转 / +//! 工具调用抬头),无需依赖模型主动选 apply_patch 工具。 +//! +//! 设计灵感来自 Antigravity Go agent(`~/.local/bin/agy`,闭源 Google +//! internal Go 1.27 RC build,无公开 source)— `strings` dump 实证 binary +//! 含 `FILE_CHANGE_TYPE_EDIT` / `file_diff` / `file_diffs` / +//! `trajectory_file_diffs` 等 token,印证它在 server 层产出统一 diff 事件 +//! 而非依赖模型工具选择;**具体的 shell → V4A patch normalize 规则**是 +//! codex-app-transfer 自创,不存在可对照的上游 source。 +//! +//! MVP scope:**只支持 Add File / 完整 overwrite 三种 shell 形态**: +//! 1. here-doc:`cat <<'EOF' > /path/file\n\nEOF` +//! 2. printf:`printf '' > /path/file`(支持 `\n` `\t` `\\` 转义) +//! 3. echo:`echo '' > /path/file`(字面 literal,不展开转义) +//! +//! 严格 reject(透传原 exec_command 不动): +//! - `>>` append / `sed -i` in-place / `tee` / `tee -a` +//! - 多命令组合(`;` `&&` `||` 在 cmd 前段) +//! - pipe `|` / 子 shell `$(...)` 或 `` `...` `` 变量替换 +//! - 多 `>` redirect / `&>` / `2>` +//! - here-doc 终止符不匹配 / `echo` / `printf` flags(`-e` `-n` 等) +//! +//! Edit / append (`>>` / `tee -a` / `cat <>`) / `sed -i` in-place / +//! output normalize 等更复杂模式落 `docs/followup-tracker.md` #39 跟踪。 +//! +//! **Corner cases**(都有 unit test 覆盖,但在 MVP scope 外的输入也明示): +//! - 空 body(`echo '' > file`)→ V4A 单 `+` 行(模拟 echo 写一个 newline 行为) +//! - body 无 trailing newline:`echo` 路径自动补 `\n`(bash `echo` 默认行为), +//! `printf` / here-doc 保持原 body 不补 +//! - double-quote literal 仅展开 `\\` / `\"` 转义(其他转义 reject); +//! `printf` literal 展开 `\n` / `\t` / `\\` / `\'` / `\"` / `\r` +//! - 相对路径(`./local.txt`)接受,绝对路径(`/tmp/x.txt`)接受; +//! 含 shell metachar / 空格 / 引号的 path reject + +/// 检测到的 file-write 操作 → V4A patch payload。 +/// +/// `patch_input` 可直接作为 Codex Desktop apply_patch 函数调用的 `input` +/// 字符串(自带 `*** Begin Patch` / `*** End Patch` 包裹 + Add File 头 + `+` +/// 前缀每一行)。 +#[derive(Debug, Clone)] +pub struct V4APatch { + /// 目标文件路径(用于 telemetry / 日志);从原始 shell `>` redirect 提取。 + pub target_path: String, + /// 完整 V4A patch envelope。 + pub patch_input: String, +} + +/// 顶层入口:接受 `exec_command` 工具调用的 `cmd` 字符串,返回 `Some(V4APatch)` +/// 当且仅当 cmd 是 MVP scope 内的纯 file-write,否则 `None`(透传原 exec_command)。 +pub fn detect_shell_file_write(raw_cmd: &str) -> Option { + let cmd = raw_cmd.trim(); + if cmd.is_empty() { + return None; + } + + // Stage 1: 全局禁用模式(无论命令是哪类,都 reject) + if has_disallowed_constructs(cmd) { + return None; + } + + // Stage 2: 按 pattern 顺序尝试匹配 + if let Some(patch) = try_here_doc(cmd) { + return Some(patch); + } + if let Some(patch) = try_printf_redirect(cmd) { + return Some(patch); + } + if let Some(patch) = try_echo_redirect(cmd) { + return Some(patch); + } + + None +} + +/// 全局禁用 token 检查 — 任意一个出现都拒绝(透传 exec_command)。 +fn has_disallowed_constructs(cmd: &str) -> bool { + // append redirect:`>>` 表示 append,无法纯 Add File 覆盖 + if cmd.contains(">>") { + return true; + } + // sed -i / sed -i '':in-place edit,需要 old content + if cmd.contains("sed -i") { + return true; + } + // tee:tee 默认 truncate,但模型用 tee 通常配 pipe(`echo x | tee file`) + // 或 `tee -a` append,边界复杂,MVP 全 reject + if regex_word_match(cmd, "tee") { + return true; + } + // 子 shell 变量替换:`$(...)` / `` `...` ``,内容动态生成无法预知 + if cmd.contains("$(") { + return true; + } + if cmd.contains('`') { + return true; + } + // bare 变量引用:`$VAR` / `${VAR}`(IMPORTANT-2 修复 / code-reviewer + // pre-push)。模型用 `echo "hello $USER" > file` 通常期待 shell 展开成 + // `hello alyse`,但我们 normalize 后直接写入字面 `$USER`,跟模型意图静默 + // divergence。配置文件 / 文档场景特别危险(`SECRET=$API_KEY` 字面落盘 + // 是真 bug)。`$(...)` 已在上面 reject,这里只补 bare var reject:任何 + // `$` 出现都 reject,简单且保险(单引号 literal 内 `$` 不展开但语义上 + // 模型既可能想字面也可能想展开,reject 强迫模型用 apply_patch 显式表达)。 + if cmd.contains('$') { + return true; + } + // multi-command:在 here-doc body 中可以有 `;` 等,但 here-doc 解析逻辑 + // 自己处理。在 cmd 前段(`> file` 之前)如果有 `;` `&&` `||` `|` `&>` `2>` + // 等,reject。这里采用更严的策略:cmd 中任何位置含 `&&` `||` `&>` + // 都拒绝;`;` `|` 在 here-doc body 内可接受,需要专门 pattern 处理时再放 + // 行。MVP 严格策略:整 cmd 不能含这些 token。 + if cmd.contains("&&") || cmd.contains("||") || cmd.contains("&>") { + return true; + } + // `2>` (stderr redirect):reject 任意 stderr 重定向 + if cmd.contains("2>") { + return true; + } + false +} + +/// 简单"独立 word"匹配(避免 `committee` 把 `tee` 当 token 命中): +/// 检查 needle 是否作为完整 word 出现(前后是 cmd 边界 / 空格 / `;` 等)。 +fn regex_word_match(cmd: &str, needle: &str) -> bool { + let bytes = cmd.as_bytes(); + let nlen = needle.len(); + let mut i = 0; + while i + nlen <= bytes.len() { + if &bytes[i..i + nlen] == needle.as_bytes() { + let prev_ok = + i == 0 || matches!(bytes[i - 1], b' ' | b'\t' | b';' | b'\n' | b'(' | b'|'); + let next_ok = i + nlen == bytes.len() + || matches!( + bytes[i + nlen], + b' ' | b'\t' | b';' | b'\n' | b')' | b'|' | b'<' | b'>' + ); + if prev_ok && next_ok { + return true; + } + } + i += 1; + } + false +} + +/// 尝试 `cat <<['"]?EOF['"]? > /path\n\nEOF` 模式。 +/// 接受 unquoted / single-quote / double-quote 终止符(三种都常见)。 +fn try_here_doc(cmd: &str) -> Option { + // 必须以 `cat <<` 开头(允许前面 leading whitespace 但 trim 过了) + let rest = cmd.strip_prefix("cat ")?.trim_start(); + let rest = rest.strip_prefix("<<")?.trim_start(); + + // 解析终止符 token:可能是 'EOF' / "EOF" / EOF / -EOF(`<<-` 已被 + // strip_prefix 排除,这里不支持 `<<-`,strict) + let (terminator, after_term) = parse_heredoc_terminator(rest)?; + let after_term = after_term.trim_start(); + + // 必须是 `> /path`(单个 redirect) + let redirect_rest = after_term.strip_prefix('>')?.trim_start(); + // path 到 newline 结束 + let nl_idx = redirect_rest.find('\n')?; + let path = redirect_rest[..nl_idx].trim(); + if path.is_empty() { + return None; + } + if !is_simple_path(path) { + return None; + } + // body 从 newline 之后开始 + let body_start = &redirect_rest[nl_idx + 1..]; + + // 找 body 末尾的 terminator 行(`` 单独占一行,trim 后等于 + // terminator) + let body = strip_heredoc_terminator(body_start, &terminator)?; + + let patch_input = build_add_file_patch(path, body); + Some(V4APatch { + target_path: path.to_string(), + patch_input, + }) +} + +/// 解析 here-doc 终止符 token,返回 (terminator_no_quotes, remaining). +fn parse_heredoc_terminator(s: &str) -> Option<(String, &str)> { + let s = s.trim_start(); + if let Some(rest) = s.strip_prefix('\'') { + let end = rest.find('\'')?; + Some((rest[..end].to_string(), &rest[end + 1..])) + } else if let Some(rest) = s.strip_prefix('"') { + let end = rest.find('"')?; + Some((rest[..end].to_string(), &rest[end + 1..])) + } else { + // unquoted:取直到 whitespace / `>` + let end = s + .find(|c: char| c.is_whitespace() || c == '>') + .unwrap_or(s.len()); + if end == 0 { + return None; + } + let token = &s[..end]; + // terminator 必须是合法 shell identifier 风格(纯 alnum + underscore) + if !token + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_') + { + return None; + } + Some((token.to_string(), &s[end..])) + } +} + +/// 在 `body_with_terminator` 中找单独占一行的 terminator,返回 terminator +/// 之前的内容(不含 terminator 行)。 +fn strip_heredoc_terminator<'a>( + body_with_terminator: &'a str, + terminator: &str, +) -> Option<&'a str> { + let mut search_from = 0usize; + while search_from <= body_with_terminator.len() { + let slice = &body_with_terminator[search_from..]; + let idx = slice.find(terminator)?; + let absolute_idx = search_from + idx; + // 检查 terminator 前一字符是不是 newline(或 body 开头) + let prev_is_nl = + absolute_idx == 0 || body_with_terminator.as_bytes()[absolute_idx - 1] == b'\n'; + // 检查 terminator 后跟的是 newline / EOF(允许 trailing whitespace + // 直到 newline / 字符串末尾) + let after = &body_with_terminator[absolute_idx + terminator.len()..]; + let trailing_only_ws_then_nl_or_eof = after + .split_once('\n') + .map(|(before_nl, _)| before_nl.chars().all(char::is_whitespace)) + .unwrap_or_else(|| after.chars().all(char::is_whitespace)); + if prev_is_nl && trailing_only_ws_then_nl_or_eof { + // body 不含末尾的 newline + terminator 行 + let body_end = if absolute_idx > 0 { + absolute_idx - 1 + } else { + 0 + }; + return Some(&body_with_terminator[..body_end]); + } + search_from = absolute_idx + terminator.len(); + } + None +} + +/// 尝试 `printf '' > /path` 或 `printf "" > /path` 模式。 +/// **支持** \n / \t / \\ / \' / \" 转义展开(shell printf 语义)。 +/// **拒绝** 含 `%` 格式串(`%s` `%d` 等,因为格式串需要额外 args)。 +fn try_printf_redirect(cmd: &str) -> Option { + let rest = cmd.strip_prefix("printf ")?.trim_start(); + let (literal, after) = parse_quoted_literal(rest)?; + // printf 格式串如果含 `%`(非 `%%` 转义)且后面没跟参数 → 模型可能误用, + // MVP 直接 reject(避免错误展开) + if has_unescaped_percent(&literal) { + return None; + } + let after = after.trim(); + // printf 后必须只剩 `> /path` + let path = after.strip_prefix('>')?.trim(); + if path.is_empty() || !is_simple_path(path) { + return None; + } + let body = expand_printf_escapes(&literal); + let patch_input = build_add_file_patch(path, &body); + Some(V4APatch { + target_path: path.to_string(), + patch_input, + }) +} + +/// 尝试 `echo '' > /path` 或 `echo "" > /path` 模式。 +/// echo **不**展开转义(MVP 拒绝 `echo -e`),body 是字面 literal。 +fn try_echo_redirect(cmd: &str) -> Option { + let rest = cmd.strip_prefix("echo ")?.trim_start(); + // 拒绝 echo flags(`-e` / `-n` / `-E`) + if rest.starts_with('-') { + return None; + } + let (literal, after) = parse_quoted_literal(rest)?; + let after = after.trim(); + let path = after.strip_prefix('>')?.trim(); + if path.is_empty() || !is_simple_path(path) { + return None; + } + // echo 默认在 body 末尾追加一个 newline(bash 行为),保持忠实模拟 + let mut body = literal; + if !body.ends_with('\n') { + body.push('\n'); + } + let patch_input = build_add_file_patch(path, &body); + Some(V4APatch { + target_path: path.to_string(), + patch_input, + }) +} + +/// 解析一段以 `'` 或 `"` 包裹的 literal,返回 (literal_no_quotes, remaining). +/// single-quote 内不支持任何转义(shell 标准:single-quote literal 不解释); +/// double-quote 内支持基础 `\"` `\\`,但拒绝变量(`$var` / `${...}` 已在 +/// `has_disallowed_constructs` 提前拒绝)。 +fn parse_quoted_literal(s: &str) -> Option<(String, &str)> { + let s = s.trim_start(); + if let Some(after_open) = s.strip_prefix('\'') { + // single-quote literal:**绝不**支持转义,直到下一个 `'` 结束 + let end = after_open.find('\'')?; + Some((after_open[..end].to_string(), &after_open[end + 1..])) + } else if let Some(after_open) = s.strip_prefix('"') { + // double-quote literal:支持 `\\` `\"` 转义,其他字面 + let mut out = String::new(); + let mut iter = after_open.char_indices(); + while let Some((idx, ch)) = iter.next() { + match ch { + '"' => return Some((out, &after_open[idx + 1..])), + '\\' => { + let (_i2, ch2) = iter.next()?; + match ch2 { + '"' => out.push('"'), + '\\' => out.push('\\'), + // 拒绝其他转义(模型可能想要 \n,但 double-quote + // shell 不展开 \n,字面保留 → 跟 printf 行为差异 + // 大,MVP 简单全 reject 其他转义) + _ => return None, + } + } + _ => out.push(ch), + } + } + None + } else { + None + } +} + +/// 检查 literal 是否含 unescaped `%`(printf 格式串需要 args)。 +fn has_unescaped_percent(literal: &str) -> bool { + let bytes = literal.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' { + // `%%` 是 literal `%` 转义,继续 + if i + 1 < bytes.len() && bytes[i + 1] == b'%' { + i += 2; + continue; + } + return true; + } + i += 1; + } + false +} + +/// 展开 printf 风格转义:`\n` `\t` `\\` `\'` `\"`。其他转义保持字面。 +fn expand_printf_escapes(literal: &str) -> String { + let mut out = String::with_capacity(literal.len()); + let mut iter = literal.chars(); + while let Some(ch) = iter.next() { + if ch == '\\' { + match iter.next() { + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some('\\') => out.push('\\'), + Some('\'') => out.push('\''), + Some('"') => out.push('"'), + Some('r') => out.push('\r'), + Some(other) => { + out.push('\\'); + out.push(other); + } + None => out.push('\\'), + } + } else { + out.push(ch); + } + } + out +} + +/// "simple path" 检查:必须是绝对路径(`/` 开头)或相对路径(`./` `../` +/// 或不含特殊 shell 字符的 token)。拒绝含空格 / 引号 / shell metachar 的 +/// path,避免歧义。 +fn is_simple_path(path: &str) -> bool { + if path.is_empty() { + return false; + } + !path.chars().any(|c| { + c.is_whitespace() + || c == '"' + || c == '\'' + || c == '$' + || c == '`' + || c == '*' + || c == '?' + || c == '[' + || c == ']' + || c == '{' + || c == '}' + || c == '|' + || c == '&' + || c == ';' + || c == '<' + || c == '>' + || c == '(' + || c == ')' + }) +} + +/// 把 (path, body) 组装成 V4A `*** Add File: ` patch envelope。 +/// body 按 `\n` 拆行,每行加 `+` 前缀。空 body 写入 1 行 `+`(空文件)。 +fn build_add_file_patch(path: &str, body: &str) -> String { + let mut out = String::with_capacity(body.len() + 128); + out.push_str("*** Begin Patch\n"); + out.push_str("*** Add File: "); + out.push_str(path); + out.push('\n'); + if body.is_empty() { + // V4A Add File 至少一行 — 空文件用单独 `+` 行 + out.push_str("+\n"); + } else { + // 注意:body 可能以 `\n` 结尾(printf '\n' / echo append 后),拆分时 + // 不要产生最后一个空 element。用 split_inclusive + manual handle 或 + // 用 lines() 但 lines() 会丢末尾 \n。这里用 split('\n'),如果末尾 + // 是 '\n',split 会产生最后一个空 element,跳过它。 + let parts: Vec<&str> = body.split('\n').collect(); + let last_idx = parts.len() - 1; + for (idx, line) in parts.iter().enumerate() { + if idx == last_idx && line.is_empty() { + // 末尾 newline 产生的空 element:不输出 `+` 行(避免多余空行) + break; + } + out.push('+'); + out.push_str(line); + out.push('\n'); + } + } + out.push_str("*** End Patch\n"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + // ===== Happy path ===== + + #[test] + fn here_doc_unquoted_terminator() { + let cmd = "cat < /tmp/a.py\nprint(\"hi\")\nEOF"; + let p = detect_shell_file_write(cmd).expect("should detect"); + assert_eq!(p.target_path, "/tmp/a.py"); + assert!(p.patch_input.contains("*** Add File: /tmp/a.py")); + assert!(p.patch_input.contains("+print(\"hi\")")); + assert!(p.patch_input.ends_with("*** End Patch\n")); + } + + #[test] + fn here_doc_single_quoted_terminator() { + let cmd = "cat <<'EOF' > /tmp/b.txt\nline1\nline2\nEOF"; + let p = detect_shell_file_write(cmd).expect("should detect"); + assert_eq!(p.target_path, "/tmp/b.txt"); + assert!(p.patch_input.contains("+line1\n+line2\n")); + } + + #[test] + fn here_doc_multiline_body() { + let cmd = + "cat < /tmp/app.py\ndef main():\n pass\n\nif __name__ == \"__main__\":\n main()\nPYEOF"; + let p = detect_shell_file_write(cmd).expect("should detect"); + assert!(p.patch_input.contains("+def main():")); + assert!(p.patch_input.contains("+ pass")); + // 空行作为单独 `+` 行 + assert!(p.patch_input.contains("+\n+if __name__")); + } + + #[test] + fn printf_single_quote_with_newlines() { + let cmd = "printf 'a\\nb\\nc\\n' > /tmp/c.txt"; + let p = detect_shell_file_write(cmd).expect("should detect"); + // \n 展开后 body = "a\nb\nc\n" + assert!(p.patch_input.contains("+a\n+b\n+c\n")); + } + + #[test] + fn echo_single_quote_simple() { + let cmd = "echo 'hello' > /tmp/d.txt"; + let p = detect_shell_file_write(cmd).expect("should detect"); + // echo 在末尾自动加 newline + assert!(p.patch_input.contains("+hello\n")); + } + + #[test] + fn echo_chinese_content() { + let cmd = "echo '# 测试笔记' > /tmp/note.md"; + let p = detect_shell_file_write(cmd).expect("should detect"); + assert!(p.patch_input.contains("+# 测试笔记")); + } + + // ===== Reject path ===== + + #[test] + fn reject_append_redirect() { + assert!(detect_shell_file_write("echo 'x' >> /tmp/a.txt").is_none()); + assert!(detect_shell_file_write("cat <> /tmp/a.txt\nx\nEOF").is_none()); + } + + #[test] + fn reject_sed_inplace() { + assert!(detect_shell_file_write("sed -i 's/x/y/' /tmp/a.txt").is_none()); + assert!(detect_shell_file_write("sed -i '' 's/x/y/' /tmp/a.txt").is_none()); + } + + #[test] + fn reject_multi_command() { + assert!(detect_shell_file_write("mkdir -p /tmp/d && echo 'x' > /tmp/d/a.txt").is_none()); + assert!(detect_shell_file_write("echo 'x' > /tmp/a.txt || true").is_none()); + } + + #[test] + fn reject_var_substitution() { + assert!(detect_shell_file_write("echo \"$(date)\" > /tmp/a.txt").is_none()); + assert!(detect_shell_file_write("echo \"`date`\" > /tmp/a.txt").is_none()); + } + + #[test] + fn reject_multi_redirect() { + assert!(detect_shell_file_write("echo 'x' > /tmp/a.txt 2> /tmp/err.txt").is_none()); + assert!(detect_shell_file_write("echo 'x' &> /tmp/a.txt").is_none()); + } + + #[test] + fn reject_tee() { + assert!(detect_shell_file_write("echo 'x' | tee /tmp/a.txt").is_none()); + } + + #[test] + fn reject_pipe() { + assert!(detect_shell_file_write("cat /tmp/a.txt | head > /tmp/b.txt").is_none()); + } + + #[test] + fn reject_bare_dollar_var() { + // IMPORTANT-2 修复(code-reviewer):`$VAR` / `${VAR}` 在 double-quote + // 内 shell 会展开,如果 normalize 成 V4A 写字面 `$VAR` 跟模型意图 + // divergence,reject 强制走 apply_patch 或 透传 shell。 + assert!(detect_shell_file_write(r#"echo "hello $USER" > /tmp/a.txt"#).is_none()); + assert!(detect_shell_file_write(r#"echo "${HOME}/file" > /tmp/a.txt"#).is_none()); + assert!(detect_shell_file_write(r#"printf "$1" > /tmp/a.txt"#).is_none()); + // 即使在 single-quote 内 $ 也 reject(简单一致策略,模型可以改 apply_patch) + assert!(detect_shell_file_write("echo '$USER literal' > /tmp/a.txt").is_none()); + } + + #[test] + fn reject_path_with_metachar() { + assert!(detect_shell_file_write("echo 'x' > /tmp/file with space.txt").is_none()); + assert!(detect_shell_file_write("echo 'x' > /tmp/*.txt").is_none()); + } + + #[test] + fn reject_echo_with_flags() { + assert!(detect_shell_file_write("echo -e 'a\\nb' > /tmp/a.txt").is_none()); + assert!(detect_shell_file_write("echo -n 'x' > /tmp/a.txt").is_none()); + } + + #[test] + fn reject_printf_format_string() { + assert!(detect_shell_file_write("printf 'hello %s' > /tmp/a.txt").is_none()); + } + + #[test] + fn passthrough_unrelated_commands() { + assert!(detect_shell_file_write("ls -la /tmp").is_none()); + assert!(detect_shell_file_write("cat /tmp/a.txt").is_none()); + assert!(detect_shell_file_write("grep -r pattern /tmp").is_none()); + assert!(detect_shell_file_write("python3 /tmp/a.py").is_none()); + } + + // ===== V4A patch envelope structure ===== + + #[test] + fn patch_envelope_correct_structure() { + let cmd = "echo 'a' > /tmp/x.txt"; + let p = detect_shell_file_write(cmd).unwrap(); + assert!(p.patch_input.starts_with("*** Begin Patch\n")); + assert!(p.patch_input.contains("*** Add File: /tmp/x.txt\n")); + assert!(p.patch_input.ends_with("*** End Patch\n")); + } + + #[test] + fn empty_body_single_plus_line() { + let cmd = "echo '' > /tmp/empty.txt"; + let p = detect_shell_file_write(cmd).unwrap(); + assert!(p + .patch_input + .contains("*** Add File: /tmp/empty.txt\n+\n*** End Patch\n")); + } + + #[test] + fn relative_path_accepted() { + let cmd = "echo 'x' > ./local-file.txt"; + let p = detect_shell_file_write(cmd).expect("relative path 应接受"); + assert_eq!(p.target_path, "./local-file.txt"); + } +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 895ff97..5170ca3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,12 +2,19 @@ 逐版本要点。详细变更见 [GitHub Releases](https://github.com/Cmochance/codex-app-transfer/releases) 与 `docs/release-notes/v*.md`。 -## Unreleased — PR #153 draft +## Unreleased — PR #153 draft + post-#236 shell→apply_patch server-side normalize **Anthropic Messages 协议适配**:新增 canonical `apiFormat=anthropic_messages`,将 Codex CLI Responses 请求转换到 Anthropic `/v1/messages`,并把 Anthropic Messages SSE 还原为 Responses SSE。当前 PR 已覆盖 text、thinking、tool_use、tool_result repair、`previous_response_id`、compact response、upstream error、provider test/model list 与 UI 保存显示路径。 Claude preset 暂不开放:需要 P7 真实 Claude text、tool-call、`previous_response_id`、upstream error 验证通过后再加入默认 preset。 +**shell→apply_patch server-side normalize**(post-PR #236 follow-up,issue #235):chat-completions provider 上模型偏好 `exec_command` 跑 `cat < file` / `printf '...' > file` / `echo '...' > file` 一次性写文件,绕过 `apply_patch` 工具 → Codex Desktop 收不到 `custom_tool_call` apply_patch 事件 → 不渲染 diff UI(代码块行号 / `+N -M` 增减 / 文件清单 / 颜色高亮 / 工具调用抬头 / 可点击文件名 6 个元素全失效)。设计灵感取自 Antigravity Go agent(`~/.local/bin/agy` strings dump 实证含 `FILE_CHANGE_TYPE_EDIT` / `file_diff` / `trajectory_file_diffs` 等 token,印证 server-side 统一 diff 事件模型),具体 shell → V4A 规则自创。本次 adapter 在 SSE 转换层归一化 file-write shell 意图为 apply_patch wire 事件: +- **MVP scope**:`cat < /path` / `printf '' > /path` / `echo '' > /path` 三种 happy pattern → 重写为 `custom_tool_call apply_patch` SSE 序列(added → input.delta → input.done → output_item.done),`call_id` 复用模型原 exec_command call_id 让下一 turn tool result 回灌正确关联。 +- **严格 reject**(透传 `function_call exec_command` 不动):`>>` append / `sed -i` in-place / `tee` / 多命令组合(`;` `&&` `||` `|`)/ 变量替换(`$(...)` `` `...` ``)/ 多 `>` / `2>` / `&>` / `echo -e/-n` flags / `printf %s` 格式串。 +- **新增模块** `crates/adapters/src/responses/shell_to_apply_patch.rs` + `crates/adapters/src/responses/converter.rs` 中加 `is_exec_command` flag + `emit_normalized_apply_patch` + `emit_deferred_exec_command_added` helper。 +- **Trade-off**:正常 shell 调用失去逐字符流式 UI(args buffer 到 close 阶段一次性 emit),但 args 通常 <1KB,first frame → close 间隔 <2s,可接受。 +- **Follow-up**(`docs/followup-tracker.md`):Edit / append (`>>`) / `sed -i` in-place / `tee -a` / output normalize(把 apply_patch 执行结果反向映射为 shell-style 输出避免模型困惑)。 + ## v2.1.6 — 2026-05-12 **关键修复**:MiniMax `role=system` 整请求 400(close #139)/ grok_web 多轮历史完整化(`assistant.tool_calls` flatten + `session_cache` 类型层面禁止 foot-gun)/ cloud_code(Gemini OAuth)多轮历史 silent loss prod bug。 diff --git a/docs/followup-tracker.md b/docs/followup-tracker.md index 84f205b..a7fbc14 100644 --- a/docs/followup-tracker.md +++ b/docs/followup-tracker.md @@ -80,6 +80,7 @@ related_pr: ## Active - [#32 P2 Plugin Unlock macOS:setAuthMethod 触发 React 整树重渲(物理消除可行性调研)](followup/32-plugin-unlock-react-context-rerender.md) — PR #191 已 P0 缓解,长期消除需 hook Codex Desktop preload 跨版本不稳 +- [#39 P2 shell→apply_patch normalize 扩展 Edit/append/sed -i + output normalize](followup/39-shell-to-apply-patch-edit-and-append.md) — issue #235 MVP 只覆盖 Add File/overwrite,Edit/append/in-place 需 old content 缓存或 IDE hook --- diff --git a/docs/followup/39-shell-to-apply-patch-edit-and-append.md b/docs/followup/39-shell-to-apply-patch-edit-and-append.md new file mode 100644 index 0000000..1dd3ddd --- /dev/null +++ b/docs/followup/39-shell-to-apply-patch-edit-and-append.md @@ -0,0 +1,69 @@ +--- +id: 39 +priority: P2 +type: refactor +status: active +created: 2026-05-21 +--- + +# shell→apply_patch normalize:扩展 Edit / append / sed -i 模式 + output normalize + +## Background + +本 PR(post-#236 follow-up,issue #235)的 MVP scope 只覆盖 **Add File / 完整 overwrite** 三种 shell file-write 模式: + +- `cat <<'EOF' > /path/file` (here-doc, single/double/unquoted terminator) +- `printf '' > /path/file` (escape \\n / \\t / \\\\ / \\' / \\" 展开) +- `echo '' > /path/file` (字面 literal,无 escape 展开) + +跑通后让 Codex Desktop 渲染 4+2 diff UI 元素(行号 / `+N -M` / 文件清单 / 颜色 / 抬头 / 可点击)。 + +**严格 reject**(透传原 exec_command 不动)的 shell 模式 + 触发条件构成本 followup: + +| 模式 | 出现场景 | 当前行为 | 期望行为 | +|------|----------|----------|----------| +| `>>` append | 追加 log line / 配置补行 | 透传 shell,UI 无 diff card | Edit hunk 加 `+` 行 in V4A | +| `sed -i 's/x/y/' file` | 替换字符串 | 透传 shell,UI 无 diff card | Edit hunk(需 read old content)| +| `sed -i 'N,Md' file` | 删除行 | 透传 | Edit hunk(删除行)| +| `tee -a file` | append via pipe | 透传 | Edit append | +| `cat <> file` | here-doc append | 透传 | Edit append | + +## 难点 + +Edit / append 模式需要 **old file content** 才能生成 V4A `-` 行(byte-exact)。adapter 不能跑 shell `cat` 拿真实文件(scope 违规 + 副作用),只能从以下来源拿: + +1. **模型在更早的 turn 用 exec_command `cat file` 已读过** → adapter 缓存 (file_path → last_cat_output)。多 turn 状态,跨 stream 持久化复杂。 +2. **Codex Desktop 提供 read_file MCP / hook** → 上游 IDE 端读,adapter 不需要持有状态。但需要 Codex Desktop 暴露这个能力(目前未确认)。 +3. **放弃 Edit,仅做 Add File + complete overwrite**(当前 MVP)。 + +## Output normalize(独立子项) + +MVP 后,模型下一 turn 看到的 tool_call_output 是 Codex Desktop apply_patch handler 返回的 `Success. Updated the following files: /tmp/x.py`。但模型期待的是 shell 输出 `Exit code: 0\n`。 + +模型可能困惑("我跑了 shell,为什么返回 apply_patch success?"),触发额外 verify turn(模型再发一个 `cat file` 确认)。 +worst case 多 1 turn 但不破坏 UI 渲染。 + +**期望优化**:adapter 把 apply_patch handler output 反向映射成 shell-style 输出格式,让模型无感: +``` +Exit code: 0 +Wall time: 0 seconds + +(file /tmp/x.py written: N lines) +``` + +## Acceptance criteria + +- Edit 模式支持(至少 `sed -i 's/x/y/' file` 单行替换):需要 adapter 自带文件缓存或 Codex Desktop hook +- append 模式支持(`>>` / `tee -a` / `cat <>`) +- output normalize:apply_patch handler output → shell-style 输出,模型 verify turn 减少 + +## 触发再开 PR 的信号 + +- 真机数据(round N+)显示用户大量使用 Edit / append 场景导致 UI 体验不一致 +- 上游 Codex Desktop 暴露 file read MCP / hook(消除 adapter 持有文件状态的复杂性) +- 用户报告"明明改了文件但 UI 没渲染"事件 ≥3 次 + +## Refs + +- 主 PR: issue #235 / PR #(本 PR) +- baseline 数据: `~/.codex-app-transfer/logs/apply-patch-debug/round{1,2,3}/`