From d69e79edd9b914338a4de1797c1fa632d854bf29 Mon Sep 17 00:00:00 2001 From: Shino Mailer Date: Wed, 1 Apr 2026 22:04:33 +0800 Subject: [PATCH 1/5] feat: add game update frequency control tools (issue #3) Adds three new MCP tools to let LLM/agents control Unity's game loop at inference speed, addressing the 60 FPS vs LLM throughput mismatch. New internal tools (exposed via unity_editor meta-tool): - editor_step_frame: advance exactly one frame via EditorApplication.Step() - editor_set_update_frequency: get/set Time.timeScale and Time.captureFramerate; no-arg call acts as a getter. captureFramerate is the recommended knob for deterministic LLM-driven simulation (deltaTime = timeScale/captureFramerate) - editor_play_for_frames: async advance N frames then pause, with full domain reload recovery via SessionState + AssemblyReloadEvents + bridge replay Also adds EditMode unit tests covering parameter validation and non-play-mode error paths for all three tools. Co-Authored-By: Claude Sonnet 4.6 --- Editor/Helpers/NativeMcpKeys.cs | 1 + Editor/Helpers/NativeMcpKeys.cs.meta | 2 + .../EditorControl/EditorPlayForFrames.cs | 259 ++++++++++++++++++ .../EditorControl/EditorPlayForFrames.cs.meta | 2 + .../EditorControl/EditorSetUpdateFrequency.cs | 56 ++++ .../EditorSetUpdateFrequency.cs.meta | 2 + Editor/Tools/EditorControl/EditorStepFrame.cs | 41 +++ .../EditorControl/EditorStepFrame.cs.meta | 2 + Editor/Tools/Meta/UnityEditorControl.cs | 20 +- Tests/Editor/EditorControlToolTests.cs | 124 +++++++++ Tests/Editor/EditorControlToolTests.cs.meta | 2 + Tests/Editor/RunTestsHelperTests.cs.meta | 2 + 12 files changed, 512 insertions(+), 1 deletion(-) create mode 100644 Editor/Helpers/NativeMcpKeys.cs.meta create mode 100644 Editor/Tools/EditorControl/EditorPlayForFrames.cs create mode 100644 Editor/Tools/EditorControl/EditorPlayForFrames.cs.meta create mode 100644 Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs create mode 100644 Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs.meta create mode 100644 Editor/Tools/EditorControl/EditorStepFrame.cs create mode 100644 Editor/Tools/EditorControl/EditorStepFrame.cs.meta create mode 100644 Tests/Editor/EditorControlToolTests.cs create mode 100644 Tests/Editor/EditorControlToolTests.cs.meta create mode 100644 Tests/Editor/RunTestsHelperTests.cs.meta diff --git a/Editor/Helpers/NativeMcpKeys.cs b/Editor/Helpers/NativeMcpKeys.cs index 420a298..7e0ddfd 100644 --- a/Editor/Helpers/NativeMcpKeys.cs +++ b/Editor/Helpers/NativeMcpKeys.cs @@ -7,6 +7,7 @@ internal static class NativeMcpKeys { // SessionState keys (per-session, lost on editor restart) public const string PendingTestRun = "NativeMcp_PendingTestRun"; + public const string PendingPlayForFrames = "NativeMcp_PendingPlayForFrames"; public const string LastPort = "NativeMcp_LastPort"; // EditorPrefs keys (persistent across sessions) diff --git a/Editor/Helpers/NativeMcpKeys.cs.meta b/Editor/Helpers/NativeMcpKeys.cs.meta new file mode 100644 index 0000000..aeeede4 --- /dev/null +++ b/Editor/Helpers/NativeMcpKeys.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 63e9566e924904ad48fa36779a8495b4 \ No newline at end of file diff --git a/Editor/Tools/EditorControl/EditorPlayForFrames.cs b/Editor/Tools/EditorControl/EditorPlayForFrames.cs new file mode 100644 index 0000000..8f84466 --- /dev/null +++ b/Editor/Tools/EditorControl/EditorPlayForFrames.cs @@ -0,0 +1,259 @@ +using System; +using System.Threading.Tasks; +using NativeMcp.Editor.Bridge; +using NativeMcp.Editor.Helpers; +using NativeMcp.Editor.Tools; +using Newtonsoft.Json.Linq; +using UnityEditor; +using UnityEngine; + +namespace NativeMcp.Editor.Tools.EditorControl +{ + [McpForUnityTool("editor_play_for_frames", Internal = true, + Description = "Play the game for a specified number of frames, then pause. Supports recovery across domain reloads.")] + public static class EditorPlayForFrames + { + private const int DefaultTimeoutSeconds = 30; + private const string PendingKey = NativeMcpKeys.PendingPlayForFrames; + + public class Parameters + { + [ToolParameter("Number of frames to advance (>= 1).")] + public int frames { get; set; } + + [ToolParameter("Timeout in seconds. Default 30.", Required = false)] + public int? timeout { get; set; } + } + + // Active operation state (in-memory, lost on domain reload) + private static TaskCompletionSource _activeTcs; + private static EditorApplication.CallbackFunction _activeTick; + private static int _framesRemaining; + private static int _framesRequested; + private static int _startFrame; + private static DateTime _deadlineUtc; + private static int _timeoutSeconds; + private static string _activeOperationId; + + // Domain reload recovery flag (set by [InitializeOnLoad] after reload) + private static bool _hasPendingRecovery; + + [Serializable] + private class PendingState + { + public string operationId; + public int framesRemaining; + public int framesRequested; + public string deadlineUtc; + } + + [InitializeOnLoad] + private static class ReloadWatcher + { + static ReloadWatcher() + { + AssemblyReloadEvents.beforeAssemblyReload += OnBeforeReload; + AssemblyReloadEvents.afterAssemblyReload += OnAfterReload; + } + } + + private static void OnBeforeReload() + { + if (_activeTcs == null || _activeTcs.Task.IsCompleted) + return; + + // Snapshot current state to SessionState + var state = new PendingState + { + operationId = _activeOperationId, + framesRemaining = _framesRemaining, + framesRequested = _framesRequested, + deadlineUtc = _deadlineUtc.ToString("O") + }; + SessionState.SetString(PendingKey, JsonUtility.ToJson(state)); + Debug.Log($"[NativeMcp] PlayForFrames: saved pending state before domain reload ({_framesRemaining} frames remaining)"); + + // Clean up in-memory state (will be lost anyway) + CancelActive(silent: true); + } + + private static void OnAfterReload() + { + string json = SessionState.GetString(PendingKey, ""); + if (!string.IsNullOrEmpty(json)) + { + _hasPendingRecovery = true; + Debug.Log("[NativeMcp] PlayForFrames: pending operation detected after domain reload, awaiting bridge replay."); + } + } + + public static async Task HandleCommand(JObject @params) + { + try + { + int frames = @params["frames"]?.Value() ?? 0; + int timeout = @params["timeout"]?.Value() ?? DefaultTimeoutSeconds; + + if (frames < 1) + return new ErrorResponse("frames must be >= 1."); + + if (!EditorApplication.isPlaying) + { + if (EditorApplication.isPlayingOrWillChangePlaymode) + return new ErrorResponse("Play mode is transitioning. Wait and try again."); + return new ErrorResponse("Not in play mode. Call 'play' first."); + } + + // Check for domain reload recovery + if (_hasPendingRecovery) + { + _hasPendingRecovery = false; + string json = SessionState.GetString(PendingKey, ""); + if (!string.IsNullOrEmpty(json)) + { + var recovered = JsonUtility.FromJson(json); + SessionState.EraseString(PendingKey); + + if (recovered.framesRemaining > 0) + { + Debug.Log($"[NativeMcp] PlayForFrames: recovering from domain reload, {recovered.framesRemaining} frames remaining."); + frames = recovered.framesRemaining; + var recoveredDeadline = DateTime.Parse(recovered.deadlineUtc).ToUniversalTime(); + if (DateTime.UtcNow >= recoveredDeadline) + { + EditorApplication.isPaused = true; + return new ErrorResponse("Timed out during domain reload.", new + { + frames_requested = recovered.framesRequested, + frames_elapsed = recovered.framesRequested - recovered.framesRemaining + }); + } + timeout = (int)Math.Max(1, (recoveredDeadline - DateTime.UtcNow).TotalSeconds); + } + } + } + + // Cancel any previous active operation + CancelActive(silent: false); + + // Set up new operation + _activeOperationId = Guid.NewGuid().ToString("N"); + _framesRequested = frames; + _framesRemaining = frames; + _startFrame = Time.frameCount; + _timeoutSeconds = timeout; + _deadlineUtc = DateTime.UtcNow.AddSeconds(timeout); + _activeTcs = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + + // Unpause if paused + EditorApplication.isPaused = false; + + // Start tick loop + EditorNudge.BeginNudge(); + + _activeTick = Tick; + EditorApplication.update += _activeTick; + + return await _activeTcs.Task; + } + catch (Exception e) + { + return new ErrorResponse($"Error in play_for_frames: {e.Message}"); + } + } + + private static void Tick() + { + if (_activeTcs == null || _activeTcs.Task.IsCompleted) + { + Cleanup(); + return; + } + + // Detect exit from play mode + if (!EditorApplication.isPlaying) + { + int elapsed = _framesRequested - _framesRemaining; + Cleanup(); + _activeTcs.TrySetResult(new ErrorResponse("Play mode exited during play_for_frames.", new + { + frames_requested = _framesRequested, + frames_elapsed = elapsed + })); + return; + } + + // Check timeout + if (DateTime.UtcNow >= _deadlineUtc) + { + int elapsed = _framesRequested - _framesRemaining; + EditorApplication.isPaused = true; + Cleanup(); + _activeTcs.TrySetResult(new ErrorResponse($"Timed out after {_timeoutSeconds}s.", new + { + frames_requested = _framesRequested, + frames_elapsed = elapsed + })); + return; + } + + // Count frame + _framesRemaining--; + + if (_framesRemaining <= 0) + { + int endFrame = Time.frameCount; + int elapsed = _framesRequested; + EditorApplication.isPaused = true; + Cleanup(); + _activeTcs.TrySetResult(new SuccessResponse($"Advanced {elapsed} frames and paused.", new + { + frames_requested = elapsed, + frames_elapsed = elapsed, + start_frame = _startFrame, + end_frame = endFrame + })); + } + } + + private static void CancelActive(bool silent) + { + if (_activeTick != null) + { + EditorApplication.update -= _activeTick; + _activeTick = null; + } + + EditorNudge.EndNudge(); + SessionState.EraseString(PendingKey); + + if (_activeTcs != null && !_activeTcs.Task.IsCompleted) + { + if (!silent) + { + _activeTcs.TrySetResult( + new ErrorResponse("Cancelled by new play_for_frames call.")); + } + else + { + _activeTcs.TrySetCanceled(); + } + } + + _activeTcs = null; + _activeOperationId = null; + } + + private static void Cleanup() + { + if (_activeTick != null) + { + EditorApplication.update -= _activeTick; + _activeTick = null; + } + EditorNudge.EndNudge(); + SessionState.EraseString(PendingKey); + } + } +} diff --git a/Editor/Tools/EditorControl/EditorPlayForFrames.cs.meta b/Editor/Tools/EditorControl/EditorPlayForFrames.cs.meta new file mode 100644 index 0000000..4db73e3 --- /dev/null +++ b/Editor/Tools/EditorControl/EditorPlayForFrames.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: f31a4f6fab66a4bb79fd32b6702711c6 \ No newline at end of file diff --git a/Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs b/Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs new file mode 100644 index 0000000..1b3edbc --- /dev/null +++ b/Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs @@ -0,0 +1,56 @@ +using System; +using NativeMcp.Editor.Helpers; +using NativeMcp.Editor.Tools; +using Newtonsoft.Json.Linq; +using UnityEngine; + +namespace NativeMcp.Editor.Tools.EditorControl +{ + [McpForUnityTool("editor_set_update_frequency", Internal = true, + Description = "Get or set game update frequency. Controls Time.timeScale and Time.captureFramerate. Call with no arguments to read current values.")] + public static class EditorSetUpdateFrequency + { + public class Parameters + { + [ToolParameter("Time scale multiplier (0=frozen, 1=normal). Range [0, 100].", Required = false)] + public float? time_scale { get; set; } + + [ToolParameter("Capture framerate for deterministic mode. When >0, deltaTime = timeScale/captureFramerate per frame regardless of real time. 0=off.", Required = false)] + public int? capture_framerate { get; set; } + } + + public static object HandleCommand(JObject @params) + { + try + { + float? timeScale = (float?)@params["time_scale"]; + int? captureFramerate = (int?)@params["capture_framerate"]; + + if (timeScale.HasValue) + { + if (timeScale.Value < 0f || timeScale.Value > 100f) + return new ErrorResponse("time_scale must be in range [0, 100]."); + Time.timeScale = timeScale.Value; + } + + if (captureFramerate.HasValue) + { + if (captureFramerate.Value < 0) + return new ErrorResponse("capture_framerate must be >= 0."); + Time.captureFramerate = captureFramerate.Value; + } + + return new SuccessResponse("Update frequency configured.", new + { + time_scale = Time.timeScale, + capture_framerate = Time.captureFramerate, + frame_count = Time.frameCount + }); + } + catch (Exception e) + { + return new ErrorResponse($"Error setting update frequency: {e.Message}"); + } + } + } +} diff --git a/Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs.meta b/Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs.meta new file mode 100644 index 0000000..83a2f19 --- /dev/null +++ b/Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: d01a7a2483cb941eeb5b3d35a71f201a \ No newline at end of file diff --git a/Editor/Tools/EditorControl/EditorStepFrame.cs b/Editor/Tools/EditorControl/EditorStepFrame.cs new file mode 100644 index 0000000..a5b0c00 --- /dev/null +++ b/Editor/Tools/EditorControl/EditorStepFrame.cs @@ -0,0 +1,41 @@ +using System; +using NativeMcp.Editor.Helpers; +using NativeMcp.Editor.Tools; +using Newtonsoft.Json.Linq; +using UnityEditor; +using UnityEngine; + +namespace NativeMcp.Editor.Tools.EditorControl +{ + [McpForUnityTool("editor_step_frame", Internal = true, + Description = "Advance exactly one frame while paused in play mode.")] + public static class EditorStepFrame + { + public static object HandleCommand(JObject @params) + { + try + { + if (!EditorApplication.isPlaying) + { + if (EditorApplication.isPlayingOrWillChangePlaymode) + return new ErrorResponse("Play mode is transitioning. Wait and try again."); + return new ErrorResponse("Not in play mode. Call 'play' first."); + } + + if (!EditorApplication.isPaused) + EditorApplication.isPaused = true; + + EditorApplication.Step(); + + return new SuccessResponse("Stepped 1 frame.", new + { + frame = Time.frameCount + }); + } + catch (Exception e) + { + return new ErrorResponse($"Error stepping frame: {e.Message}"); + } + } + } +} diff --git a/Editor/Tools/EditorControl/EditorStepFrame.cs.meta b/Editor/Tools/EditorControl/EditorStepFrame.cs.meta new file mode 100644 index 0000000..992de00 --- /dev/null +++ b/Editor/Tools/EditorControl/EditorStepFrame.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 3d40dbecdb0ae4c7a9f865b48d189cab \ No newline at end of file diff --git a/Editor/Tools/Meta/UnityEditorControl.cs b/Editor/Tools/Meta/UnityEditorControl.cs index f13bb69..0d0511f 100644 --- a/Editor/Tools/Meta/UnityEditorControl.cs +++ b/Editor/Tools/Meta/UnityEditorControl.cs @@ -12,6 +12,9 @@ namespace NativeMcp.Editor.Tools.Meta "- play() — enter play mode\n" + "- stop() — exit play mode\n" + "- pause() — toggle pause in play mode\n" + + "- step_frame() — advance exactly one frame while paused in play mode\n" + + "- play_for_frames(frames, timeout?) — play for N frames then pause. Supports domain reload recovery\n" + + "- set_update_frequency(time_scale?, capture_framerate?) — get/set game update frequency. No args = getter\n" + "- add_tag(tagName) — add a project tag\n" + "- remove_tag(tagName) — remove a project tag\n" + "- add_layer(layerName) — add a project layer\n" + @@ -25,6 +28,9 @@ public static class UnityEditorControl ["play"] = "editor_play", ["stop"] = "editor_stop", ["pause"] = "editor_pause", + ["step_frame"] = "editor_step_frame", + ["play_for_frames"] = "editor_play_for_frames", + ["set_update_frequency"] = "editor_set_update_frequency", ["add_tag"] = "editor_add_tag", ["remove_tag"] = "editor_remove_tag", ["add_layer"] = "editor_add_layer", @@ -35,9 +41,21 @@ public static class UnityEditorControl public class Parameters { - [ToolParameter("Action to perform: play, stop, pause, add_tag, remove_tag, add_layer, remove_layer, set_active_tool, refresh")] + [ToolParameter("Action to perform: play, stop, pause, step_frame, play_for_frames, set_update_frequency, add_tag, remove_tag, add_layer, remove_layer, set_active_tool, refresh")] public string action { get; set; } + [ToolParameter("Number of frames to advance for play_for_frames (>= 1)", Required = false)] + public int? frames { get; set; } + + [ToolParameter("Timeout in seconds for play_for_frames (default 30)", Required = false)] + public int? timeout { get; set; } + + [ToolParameter("Time scale for set_update_frequency (0=frozen, 1=normal, range [0,100])", Required = false)] + public float? time_scale { get; set; } + + [ToolParameter("Capture framerate for deterministic mode (0=off). deltaTime = timeScale/captureFramerate", Required = false)] + public int? capture_framerate { get; set; } + [ToolParameter("Tag name for add_tag/remove_tag actions", Required = false)] public string tagName { get; set; } diff --git a/Tests/Editor/EditorControlToolTests.cs b/Tests/Editor/EditorControlToolTests.cs new file mode 100644 index 0000000..e091622 --- /dev/null +++ b/Tests/Editor/EditorControlToolTests.cs @@ -0,0 +1,124 @@ +using NativeMcp.Editor.Helpers; +using NativeMcp.Editor.Tools.EditorControl; +using Newtonsoft.Json.Linq; +using NUnit.Framework; + +namespace NativeMcp.Editor.Tests +{ + [TestFixture] + public class EditorControlToolTests + { + // --- EditorStepFrame --- + + [Test] + public void StepFrame_NotInPlayMode_ReturnsError() + { + var result = EditorStepFrame.HandleCommand(new JObject()); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("Not in play mode")); + } + + // --- EditorSetUpdateFrequency --- + + [Test] + public void SetUpdateFrequency_NoArgs_ReturnsCurrentValues() + { + var result = EditorSetUpdateFrequency.HandleCommand(new JObject()); + Assert.IsInstanceOf(result); + + var success = (SuccessResponse)result; + Assert.That(success.Message, Does.Contain("configured")); + Assert.IsNotNull(success.Data); + } + + [Test] + public void SetUpdateFrequency_TimeScaleNegative_ReturnsError() + { + var @params = new JObject { ["time_scale"] = -1f }; + var result = EditorSetUpdateFrequency.HandleCommand(@params); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("time_scale")); + } + + [Test] + public void SetUpdateFrequency_TimeScaleAboveMax_ReturnsError() + { + var @params = new JObject { ["time_scale"] = 101f }; + var result = EditorSetUpdateFrequency.HandleCommand(@params); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("time_scale")); + } + + [Test] + public void SetUpdateFrequency_CaptureFramerateNegative_ReturnsError() + { + var @params = new JObject { ["capture_framerate"] = -1 }; + var result = EditorSetUpdateFrequency.HandleCommand(@params); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("capture_framerate")); + } + + [Test] + public void SetUpdateFrequency_ValidTimeScale_AppliesAndReturns() + { + float original = UnityEngine.Time.timeScale; + try + { + var @params = new JObject { ["time_scale"] = 0.5f }; + var result = EditorSetUpdateFrequency.HandleCommand(@params); + Assert.IsInstanceOf(result); + Assert.AreEqual(0.5f, UnityEngine.Time.timeScale, 0.001f); + } + finally + { + UnityEngine.Time.timeScale = original; + } + } + + [Test] + public void SetUpdateFrequency_ValidCaptureFramerate_AppliesAndReturns() + { + int original = UnityEngine.Time.captureFramerate; + try + { + var @params = new JObject { ["capture_framerate"] = 10 }; + var result = EditorSetUpdateFrequency.HandleCommand(@params); + Assert.IsInstanceOf(result); + Assert.AreEqual(10, UnityEngine.Time.captureFramerate); + } + finally + { + UnityEngine.Time.captureFramerate = original; + } + } + + // --- EditorPlayForFrames --- + + [Test] + public void PlayForFrames_FramesZero_ReturnsError() + { + var @params = new JObject { ["frames"] = 0 }; + var result = EditorPlayForFrames.HandleCommand(@params).Result; + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("frames")); + } + + [Test] + public void PlayForFrames_FramesNegative_ReturnsError() + { + var @params = new JObject { ["frames"] = -5 }; + var result = EditorPlayForFrames.HandleCommand(@params).Result; + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("frames")); + } + + [Test] + public void PlayForFrames_NotInPlayMode_ReturnsError() + { + var @params = new JObject { ["frames"] = 10 }; + var result = EditorPlayForFrames.HandleCommand(@params).Result; + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("Not in play mode")); + } + } +} diff --git a/Tests/Editor/EditorControlToolTests.cs.meta b/Tests/Editor/EditorControlToolTests.cs.meta new file mode 100644 index 0000000..0ab83f6 --- /dev/null +++ b/Tests/Editor/EditorControlToolTests.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 36ee3a523fd874daf8d0f6a9b8baffd3 \ No newline at end of file diff --git a/Tests/Editor/RunTestsHelperTests.cs.meta b/Tests/Editor/RunTestsHelperTests.cs.meta new file mode 100644 index 0000000..f0e231e --- /dev/null +++ b/Tests/Editor/RunTestsHelperTests.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 08f0d159925114e53b8127a362cbf0bd \ No newline at end of file From f325c7b118c211738c01f8433d3f59e664552da4 Mon Sep 17 00:00:00 2001 From: Shino Mailer Date: Wed, 1 Apr 2026 22:04:33 +0800 Subject: [PATCH 2/5] feat: add game update frequency control tools (issue #3) Adds three new MCP tools to let LLM/agents control Unity's game loop at inference speed, addressing the 60 FPS vs LLM throughput mismatch. New internal tools (exposed via unity_editor meta-tool): - editor_step_frame: advance exactly one frame via EditorApplication.Step() - editor_set_update_frequency: get/set Time.timeScale and Time.captureFramerate; no-arg call acts as a getter. captureFramerate is the recommended knob for deterministic LLM-driven simulation (deltaTime = timeScale/captureFramerate) - editor_play_for_frames: async advance N frames then pause, with full domain reload recovery via SessionState + AssemblyReloadEvents + bridge replay Also adds EditMode unit tests covering parameter validation and non-play-mode error paths for all three tools. Co-Authored-By: Claude Sonnet 4.6 --- Editor/Helpers/NativeMcpKeys.cs | 1 + Editor/Helpers/NativeMcpKeys.cs.meta | 2 + .../EditorControl/EditorPlayForFrames.cs | 259 ++++++++++++++++++ .../EditorControl/EditorPlayForFrames.cs.meta | 2 + .../EditorControl/EditorSetUpdateFrequency.cs | 56 ++++ .../EditorSetUpdateFrequency.cs.meta | 2 + Editor/Tools/EditorControl/EditorStepFrame.cs | 41 +++ .../EditorControl/EditorStepFrame.cs.meta | 2 + Editor/Tools/Meta/UnityEditorControl.cs | 20 +- Tests/Editor/EditorControlToolTests.cs | 124 +++++++++ Tests/Editor/EditorControlToolTests.cs.meta | 2 + Tests/Editor/RunTestsHelperTests.cs.meta | 2 + 12 files changed, 512 insertions(+), 1 deletion(-) create mode 100644 Editor/Helpers/NativeMcpKeys.cs.meta create mode 100644 Editor/Tools/EditorControl/EditorPlayForFrames.cs create mode 100644 Editor/Tools/EditorControl/EditorPlayForFrames.cs.meta create mode 100644 Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs create mode 100644 Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs.meta create mode 100644 Editor/Tools/EditorControl/EditorStepFrame.cs create mode 100644 Editor/Tools/EditorControl/EditorStepFrame.cs.meta create mode 100644 Tests/Editor/EditorControlToolTests.cs create mode 100644 Tests/Editor/EditorControlToolTests.cs.meta create mode 100644 Tests/Editor/RunTestsHelperTests.cs.meta diff --git a/Editor/Helpers/NativeMcpKeys.cs b/Editor/Helpers/NativeMcpKeys.cs index 420a298..7e0ddfd 100644 --- a/Editor/Helpers/NativeMcpKeys.cs +++ b/Editor/Helpers/NativeMcpKeys.cs @@ -7,6 +7,7 @@ internal static class NativeMcpKeys { // SessionState keys (per-session, lost on editor restart) public const string PendingTestRun = "NativeMcp_PendingTestRun"; + public const string PendingPlayForFrames = "NativeMcp_PendingPlayForFrames"; public const string LastPort = "NativeMcp_LastPort"; // EditorPrefs keys (persistent across sessions) diff --git a/Editor/Helpers/NativeMcpKeys.cs.meta b/Editor/Helpers/NativeMcpKeys.cs.meta new file mode 100644 index 0000000..aeeede4 --- /dev/null +++ b/Editor/Helpers/NativeMcpKeys.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 63e9566e924904ad48fa36779a8495b4 \ No newline at end of file diff --git a/Editor/Tools/EditorControl/EditorPlayForFrames.cs b/Editor/Tools/EditorControl/EditorPlayForFrames.cs new file mode 100644 index 0000000..8f84466 --- /dev/null +++ b/Editor/Tools/EditorControl/EditorPlayForFrames.cs @@ -0,0 +1,259 @@ +using System; +using System.Threading.Tasks; +using NativeMcp.Editor.Bridge; +using NativeMcp.Editor.Helpers; +using NativeMcp.Editor.Tools; +using Newtonsoft.Json.Linq; +using UnityEditor; +using UnityEngine; + +namespace NativeMcp.Editor.Tools.EditorControl +{ + [McpForUnityTool("editor_play_for_frames", Internal = true, + Description = "Play the game for a specified number of frames, then pause. Supports recovery across domain reloads.")] + public static class EditorPlayForFrames + { + private const int DefaultTimeoutSeconds = 30; + private const string PendingKey = NativeMcpKeys.PendingPlayForFrames; + + public class Parameters + { + [ToolParameter("Number of frames to advance (>= 1).")] + public int frames { get; set; } + + [ToolParameter("Timeout in seconds. Default 30.", Required = false)] + public int? timeout { get; set; } + } + + // Active operation state (in-memory, lost on domain reload) + private static TaskCompletionSource _activeTcs; + private static EditorApplication.CallbackFunction _activeTick; + private static int _framesRemaining; + private static int _framesRequested; + private static int _startFrame; + private static DateTime _deadlineUtc; + private static int _timeoutSeconds; + private static string _activeOperationId; + + // Domain reload recovery flag (set by [InitializeOnLoad] after reload) + private static bool _hasPendingRecovery; + + [Serializable] + private class PendingState + { + public string operationId; + public int framesRemaining; + public int framesRequested; + public string deadlineUtc; + } + + [InitializeOnLoad] + private static class ReloadWatcher + { + static ReloadWatcher() + { + AssemblyReloadEvents.beforeAssemblyReload += OnBeforeReload; + AssemblyReloadEvents.afterAssemblyReload += OnAfterReload; + } + } + + private static void OnBeforeReload() + { + if (_activeTcs == null || _activeTcs.Task.IsCompleted) + return; + + // Snapshot current state to SessionState + var state = new PendingState + { + operationId = _activeOperationId, + framesRemaining = _framesRemaining, + framesRequested = _framesRequested, + deadlineUtc = _deadlineUtc.ToString("O") + }; + SessionState.SetString(PendingKey, JsonUtility.ToJson(state)); + Debug.Log($"[NativeMcp] PlayForFrames: saved pending state before domain reload ({_framesRemaining} frames remaining)"); + + // Clean up in-memory state (will be lost anyway) + CancelActive(silent: true); + } + + private static void OnAfterReload() + { + string json = SessionState.GetString(PendingKey, ""); + if (!string.IsNullOrEmpty(json)) + { + _hasPendingRecovery = true; + Debug.Log("[NativeMcp] PlayForFrames: pending operation detected after domain reload, awaiting bridge replay."); + } + } + + public static async Task HandleCommand(JObject @params) + { + try + { + int frames = @params["frames"]?.Value() ?? 0; + int timeout = @params["timeout"]?.Value() ?? DefaultTimeoutSeconds; + + if (frames < 1) + return new ErrorResponse("frames must be >= 1."); + + if (!EditorApplication.isPlaying) + { + if (EditorApplication.isPlayingOrWillChangePlaymode) + return new ErrorResponse("Play mode is transitioning. Wait and try again."); + return new ErrorResponse("Not in play mode. Call 'play' first."); + } + + // Check for domain reload recovery + if (_hasPendingRecovery) + { + _hasPendingRecovery = false; + string json = SessionState.GetString(PendingKey, ""); + if (!string.IsNullOrEmpty(json)) + { + var recovered = JsonUtility.FromJson(json); + SessionState.EraseString(PendingKey); + + if (recovered.framesRemaining > 0) + { + Debug.Log($"[NativeMcp] PlayForFrames: recovering from domain reload, {recovered.framesRemaining} frames remaining."); + frames = recovered.framesRemaining; + var recoveredDeadline = DateTime.Parse(recovered.deadlineUtc).ToUniversalTime(); + if (DateTime.UtcNow >= recoveredDeadline) + { + EditorApplication.isPaused = true; + return new ErrorResponse("Timed out during domain reload.", new + { + frames_requested = recovered.framesRequested, + frames_elapsed = recovered.framesRequested - recovered.framesRemaining + }); + } + timeout = (int)Math.Max(1, (recoveredDeadline - DateTime.UtcNow).TotalSeconds); + } + } + } + + // Cancel any previous active operation + CancelActive(silent: false); + + // Set up new operation + _activeOperationId = Guid.NewGuid().ToString("N"); + _framesRequested = frames; + _framesRemaining = frames; + _startFrame = Time.frameCount; + _timeoutSeconds = timeout; + _deadlineUtc = DateTime.UtcNow.AddSeconds(timeout); + _activeTcs = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + + // Unpause if paused + EditorApplication.isPaused = false; + + // Start tick loop + EditorNudge.BeginNudge(); + + _activeTick = Tick; + EditorApplication.update += _activeTick; + + return await _activeTcs.Task; + } + catch (Exception e) + { + return new ErrorResponse($"Error in play_for_frames: {e.Message}"); + } + } + + private static void Tick() + { + if (_activeTcs == null || _activeTcs.Task.IsCompleted) + { + Cleanup(); + return; + } + + // Detect exit from play mode + if (!EditorApplication.isPlaying) + { + int elapsed = _framesRequested - _framesRemaining; + Cleanup(); + _activeTcs.TrySetResult(new ErrorResponse("Play mode exited during play_for_frames.", new + { + frames_requested = _framesRequested, + frames_elapsed = elapsed + })); + return; + } + + // Check timeout + if (DateTime.UtcNow >= _deadlineUtc) + { + int elapsed = _framesRequested - _framesRemaining; + EditorApplication.isPaused = true; + Cleanup(); + _activeTcs.TrySetResult(new ErrorResponse($"Timed out after {_timeoutSeconds}s.", new + { + frames_requested = _framesRequested, + frames_elapsed = elapsed + })); + return; + } + + // Count frame + _framesRemaining--; + + if (_framesRemaining <= 0) + { + int endFrame = Time.frameCount; + int elapsed = _framesRequested; + EditorApplication.isPaused = true; + Cleanup(); + _activeTcs.TrySetResult(new SuccessResponse($"Advanced {elapsed} frames and paused.", new + { + frames_requested = elapsed, + frames_elapsed = elapsed, + start_frame = _startFrame, + end_frame = endFrame + })); + } + } + + private static void CancelActive(bool silent) + { + if (_activeTick != null) + { + EditorApplication.update -= _activeTick; + _activeTick = null; + } + + EditorNudge.EndNudge(); + SessionState.EraseString(PendingKey); + + if (_activeTcs != null && !_activeTcs.Task.IsCompleted) + { + if (!silent) + { + _activeTcs.TrySetResult( + new ErrorResponse("Cancelled by new play_for_frames call.")); + } + else + { + _activeTcs.TrySetCanceled(); + } + } + + _activeTcs = null; + _activeOperationId = null; + } + + private static void Cleanup() + { + if (_activeTick != null) + { + EditorApplication.update -= _activeTick; + _activeTick = null; + } + EditorNudge.EndNudge(); + SessionState.EraseString(PendingKey); + } + } +} diff --git a/Editor/Tools/EditorControl/EditorPlayForFrames.cs.meta b/Editor/Tools/EditorControl/EditorPlayForFrames.cs.meta new file mode 100644 index 0000000..4db73e3 --- /dev/null +++ b/Editor/Tools/EditorControl/EditorPlayForFrames.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: f31a4f6fab66a4bb79fd32b6702711c6 \ No newline at end of file diff --git a/Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs b/Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs new file mode 100644 index 0000000..1b3edbc --- /dev/null +++ b/Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs @@ -0,0 +1,56 @@ +using System; +using NativeMcp.Editor.Helpers; +using NativeMcp.Editor.Tools; +using Newtonsoft.Json.Linq; +using UnityEngine; + +namespace NativeMcp.Editor.Tools.EditorControl +{ + [McpForUnityTool("editor_set_update_frequency", Internal = true, + Description = "Get or set game update frequency. Controls Time.timeScale and Time.captureFramerate. Call with no arguments to read current values.")] + public static class EditorSetUpdateFrequency + { + public class Parameters + { + [ToolParameter("Time scale multiplier (0=frozen, 1=normal). Range [0, 100].", Required = false)] + public float? time_scale { get; set; } + + [ToolParameter("Capture framerate for deterministic mode. When >0, deltaTime = timeScale/captureFramerate per frame regardless of real time. 0=off.", Required = false)] + public int? capture_framerate { get; set; } + } + + public static object HandleCommand(JObject @params) + { + try + { + float? timeScale = (float?)@params["time_scale"]; + int? captureFramerate = (int?)@params["capture_framerate"]; + + if (timeScale.HasValue) + { + if (timeScale.Value < 0f || timeScale.Value > 100f) + return new ErrorResponse("time_scale must be in range [0, 100]."); + Time.timeScale = timeScale.Value; + } + + if (captureFramerate.HasValue) + { + if (captureFramerate.Value < 0) + return new ErrorResponse("capture_framerate must be >= 0."); + Time.captureFramerate = captureFramerate.Value; + } + + return new SuccessResponse("Update frequency configured.", new + { + time_scale = Time.timeScale, + capture_framerate = Time.captureFramerate, + frame_count = Time.frameCount + }); + } + catch (Exception e) + { + return new ErrorResponse($"Error setting update frequency: {e.Message}"); + } + } + } +} diff --git a/Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs.meta b/Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs.meta new file mode 100644 index 0000000..83a2f19 --- /dev/null +++ b/Editor/Tools/EditorControl/EditorSetUpdateFrequency.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: d01a7a2483cb941eeb5b3d35a71f201a \ No newline at end of file diff --git a/Editor/Tools/EditorControl/EditorStepFrame.cs b/Editor/Tools/EditorControl/EditorStepFrame.cs new file mode 100644 index 0000000..a5b0c00 --- /dev/null +++ b/Editor/Tools/EditorControl/EditorStepFrame.cs @@ -0,0 +1,41 @@ +using System; +using NativeMcp.Editor.Helpers; +using NativeMcp.Editor.Tools; +using Newtonsoft.Json.Linq; +using UnityEditor; +using UnityEngine; + +namespace NativeMcp.Editor.Tools.EditorControl +{ + [McpForUnityTool("editor_step_frame", Internal = true, + Description = "Advance exactly one frame while paused in play mode.")] + public static class EditorStepFrame + { + public static object HandleCommand(JObject @params) + { + try + { + if (!EditorApplication.isPlaying) + { + if (EditorApplication.isPlayingOrWillChangePlaymode) + return new ErrorResponse("Play mode is transitioning. Wait and try again."); + return new ErrorResponse("Not in play mode. Call 'play' first."); + } + + if (!EditorApplication.isPaused) + EditorApplication.isPaused = true; + + EditorApplication.Step(); + + return new SuccessResponse("Stepped 1 frame.", new + { + frame = Time.frameCount + }); + } + catch (Exception e) + { + return new ErrorResponse($"Error stepping frame: {e.Message}"); + } + } + } +} diff --git a/Editor/Tools/EditorControl/EditorStepFrame.cs.meta b/Editor/Tools/EditorControl/EditorStepFrame.cs.meta new file mode 100644 index 0000000..992de00 --- /dev/null +++ b/Editor/Tools/EditorControl/EditorStepFrame.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 3d40dbecdb0ae4c7a9f865b48d189cab \ No newline at end of file diff --git a/Editor/Tools/Meta/UnityEditorControl.cs b/Editor/Tools/Meta/UnityEditorControl.cs index f13bb69..0d0511f 100644 --- a/Editor/Tools/Meta/UnityEditorControl.cs +++ b/Editor/Tools/Meta/UnityEditorControl.cs @@ -12,6 +12,9 @@ namespace NativeMcp.Editor.Tools.Meta "- play() — enter play mode\n" + "- stop() — exit play mode\n" + "- pause() — toggle pause in play mode\n" + + "- step_frame() — advance exactly one frame while paused in play mode\n" + + "- play_for_frames(frames, timeout?) — play for N frames then pause. Supports domain reload recovery\n" + + "- set_update_frequency(time_scale?, capture_framerate?) — get/set game update frequency. No args = getter\n" + "- add_tag(tagName) — add a project tag\n" + "- remove_tag(tagName) — remove a project tag\n" + "- add_layer(layerName) — add a project layer\n" + @@ -25,6 +28,9 @@ public static class UnityEditorControl ["play"] = "editor_play", ["stop"] = "editor_stop", ["pause"] = "editor_pause", + ["step_frame"] = "editor_step_frame", + ["play_for_frames"] = "editor_play_for_frames", + ["set_update_frequency"] = "editor_set_update_frequency", ["add_tag"] = "editor_add_tag", ["remove_tag"] = "editor_remove_tag", ["add_layer"] = "editor_add_layer", @@ -35,9 +41,21 @@ public static class UnityEditorControl public class Parameters { - [ToolParameter("Action to perform: play, stop, pause, add_tag, remove_tag, add_layer, remove_layer, set_active_tool, refresh")] + [ToolParameter("Action to perform: play, stop, pause, step_frame, play_for_frames, set_update_frequency, add_tag, remove_tag, add_layer, remove_layer, set_active_tool, refresh")] public string action { get; set; } + [ToolParameter("Number of frames to advance for play_for_frames (>= 1)", Required = false)] + public int? frames { get; set; } + + [ToolParameter("Timeout in seconds for play_for_frames (default 30)", Required = false)] + public int? timeout { get; set; } + + [ToolParameter("Time scale for set_update_frequency (0=frozen, 1=normal, range [0,100])", Required = false)] + public float? time_scale { get; set; } + + [ToolParameter("Capture framerate for deterministic mode (0=off). deltaTime = timeScale/captureFramerate", Required = false)] + public int? capture_framerate { get; set; } + [ToolParameter("Tag name for add_tag/remove_tag actions", Required = false)] public string tagName { get; set; } diff --git a/Tests/Editor/EditorControlToolTests.cs b/Tests/Editor/EditorControlToolTests.cs new file mode 100644 index 0000000..e091622 --- /dev/null +++ b/Tests/Editor/EditorControlToolTests.cs @@ -0,0 +1,124 @@ +using NativeMcp.Editor.Helpers; +using NativeMcp.Editor.Tools.EditorControl; +using Newtonsoft.Json.Linq; +using NUnit.Framework; + +namespace NativeMcp.Editor.Tests +{ + [TestFixture] + public class EditorControlToolTests + { + // --- EditorStepFrame --- + + [Test] + public void StepFrame_NotInPlayMode_ReturnsError() + { + var result = EditorStepFrame.HandleCommand(new JObject()); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("Not in play mode")); + } + + // --- EditorSetUpdateFrequency --- + + [Test] + public void SetUpdateFrequency_NoArgs_ReturnsCurrentValues() + { + var result = EditorSetUpdateFrequency.HandleCommand(new JObject()); + Assert.IsInstanceOf(result); + + var success = (SuccessResponse)result; + Assert.That(success.Message, Does.Contain("configured")); + Assert.IsNotNull(success.Data); + } + + [Test] + public void SetUpdateFrequency_TimeScaleNegative_ReturnsError() + { + var @params = new JObject { ["time_scale"] = -1f }; + var result = EditorSetUpdateFrequency.HandleCommand(@params); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("time_scale")); + } + + [Test] + public void SetUpdateFrequency_TimeScaleAboveMax_ReturnsError() + { + var @params = new JObject { ["time_scale"] = 101f }; + var result = EditorSetUpdateFrequency.HandleCommand(@params); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("time_scale")); + } + + [Test] + public void SetUpdateFrequency_CaptureFramerateNegative_ReturnsError() + { + var @params = new JObject { ["capture_framerate"] = -1 }; + var result = EditorSetUpdateFrequency.HandleCommand(@params); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("capture_framerate")); + } + + [Test] + public void SetUpdateFrequency_ValidTimeScale_AppliesAndReturns() + { + float original = UnityEngine.Time.timeScale; + try + { + var @params = new JObject { ["time_scale"] = 0.5f }; + var result = EditorSetUpdateFrequency.HandleCommand(@params); + Assert.IsInstanceOf(result); + Assert.AreEqual(0.5f, UnityEngine.Time.timeScale, 0.001f); + } + finally + { + UnityEngine.Time.timeScale = original; + } + } + + [Test] + public void SetUpdateFrequency_ValidCaptureFramerate_AppliesAndReturns() + { + int original = UnityEngine.Time.captureFramerate; + try + { + var @params = new JObject { ["capture_framerate"] = 10 }; + var result = EditorSetUpdateFrequency.HandleCommand(@params); + Assert.IsInstanceOf(result); + Assert.AreEqual(10, UnityEngine.Time.captureFramerate); + } + finally + { + UnityEngine.Time.captureFramerate = original; + } + } + + // --- EditorPlayForFrames --- + + [Test] + public void PlayForFrames_FramesZero_ReturnsError() + { + var @params = new JObject { ["frames"] = 0 }; + var result = EditorPlayForFrames.HandleCommand(@params).Result; + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("frames")); + } + + [Test] + public void PlayForFrames_FramesNegative_ReturnsError() + { + var @params = new JObject { ["frames"] = -5 }; + var result = EditorPlayForFrames.HandleCommand(@params).Result; + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("frames")); + } + + [Test] + public void PlayForFrames_NotInPlayMode_ReturnsError() + { + var @params = new JObject { ["frames"] = 10 }; + var result = EditorPlayForFrames.HandleCommand(@params).Result; + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("Not in play mode")); + } + } +} diff --git a/Tests/Editor/EditorControlToolTests.cs.meta b/Tests/Editor/EditorControlToolTests.cs.meta new file mode 100644 index 0000000..0ab83f6 --- /dev/null +++ b/Tests/Editor/EditorControlToolTests.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 36ee3a523fd874daf8d0f6a9b8baffd3 \ No newline at end of file diff --git a/Tests/Editor/RunTestsHelperTests.cs.meta b/Tests/Editor/RunTestsHelperTests.cs.meta new file mode 100644 index 0000000..f0e231e --- /dev/null +++ b/Tests/Editor/RunTestsHelperTests.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 08f0d159925114e53b8127a362cbf0bd \ No newline at end of file From e71fffa465e762ba82773e91a497976389f4143c Mon Sep 17 00:00:00 2001 From: Shino Mailer Date: Sat, 4 Apr 2026 14:29:10 +0800 Subject: [PATCH 3/5] feat: enhance invoke tool with instance targeting, DDOL visibility, serialization fixes, and member enumeration Address four key gaps identified in the MCP gameplay report: - D2: Add instance_id and game_object params to invoke for targeting specific component instances instead of always getting the first FindObjectOfType match - D3/D4: Add DontDestroyOnLoad fallback in ResolveInstance and expose DDOL roots in SceneGetHierarchy during play mode - D6: Use UnityJsonSerializer.Settings in WrapResult so Vector3/Quaternion/Color etc. serialize cleanly as {x,y,z} instead of including computed properties; expand CoerceArg to handle Vector4, Quaternion, Color, Rect, Bounds - D9: Support wildcard resolve_method queries (Type.* or Type.) to enumerate all public members of a type, enabling LLM self-discovery of available APIs Co-Authored-By: Claude Opus 4.6 (1M context) --- Editor/Bridge/UnityToolBridge.cs | 4 +- Editor/Helpers/GameObjectLookup.cs | 4 +- Editor/Helpers/UnityJsonSerializer.cs | 15 +- Editor/Tools/InvokeDynamic.cs | 223 +++++++++++++++++++- Editor/Tools/Meta/UnityInvoke.cs | 11 +- Editor/Tools/Scene/SceneGetHierarchy.cs | 12 ++ Tests/Editor/InvokeDynamicTests.cs | 257 ++++++++++++++++++++++++ Tests/Editor/InvokeDynamicTests.cs.meta | 2 + 8 files changed, 511 insertions(+), 17 deletions(-) create mode 100644 Tests/Editor/InvokeDynamicTests.cs create mode 100644 Tests/Editor/InvokeDynamicTests.cs.meta diff --git a/Editor/Bridge/UnityToolBridge.cs b/Editor/Bridge/UnityToolBridge.cs index 4be887f..c93ba9d 100644 --- a/Editor/Bridge/UnityToolBridge.cs +++ b/Editor/Bridge/UnityToolBridge.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using NativeMcp.Editor.Helpers; using NativeMcp.Editor.Services; using NativeMcp.Editor.Tools; using NativeMcp.Editor.Protocol; @@ -251,7 +252,8 @@ private static McpToolCallResult WrapResult(object rawResult) // Serialize the result object to JSON try { - text = JsonConvert.SerializeObject(rawResult, Formatting.Indented); + text = JsonConvert.SerializeObject(rawResult, Formatting.Indented, + UnityJsonSerializer.Settings); // Check if the result contains an error status if (rawResult is JObject jObj) diff --git a/Editor/Helpers/GameObjectLookup.cs b/Editor/Helpers/GameObjectLookup.cs index b46bea8..d7e6a23 100644 --- a/Editor/Helpers/GameObjectLookup.cs +++ b/Editor/Helpers/GameObjectLookup.cs @@ -335,7 +335,7 @@ public static IEnumerable GetAllSceneObjects(bool includeInactive) /// Gets all GameObjects in the DontDestroyOnLoad scene. /// Uses a temporary helper object to discover the hidden scene. /// - private static IEnumerable GetDontDestroyOnLoadObjects(bool includeInactive) + internal static IEnumerable GetDontDestroyOnLoadObjects(bool includeInactive) { if (!TryGetDontDestroyOnLoadScene(out var ddolScene)) yield break; @@ -353,7 +353,7 @@ private static IEnumerable GetDontDestroyOnLoadObjects(bool includeI } } - private static bool TryGetDontDestroyOnLoadScene(out Scene scene) + internal static bool TryGetDontDestroyOnLoadScene(out Scene scene) { scene = default; diff --git a/Editor/Helpers/UnityJsonSerializer.cs b/Editor/Helpers/UnityJsonSerializer.cs index 28004c6..be63c9a 100644 --- a/Editor/Helpers/UnityJsonSerializer.cs +++ b/Editor/Helpers/UnityJsonSerializer.cs @@ -11,10 +11,10 @@ namespace NativeMcp.Editor.Helpers public static class UnityJsonSerializer { /// - /// Shared JsonSerializer instance with converters for Unity types. - /// Use this for all JToken-to-Unity-type conversions. + /// Shared serializer settings with converters for Unity types. + /// Use with JsonConvert.SerializeObject for consistent Unity type serialization. /// - public static readonly JsonSerializer Instance = JsonSerializer.Create(new JsonSerializerSettings + public static readonly JsonSerializerSettings Settings = new JsonSerializerSettings { Converters = new List { @@ -25,9 +25,16 @@ public static class UnityJsonSerializer new ColorConverter(), new RectConverter(), new BoundsConverter(), + new Matrix4x4Converter(), new UnityEngineObjectConverter() } - }); + }; + + /// + /// Shared JsonSerializer instance with converters for Unity types. + /// Use this for all JToken-to-Unity-type conversions. + /// + public static readonly JsonSerializer Instance = JsonSerializer.Create(Settings); } } diff --git a/Editor/Tools/InvokeDynamic.cs b/Editor/Tools/InvokeDynamic.cs index 3a410f7..d487c4f 100644 --- a/Editor/Tools/InvokeDynamic.cs +++ b/Editor/Tools/InvokeDynamic.cs @@ -5,6 +5,7 @@ using NativeMcp.Editor.Helpers; using NativeMcp.Runtime; using Newtonsoft.Json.Linq; +using UnityEditor; using UnityEngine; namespace NativeMcp.Editor.Tools @@ -21,10 +22,11 @@ namespace NativeMcp.Editor.Tools /// [McpForUnityTool("invoke_dynamic", Internal = true, Description = "Reflect-invoke any method or access any property. Two-step workflow: " + - "1) action='resolve_method' with method='Type.Member' to inspect candidates. " + + "1) action='resolve_method' with method='Type.Member' to inspect candidates (use 'Type.*' to list all). " + "2) action='call_method' with method='Type.ExactName' to execute. " + "Supports static/instance methods, properties. " + "Instance methods on MonoBehaviours are auto-located in the scene. " + + "Use instance_id or game_object to target a specific instance when multiple exist. " + "Also supports action='list'/'call'/'describe' for pre-registered dynamic tools.")] public static class InvokeDynamic { @@ -53,6 +55,14 @@ public class Parameters [ToolParameter("JSON object of arguments. Keys mapped to parameter names.", Required = false)] public string args { get; set; } + + [ToolParameter("Instance ID of the target object (from get_scene_tree or get_hierarchy). " + + "Targets a specific instance when multiple exist.", Required = false)] + public int? instance_id { get; set; } + + [ToolParameter("GameObject name or hierarchy path to find the target instance on. " + + "Alternative to instance_id.", Required = false)] + public string game_object { get; set; } } public static object HandleCommand(JObject @params) @@ -112,6 +122,10 @@ private static object HandleResolveMethod(JObject @params) if (targetType == null) return new ErrorResponse($"Type '{typePart}' not found. Try full name like 'Namespace.ClassName'."); + // Wildcard: "Type.*" or "Type." lists all members + if (memberName == "*" || memberName == "") + return HandleListAllMembers(targetType); + var candidates = new List(); // Properties (case-insensitive) @@ -162,6 +176,66 @@ private static object HandleResolveMethod(JObject @params) new { type = targetType.FullName, query = memberName, candidates }); } + // ──────────────────────────────────────────────────────────── + // list all members — wildcard resolve + // ──────────────────────────────────────────────────────────── + + private static object HandleListAllMembers(Type targetType) + { + const int maxProperties = 50; + const int maxMethods = 50; + + var properties = new List(); + bool propsTruncated = false; + + foreach (var p in GetPropertiesCached(targetType)) + { + if (properties.Count >= maxProperties) { propsTruncated = true; break; } + properties.Add(new + { + kind = "property", + name = p.Name, + type = p.PropertyType.Name, + is_static = p.GetGetMethod(true)?.IsStatic ?? p.GetSetMethod(true)?.IsStatic ?? false, + can_read = p.CanRead, + can_write = p.CanWrite, + call_as = $"{targetType.Name}.{p.Name}" + }); + } + + var methods = new List(); + bool methodsTruncated = false; + var seen = new HashSet(StringComparer.Ordinal); + + foreach (var m in GetMethodsCached(targetType).Where(m => !m.IsSpecialName)) + { + if (!seen.Add(m.Name)) continue; // skip overloads (use exact resolve to see all) + if (methods.Count >= maxMethods) { methodsTruncated = true; break; } + methods.Add(new + { + kind = "method", + name = m.Name, + signature = $"{m.Name}({string.Join(", ", m.GetParameters().Select(p => $"{p.ParameterType.Name} {p.Name}"))})", + return_type = m.ReturnType.Name, + is_static = m.IsStatic, + call_as = $"{targetType.Name}.{m.Name}" + }); + } + + bool truncated = propsTruncated || methodsTruncated; + return new SuccessResponse( + $"{targetType.Name}: {properties.Count} properties, {methods.Count} methods" + + (truncated ? " (truncated, use 'Type.Name' for specific member)" : "") + + ". Use action='call_method' with the exact call_as value to execute.", + new + { + type = targetType.FullName, + properties, + methods, + truncated + }); + } + // ──────────────────────────────────────────────────────────── // call_method — exact match, execute directly // ──────────────────────────────────────────────────────────── @@ -187,6 +261,11 @@ private static object HandleCallMethod(JObject @params) if (argDict == null) return new ErrorResponse("Failed to parse 'args'."); + // Extract instance targeting params (separate from method args) + int? instanceId = @params["instance_id"]?.Type == JTokenType.Integer + ? @params["instance_id"].Value() : null; + string gameObjectName = @params["game_object"]?.ToString(); + // 1) Try exact property match var prop = GetPropertiesCached(targetType).FirstOrDefault(p => string.Equals(p.Name, memberName, StringComparison.Ordinal)); @@ -197,7 +276,7 @@ private static object HandleCallMethod(JObject @params) : prop.GetGetMethod(true); if (accessor == null) return new ErrorResponse($"Property '{memberName}' has no {(argDict.Count > 0 ? "setter" : "getter")}."); - return InvokeMethod(accessor, targetType, argDict); + return InvokeMethod(accessor, targetType, argDict, instanceId, gameObjectName); } // 2) Try exact method match @@ -213,7 +292,7 @@ private static object HandleCallMethod(JObject @params) return new ErrorResponse($"No matching overload. Candidates:\n{string.Join("\n", sigs)}"); } - return InvokeMethod(method, targetType, argDict); + return InvokeMethod(method, targetType, argDict, instanceId, gameObjectName); } // ──────────────────────────────────────────────────────────── @@ -531,15 +610,73 @@ private static PropertyInfo[] GetPropertiesCached(Type type) } /// - /// Find a live instance of the given type in the scene. + /// Find a live instance with explicit targeting (instance_id or game_object name). + /// Falls back to auto-discovery when neither is specified. /// - private static object ResolveInstance(Type type) + private static object ResolveInstance(Type type, int? instanceId, string gameObjectName) { - if (typeof(UnityEngine.Object).IsAssignableFrom(type)) + // Path 1: Exact lookup by instance ID + if (instanceId.HasValue) + { + var obj = EditorUtility.InstanceIDToObject(instanceId.Value); + if (obj == null) return null; + if (type.IsInstanceOfType(obj)) return obj; + if (obj is GameObject go && typeof(Component).IsAssignableFrom(type)) + return go.GetComponent(type); + if (obj is Component comp && typeof(Component).IsAssignableFrom(type)) + return comp.gameObject.GetComponent(type); + return null; + } + + // Path 2: Find by GameObject name or hierarchy path + if (!string.IsNullOrEmpty(gameObjectName)) { + var go = GameObject.Find(gameObjectName); + if (go == null) + { + var ids = GameObjectLookup.SearchGameObjects("by_name", gameObjectName, true, 1); + int firstId = ids.FirstOrDefault(); + if (firstId != 0) + go = EditorUtility.InstanceIDToObject(firstId) as GameObject; + } + if (go == null) return null; + if (type == typeof(GameObject) || type.IsAssignableFrom(typeof(GameObject))) + return go; + if (typeof(Component).IsAssignableFrom(type)) + return go.GetComponent(type); + return null; + } + + // Path 3: Auto-discover (original behavior) + return ResolveInstance(type); + } + + /// + /// Find a live instance of the given type in the scene (auto-discovery fallback). + /// Searches DontDestroyOnLoad objects when FindObjectOfType returns null in play mode. + /// + private static object ResolveInstance(Type type) + { + if (!typeof(UnityEngine.Object).IsAssignableFrom(type)) + return null; + #pragma warning disable CS0618 - return UnityEngine.Object.FindObjectOfType(type); + var found = UnityEngine.Object.FindObjectOfType(type); #pragma warning restore CS0618 + if (found != null) return found; + + // FindObjectOfType does not search the DontDestroyOnLoad scene + if (!Application.isPlaying) return null; + + foreach (var go in GameObjectLookup.GetDontDestroyOnLoadObjects(true)) + { + if (type.IsAssignableFrom(typeof(GameObject))) + return go; + if (typeof(Component).IsAssignableFrom(type)) + { + var comp = go.GetComponent(type); + if (comp != null) return comp; + } } return null; } @@ -595,6 +732,63 @@ private static object CoerceArg(object value, Type target) Convert.ToSingle(v2.GetValueOrDefault("y", 0f))); } + // Vector4 + if (target == typeof(Vector4) && value is Dictionary v4) + { + return new Vector4( + Convert.ToSingle(v4.GetValueOrDefault("x", 0f)), + Convert.ToSingle(v4.GetValueOrDefault("y", 0f)), + Convert.ToSingle(v4.GetValueOrDefault("z", 0f)), + Convert.ToSingle(v4.GetValueOrDefault("w", 0f))); + } + + // Quaternion + if (target == typeof(Quaternion) && value is Dictionary qd) + { + return new Quaternion( + Convert.ToSingle(qd.GetValueOrDefault("x", 0f)), + Convert.ToSingle(qd.GetValueOrDefault("y", 0f)), + Convert.ToSingle(qd.GetValueOrDefault("z", 0f)), + Convert.ToSingle(qd.GetValueOrDefault("w", 0f))); + } + + // Color + if (target == typeof(Color) && value is Dictionary cd) + { + return new Color( + Convert.ToSingle(cd.GetValueOrDefault("r", 0f)), + Convert.ToSingle(cd.GetValueOrDefault("g", 0f)), + Convert.ToSingle(cd.GetValueOrDefault("b", 0f)), + Convert.ToSingle(cd.GetValueOrDefault("a", 1f))); + } + + // Rect + if (target == typeof(Rect) && value is Dictionary rd) + { + return new Rect( + Convert.ToSingle(rd.GetValueOrDefault("x", 0f)), + Convert.ToSingle(rd.GetValueOrDefault("y", 0f)), + Convert.ToSingle(rd.GetValueOrDefault("width", 0f)), + Convert.ToSingle(rd.GetValueOrDefault("height", 0f))); + } + + // Bounds + if (target == typeof(Bounds) && value is Dictionary bd) + { + Vector3 center = default, size = default; + if (bd.TryGetValue("center", out var cv) && cv is Dictionary cd2) + center = new Vector3( + Convert.ToSingle(cd2.GetValueOrDefault("x", 0f)), + Convert.ToSingle(cd2.GetValueOrDefault("y", 0f)), + Convert.ToSingle(cd2.GetValueOrDefault("z", 0f))); + if (bd.TryGetValue("size", out var sv) && sv is Dictionary sd) + size = new Vector3( + Convert.ToSingle(sd.GetValueOrDefault("x", 0f)), + Convert.ToSingle(sd.GetValueOrDefault("y", 0f)), + Convert.ToSingle(sd.GetValueOrDefault("z", 0f))); + return new Bounds(center, size); + } + // Unwrap Nullable → T so Convert.ChangeType works (it cannot convert to Nullable directly) Type underlying = Nullable.GetUnderlyingType(target); if (underlying != null) @@ -639,7 +833,8 @@ private static Dictionary ParseArgsFromParams(JObject @params) /// /// Shared invoke: build args, resolve instance, invoke, return result. /// - private static object InvokeMethod(MethodInfo method, Type targetType, Dictionary argDict) + private static object InvokeMethod(MethodInfo method, Type targetType, + Dictionary argDict, int? instanceId = null, string gameObjectName = null) { var methodParams = method.GetParameters(); object[] invokeArgs = new object[methodParams.Length]; @@ -662,9 +857,19 @@ private static object InvokeMethod(MethodInfo method, Type targetType, Dictionar object target = null; if (!method.IsStatic) { - target = ResolveInstance(targetType); + target = ResolveInstance(targetType, instanceId, gameObjectName); if (target == null) + { + if (instanceId.HasValue) + return new ErrorResponse( + $"Object with instanceID {instanceId.Value} not found or does not have " + + $"component '{targetType.Name}'. IDs may be stale after domain reload."); + if (!string.IsNullOrEmpty(gameObjectName)) + return new ErrorResponse( + $"GameObject '{gameObjectName}' not found or does not have " + + $"component '{targetType.Name}'."); return new ErrorResponse($"No instance of '{targetType.FullName}' found in scene."); + } } try diff --git a/Editor/Tools/Meta/UnityInvoke.cs b/Editor/Tools/Meta/UnityInvoke.cs index 2f2d540..9542534 100644 --- a/Editor/Tools/Meta/UnityInvoke.cs +++ b/Editor/Tools/Meta/UnityInvoke.cs @@ -8,9 +8,10 @@ namespace NativeMcp.Editor.Tools.Meta [McpForUnityTool("unity_invoke", Description = "Reflect-invoke any C# method or access any property in Unity. Two-step workflow:\n" + - "1) action='resolve_method' with method='Type.Member' to inspect candidates.\n" + + "1) action='resolve_method' with method='Type.Member' to inspect candidates (use 'Type.*' to list all members).\n" + "2) action='call_method' with method='Type.ExactName' and args={...} to execute.\n" + "Supports static/instance methods and properties. Instance methods on MonoBehaviours are auto-located in the scene.\n" + + "Use instance_id or game_object to target a specific instance when multiple exist.\n" + "Also supports action='list'/'call'/'describe' for pre-registered dynamic tools.")] public static class UnityInvoke { @@ -32,6 +33,14 @@ public class Parameters [ToolParameter("JSON object of arguments. Keys mapped to parameter names.", Required = false)] public object args { get; set; } + + [ToolParameter("Instance ID of the target object (from get_scene_tree or get_hierarchy). " + + "Targets a specific instance when multiple exist.", Required = false)] + public int? instance_id { get; set; } + + [ToolParameter("GameObject name or hierarchy path to find the target instance on. " + + "Alternative to instance_id.", Required = false)] + public string game_object { get; set; } } public static object HandleCommand(JObject @params) diff --git a/Editor/Tools/Scene/SceneGetHierarchy.cs b/Editor/Tools/Scene/SceneGetHierarchy.cs index 2fad717..2ac6046 100644 --- a/Editor/Tools/Scene/SceneGetHierarchy.cs +++ b/Editor/Tools/Scene/SceneGetHierarchy.cs @@ -5,6 +5,7 @@ using Newtonsoft.Json.Linq; using UnityEditor.SceneManagement; using UnityEngine; +using UnityEngine.SceneManagement; namespace NativeMcp.Editor.Tools.Scene { @@ -84,6 +85,17 @@ public static object HandleCommand(JObject @params) { nodes = activeScene.GetRootGameObjects().Where(go => go != null).ToList(); scope = "roots"; + + // Include DontDestroyOnLoad root objects in Play Mode + if (Application.isPlaying + && GameObjectLookup.TryGetDontDestroyOnLoadScene(out var ddolScene)) + { + foreach (var root in ddolScene.GetRootGameObjects()) + { + if (root != null && root.name != "__MCP_DDOL_Probe__") + nodes.Add(root); + } + } } else { diff --git a/Tests/Editor/InvokeDynamicTests.cs b/Tests/Editor/InvokeDynamicTests.cs new file mode 100644 index 0000000..c94357e --- /dev/null +++ b/Tests/Editor/InvokeDynamicTests.cs @@ -0,0 +1,257 @@ +using System.Collections.Generic; +using NativeMcp.Editor.Helpers; +using NativeMcp.Editor.Tools; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NUnit.Framework; +using UnityEngine; + +namespace NativeMcp.Editor.Tests +{ + [TestFixture] + public class InvokeDynamicTests + { + // ── D9: Wildcard member enumeration ── + + [Test] + public void ResolveMethod_Wildcard_ReturnsPropertiesAndMethods() + { + var result = InvokeDynamic.HandleCommand(new JObject + { + ["action"] = "resolve_method", + ["method"] = "Transform.*" + }); + + Assert.IsInstanceOf(result); + var success = (SuccessResponse)result; + Assert.That(success.Message, Does.Contain("properties")); + Assert.That(success.Message, Does.Contain("methods")); + + var data = JObject.FromObject(success.Data); + Assert.That(data["properties"], Is.Not.Null); + Assert.That(data["methods"], Is.Not.Null); + Assert.That(data["properties"].HasValues, Is.True, "Should have properties"); + Assert.That(data["methods"].HasValues, Is.True, "Should have methods"); + } + + [Test] + public void ResolveMethod_DotOnly_ReturnsAllMembers() + { + var result = InvokeDynamic.HandleCommand(new JObject + { + ["action"] = "resolve_method", + ["method"] = "Transform." + }); + + Assert.IsInstanceOf(result); + var data = JObject.FromObject(((SuccessResponse)result).Data); + Assert.That(data["properties"].HasValues, Is.True); + Assert.That(data["methods"].HasValues, Is.True); + } + + [Test] + public void ResolveMethod_Wildcard_ExcludesSpecialNames() + { + var result = InvokeDynamic.HandleCommand(new JObject + { + ["action"] = "resolve_method", + ["method"] = "Transform.*" + }); + + Assert.IsInstanceOf(result); + var data = JObject.FromObject(((SuccessResponse)result).Data); + var methods = data["methods"] as JArray; + Assert.That(methods, Is.Not.Null); + + foreach (var m in methods) + { + string name = m["name"]?.ToString(); + Assert.That(name, Does.Not.StartWith("get_"), + $"Special name '{name}' should be excluded"); + Assert.That(name, Does.Not.StartWith("set_"), + $"Special name '{name}' should be excluded"); + Assert.That(name, Does.Not.StartWith("op_"), + $"Special name '{name}' should be excluded"); + } + } + + [Test] + public void ResolveMethod_SpecificMember_StillWorks() + { + var result = InvokeDynamic.HandleCommand(new JObject + { + ["action"] = "resolve_method", + ["method"] = "Transform.position" + }); + + Assert.IsInstanceOf(result); + var success = (SuccessResponse)result; + Assert.That(success.Message, Does.Contain("candidate")); + } + + // ── D2: Instance targeting ── + + [Test] + public void CallMethod_InvalidInstanceId_ReturnsError() + { + var result = InvokeDynamic.HandleCommand(new JObject + { + ["action"] = "call_method", + ["method"] = "Transform.position", + ["instance_id"] = 999999 + }); + + Assert.IsInstanceOf(result); + var error = (ErrorResponse)result; + Assert.That(error.Error, Does.Contain("instanceID")); + } + + [Test] + public void CallMethod_InvalidGameObject_ReturnsError() + { + var result = InvokeDynamic.HandleCommand(new JObject + { + ["action"] = "call_method", + ["method"] = "Transform.position", + ["game_object"] = "__NonExistentObject_12345__" + }); + + Assert.IsInstanceOf(result); + var error = (ErrorResponse)result; + Assert.That(error.Error, Does.Contain("__NonExistentObject_12345__")); + } + + [Test] + public void CallMethod_WithInstanceId_FindsCorrectObject() + { + var go = new GameObject("InvokeDynamic_Test_InstanceId"); + try + { + int id = go.GetInstanceID(); + var result = InvokeDynamic.HandleCommand(new JObject + { + ["action"] = "call_method", + ["method"] = "Transform.position", + ["instance_id"] = id + }); + + Assert.IsInstanceOf(result); + } + finally + { + Object.DestroyImmediate(go); + } + } + + [Test] + public void CallMethod_WithGameObject_FindsByName() + { + var go = new GameObject("InvokeDynamic_Test_ByName"); + try + { + var result = InvokeDynamic.HandleCommand(new JObject + { + ["action"] = "call_method", + ["method"] = "Transform.position", + ["game_object"] = "InvokeDynamic_Test_ByName" + }); + + Assert.IsInstanceOf(result); + } + finally + { + Object.DestroyImmediate(go); + } + } + + [Test] + public void CallMethod_StaticMethod_IgnoresInstanceParams() + { + // Static methods should work regardless of instance_id/game_object + var result = InvokeDynamic.HandleCommand(new JObject + { + ["action"] = "call_method", + ["method"] = "Time.frameCount", + ["instance_id"] = 999999 + }); + + Assert.IsInstanceOf(result); + } + + // ── D6: Serialization ── + + [Test] + public void Settings_Vector3_SerializesClean() + { + var v = new Vector3(1f, 2f, 3f); + string json = JsonConvert.SerializeObject(v, Formatting.None, UnityJsonSerializer.Settings); + var obj = JObject.Parse(json); + + Assert.AreEqual(1f, obj["x"].Value()); + Assert.AreEqual(2f, obj["y"].Value()); + Assert.AreEqual(3f, obj["z"].Value()); + // Should NOT contain computed properties like magnitude + Assert.That(obj["magnitude"], Is.Null, "Should not serialize computed properties"); + } + + [Test] + public void Settings_Quaternion_SerializesClean() + { + var q = new Quaternion(0f, 0.707f, 0f, 0.707f); + string json = JsonConvert.SerializeObject(q, Formatting.None, UnityJsonSerializer.Settings); + var obj = JObject.Parse(json); + + Assert.AreEqual(4, obj.Count, "Should have exactly x, y, z, w"); + Assert.That(obj["x"], Is.Not.Null); + Assert.That(obj["w"], Is.Not.Null); + Assert.That(obj["eulerAngles"], Is.Null, "Should not serialize computed properties"); + } + + [Test] + public void Settings_Color_SerializesClean() + { + var c = new Color(0.5f, 0.6f, 0.7f, 1f); + string json = JsonConvert.SerializeObject(c, Formatting.None, UnityJsonSerializer.Settings); + var obj = JObject.Parse(json); + + Assert.AreEqual(0.5f, obj["r"].Value(), 0.001f); + Assert.AreEqual(0.6f, obj["g"].Value(), 0.001f); + Assert.AreEqual(0.7f, obj["b"].Value(), 0.001f); + Assert.AreEqual(1f, obj["a"].Value(), 0.001f); + } + + [Test] + public void Settings_UnityObject_SerializesNameAndId() + { + var go = new GameObject("InvokeDynamic_Test_Serialize"); + try + { + string json = JsonConvert.SerializeObject(go, Formatting.None, UnityJsonSerializer.Settings); + var obj = JObject.Parse(json); + + Assert.That(obj["name"]?.ToString(), Is.EqualTo("InvokeDynamic_Test_Serialize")); + Assert.That(obj["instanceID"]?.Value(), Is.EqualTo(go.GetInstanceID())); + // Should NOT contain the full GameObject dump + Assert.That(obj["transform"], Is.Null); + } + finally + { + Object.DestroyImmediate(go); + } + } + + [Test] + public void Settings_Matrix4x4_SerializesRawElements() + { + var m = Matrix4x4.identity; + string json = JsonConvert.SerializeObject(m, Formatting.None, UnityJsonSerializer.Settings); + var obj = JObject.Parse(json); + + Assert.AreEqual(1f, obj["m00"].Value()); + Assert.AreEqual(0f, obj["m01"].Value()); + Assert.AreEqual(1f, obj["m11"].Value()); + // Should not contain computed properties like inverse, determinant + Assert.That(obj["inverse"], Is.Null); + } + } +} diff --git a/Tests/Editor/InvokeDynamicTests.cs.meta b/Tests/Editor/InvokeDynamicTests.cs.meta new file mode 100644 index 0000000..0f50836 --- /dev/null +++ b/Tests/Editor/InvokeDynamicTests.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 022f88e6b058548778244e4c5b20bacc \ No newline at end of file From 6ae1b75a758350a2d2e0b99a8cfbee8ca2f5ac3f Mon Sep 17 00:00:00 2001 From: Shino Mailer Date: Sun, 5 Apr 2026 02:35:36 +0800 Subject: [PATCH 4/5] feat: add input simulation, screenshot improvements, and menu item execution - Add keyboard/mouse input simulation via InputSystem low-level events - Hook NativeInputRuntime.onUpdate to prevent hardware state overwrite - Track held keys/buttons and re-inject after each native update - Auto Y-flip mouse coordinates from screenshot space to screen space - Focus Game View on mouse actions to fix Screen.width/height for UGUI raycasting - Fix screenshot Y-flip for window capture mode (ReadPixels from RenderTexture) - Add multi-mode screenshot capture (auto/game_view/window/camera) with fallback chain - Add execute_menu_item action to unity_editor tool - Improve search method error messages in GameObjectLookup - Add NATIVE_MCP_HAS_INPUT_SYSTEM version define for InputSystem detection Co-Authored-By: Claude Opus 4.6 (1M context) --- Editor/Helpers/GameObjectLookup.cs | 4 +- Editor/NativeMcp.Editor.asmdef | 8 +- Editor/Tools/EditorControl/ExecuteMenuItem.cs | 42 + .../EditorControl/ExecuteMenuItem.cs.meta | 2 + Editor/Tools/Input.meta | 8 + Editor/Tools/Input/SimulateInput.cs | 768 ++++++++++++++++++ Editor/Tools/Input/SimulateInput.cs.meta | 2 + Editor/Tools/ManageScene.cs | 8 +- Editor/Tools/Meta/UnityEditorControl.cs | 10 +- Editor/Tools/Meta/UnityInput.cs | 70 ++ Editor/Tools/Meta/UnityInput.cs.meta | 2 + Editor/Tools/Meta/UnityScene.cs | 5 +- Editor/Tools/Scene/SceneScreenshot.cs | 444 ++++++++-- Tests/Editor/InputToolTests.cs | 184 +++++ Tests/Editor/InputToolTests.cs.meta | 2 + Tests/Editor/NativeMcp.Editor.Tests.asmdef | 8 +- 16 files changed, 1472 insertions(+), 95 deletions(-) create mode 100644 Editor/Tools/EditorControl/ExecuteMenuItem.cs create mode 100644 Editor/Tools/EditorControl/ExecuteMenuItem.cs.meta create mode 100644 Editor/Tools/Input.meta create mode 100644 Editor/Tools/Input/SimulateInput.cs create mode 100644 Editor/Tools/Input/SimulateInput.cs.meta create mode 100644 Editor/Tools/Meta/UnityInput.cs create mode 100644 Editor/Tools/Meta/UnityInput.cs.meta create mode 100644 Tests/Editor/InputToolTests.cs create mode 100644 Tests/Editor/InputToolTests.cs.meta diff --git a/Editor/Helpers/GameObjectLookup.cs b/Editor/Helpers/GameObjectLookup.cs index d7e6a23..cc3dc99 100644 --- a/Editor/Helpers/GameObjectLookup.cs +++ b/Editor/Helpers/GameObjectLookup.cs @@ -52,7 +52,9 @@ public static SearchMethod ParseSearchMethod(string method) "by_component" => SearchMethod.ByComponent, "by_path" => SearchMethod.ByPath, "by_id" => SearchMethod.ById, - _ => SearchMethod.ByName + _ => throw new ArgumentException( + $"Unknown search method '{method}'. " + + "Valid: by_name, by_tag, by_layer, by_component, by_path, by_id.") }; } diff --git a/Editor/NativeMcp.Editor.asmdef b/Editor/NativeMcp.Editor.asmdef index bc1b4c2..de0d57f 100644 --- a/Editor/NativeMcp.Editor.asmdef +++ b/Editor/NativeMcp.Editor.asmdef @@ -16,6 +16,12 @@ "precompiledReferences": [], "autoReferenced": true, "defineConstraints": [], - "versionDefines": [], + "versionDefines": [ + { + "name": "com.unity.inputsystem", + "expression": "1.0.0", + "define": "NATIVE_MCP_HAS_INPUT_SYSTEM" + } + ], "noEngineReferences": false } diff --git a/Editor/Tools/EditorControl/ExecuteMenuItem.cs b/Editor/Tools/EditorControl/ExecuteMenuItem.cs new file mode 100644 index 0000000..7ecca22 --- /dev/null +++ b/Editor/Tools/EditorControl/ExecuteMenuItem.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using NativeMcp.Editor.Helpers; +using Newtonsoft.Json.Linq; +using UnityEditor; + +namespace NativeMcp.Editor.Tools.EditorControl +{ + [McpForUnityTool("execute_menu_item", Internal = true, + Description = "Execute a Unity Editor menu item by path (e.g. 'GameObject/3D Object/Cube').")] + public static class ExecuteMenuItem + { + private static readonly HashSet Blacklist = new(StringComparer.OrdinalIgnoreCase) + { + "File/Quit", + "File/Exit", + }; + + public class Parameters + { + [ToolParameter("Menu item path, e.g. 'GameObject/3D Object/Cube' or 'Window/General/Console'")] + public string menu_path { get; set; } + } + + public static object HandleCommand(JObject @params) + { + string menuPath = @params["menu_path"]?.ToString(); + if (string.IsNullOrWhiteSpace(menuPath)) + return new ErrorResponse("Required parameter 'menu_path' is missing."); + + if (Blacklist.Contains(menuPath)) + return new ErrorResponse($"Menu item '{menuPath}' is blocked for safety."); + + bool ok = EditorApplication.ExecuteMenuItem(menuPath); + if (!ok) + return new ErrorResponse( + $"Failed to execute '{menuPath}'. It may be invalid, disabled, or context-dependent."); + + return new SuccessResponse($"Executed menu item: '{menuPath}'."); + } + } +} diff --git a/Editor/Tools/EditorControl/ExecuteMenuItem.cs.meta b/Editor/Tools/EditorControl/ExecuteMenuItem.cs.meta new file mode 100644 index 0000000..201c07b --- /dev/null +++ b/Editor/Tools/EditorControl/ExecuteMenuItem.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 811eebdeaf38d4ec5839fd32a827b9d0 \ No newline at end of file diff --git a/Editor/Tools/Input.meta b/Editor/Tools/Input.meta new file mode 100644 index 0000000..1e85e7f --- /dev/null +++ b/Editor/Tools/Input.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: a5a3168c37f064dffa58b751057ad03d +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Tools/Input/SimulateInput.cs b/Editor/Tools/Input/SimulateInput.cs new file mode 100644 index 0000000..766dd2f --- /dev/null +++ b/Editor/Tools/Input/SimulateInput.cs @@ -0,0 +1,768 @@ +#if NATIVE_MCP_HAS_INPUT_SYSTEM +using System; +using System.Collections.Generic; +using System.Reflection; +using NativeMcp.Editor.Helpers; +using Newtonsoft.Json.Linq; +using UnityEditor; +using UnityEngine; +using UnityEngine.InputSystem; +using UnityEngine.InputSystem.Controls; +using UnityEngine.InputSystem.LowLevel; + +namespace NativeMcp.Editor.Tools.Input +{ + [McpForUnityTool("simulate_input", Internal = true, + Description = "Simulate keyboard and mouse input in Play Mode via InputSystem low-level events.")] + public static class SimulateInput + { + // ── Native input interception ───────────────────────────────── + // The real problem: each frame, NativeInputRuntime delivers hardware + // events that overwrite any state we injected via InputState.Change. + // Solution: hook NativeInputRuntime.instance.onUpdate via reflection + // and discard native hardware events while MCP has synthetic input active. + // This is the same technique Unity's own InputTestFixture uses. + + private static readonly HashSet s_heldKeys = new HashSet(); + private static readonly HashSet s_heldMouseButtons = new HashSet(); + private static bool s_nativeHookInstalled; + private static Delegate s_originalOnUpdate; + private static PropertyInfo s_onUpdateProp; + private static object s_nativeRuntimeInstance; + + private static bool HasSyntheticInput => s_heldKeys.Count > 0 || s_heldMouseButtons.Count > 0; + + /// + /// Hook into NativeInputRuntime.instance.onUpdate to intercept and + /// discard hardware input events while MCP synthetic input is active. + /// Uses the onUpdate property setter which wraps our delegate into the + /// native callback chain. Our delegate discards hardware events when + /// synthetic input is held, then re-injects our state after the update. + /// + private static void EnsureNativeInputHook() + { + if (s_nativeHookInstalled) return; + + try + { + // Resolve NativeInputRuntime.instance via reflection (it's internal) + var isAssembly = typeof(InputSystem).Assembly; + var nativeRuntimeType = isAssembly.GetType( + "UnityEngine.InputSystem.LowLevel.NativeInputRuntime"); + if (nativeRuntimeType == null) + { + Debug.LogWarning("[SimulateInput] Cannot find NativeInputRuntime type."); + return; + } + + var instanceField = nativeRuntimeType.GetField("instance", + BindingFlags.Public | BindingFlags.Static); + s_nativeRuntimeInstance = instanceField?.GetValue(null); + if (s_nativeRuntimeInstance == null) + { + Debug.LogWarning("[SimulateInput] NativeInputRuntime.instance is null."); + return; + } + + // Get the onUpdate property — its type is the internal InputUpdateDelegate + s_onUpdateProp = nativeRuntimeType.GetProperty("onUpdate", + BindingFlags.Public | BindingFlags.Instance | BindingFlags.NonPublic); + if (s_onUpdateProp == null) + { + Debug.LogWarning("[SimulateInput] Cannot find onUpdate property."); + return; + } + + // Save the original delegate (InputUpdateDelegate) + s_originalOnUpdate = s_onUpdateProp.GetValue(s_nativeRuntimeInstance) as Delegate; + + // InputUpdateDelegate is internal, so we need to get the Type and use + // Delegate.CreateDelegate with our method that has matching signature + var delegateType = isAssembly.GetType( + "UnityEngine.InputSystem.LowLevel.InputUpdateDelegate"); + if (delegateType == null) + { + Debug.LogWarning("[SimulateInput] Cannot find InputUpdateDelegate type."); + return; + } + + var interceptMethod = typeof(SimulateInput).GetMethod( + nameof(NativeInputInterceptor), BindingFlags.NonPublic | BindingFlags.Static); + var interceptDelegate = Delegate.CreateDelegate(delegateType, interceptMethod); + + s_onUpdateProp.SetValue(s_nativeRuntimeInstance, interceptDelegate); + + EditorApplication.playModeStateChanged += OnPlayModeChanged; + s_nativeHookInstalled = true; + + Debug.Log("[SimulateInput] Native input hook installed successfully."); + } + catch (Exception ex) + { + Debug.LogError($"[SimulateInput] Failed to install native input hook: {ex}"); + } + } + + /// + /// The interceptor that replaces NativeInputRuntime.onUpdate. + /// Signature must match: delegate void InputUpdateDelegate(InputUpdateType, ref InputEventBuffer) + /// When MCP synthetic input is active, we discard hardware events and re-inject ours. + /// + private static void NativeInputInterceptor(InputUpdateType updateType, ref InputEventBuffer eventBuffer) + { + // Do NOT call eventBuffer.Reset() — that would discard ALL native events + // including window resize, display config changes, etc., which breaks + // Screen.width/height and other system state. + // Instead, let all events through normally, then overwrite keyboard/mouse + // state with our synthetic values AFTER the update processes them. + + // Call the original handler (triggers InputSystem.Update internally) + if (s_originalOnUpdate != null) + { + var method = s_originalOnUpdate.Method; + var target = s_originalOnUpdate.Target; + var args = new object[] { updateType, eventBuffer }; + method.Invoke(target, args); + eventBuffer = (InputEventBuffer)args[1]; + } + + if (HasSyntheticInput) + { + // Overwrite keyboard/mouse state with our held keys/buttons + // after hardware events have been processed + ReapplySyntheticState(); + } + } + + private static void ReapplySyntheticState() + { + if (s_heldKeys.Count > 0) + { + var keyboard = Keyboard.current; + if (keyboard != null) + { + using (StateEvent.From(keyboard, out var eventPtr)) + { + foreach (var key in s_heldKeys) + keyboard[key].WriteValueIntoEvent(1f, eventPtr); + InputState.Change(keyboard, eventPtr); + } + } + } + + if (s_heldMouseButtons.Count > 0) + { + var mouse = Mouse.current; + if (mouse != null) + { + using (StateEvent.From(mouse, out var eventPtr)) + { + foreach (var btn in s_heldMouseButtons) + { + var control = GetMouseButtonControl(mouse, btn); + if (control != null) + control.WriteValueIntoEvent(1f, eventPtr); + } + InputState.Change(mouse, eventPtr); + } + } + } + } + + private static void OnPlayModeChanged(PlayModeStateChange change) + { + if (change == PlayModeStateChange.ExitingPlayMode) + { + s_heldKeys.Clear(); + s_heldMouseButtons.Clear(); + } + } + + // ── MCP parameter schema ─────────────────────────────────────── + + public class Parameters + { + [ToolParameter("Action: key_down, key_up, type_text, mouse_button_down, mouse_button_up, " + + "mouse_move, mouse_scroll, click, release_all")] + public string action { get; set; } + + [ToolParameter("Key name (e.g. 'w', 'space', 'shift', 'escape'). Case-insensitive.", Required = false)] + public string key { get; set; } + + [ToolParameter("Mouse button: 'left', 'right', 'middle' (or 0, 1, 2)", Required = false)] + public string button { get; set; } + + [ToolParameter("Relative mouse X movement in pixels", Required = false)] + public float? delta_x { get; set; } + + [ToolParameter("Relative mouse Y movement in pixels", Required = false)] + public float? delta_y { get; set; } + + [ToolParameter("Absolute mouse X position in screen pixels", Required = false)] + public float? position_x { get; set; } + + [ToolParameter("Absolute mouse Y position in screen pixels", Required = false)] + public float? position_y { get; set; } + + [ToolParameter("Horizontal scroll amount (120 = one notch)", Required = false)] + public float? scroll_x { get; set; } + + [ToolParameter("Vertical scroll amount (120 = one notch)", Required = false)] + public float? scroll_y { get; set; } + + [ToolParameter("Key name or JSON array of key names for type_text", Required = false)] + public string keys { get; set; } + } + + // ── Maps ─────────────────────────────────────────────────────── + + private static readonly Dictionary KeyNameMap = BuildKeyNameMap(); + + private static readonly Dictionary MouseButtonMap = new(StringComparer.OrdinalIgnoreCase) + { + { "left", 0 }, { "right", 1 }, { "middle", 2 }, { "forward", 3 }, { "back", 4 }, + }; + + // ==================================================================== + // Main handler + // ==================================================================== + + public static object HandleCommand(JObject @params) + { + if (!EditorApplication.isPlaying) + return new ErrorResponse( + "Input simulation requires Play Mode. Use unity_editor action 'play' first."); + + // Force all device input to route to Game View regardless of focus + EnsureInputRoutesToGameView(); + + // Install native input hook to prevent hardware from overwriting synthetic state + EnsureNativeInputHook(); + + string action = @params["action"]?.ToString(); + if (string.IsNullOrEmpty(action)) + return new ErrorResponse( + "Missing required parameter 'action'. " + + "Supported: key_down, key_up, type_text, mouse_button_down, mouse_button_up, " + + "mouse_move, mouse_scroll, click, release_all"); + + try + { + switch (action.ToLowerInvariant()) + { + case "key_down": return HandleKeyDown(@params); + case "key_up": return HandleKeyUp(@params); + case "type_text": return HandleTypeText(@params); + case "mouse_button_down": return HandleMouseButtonDown(@params); + case "mouse_button_up": return HandleMouseButtonUp(@params); + case "mouse_move": return HandleMouseMove(@params); + case "mouse_scroll": return HandleMouseScroll(@params); + case "click": return HandleClick(@params); + case "release_all": return HandleReleaseAll(); + default: + return new ErrorResponse( + $"Unknown action: '{action}'. " + + "Supported: key_down, key_up, type_text, mouse_button_down, mouse_button_up, " + + "mouse_move, mouse_scroll, click, release_all"); + } + } + catch (Exception ex) + { + return new ErrorResponse($"Input simulation error: {ex.Message}"); + } + } + + // ==================================================================== + // Keyboard actions + // ==================================================================== + + private static object HandleKeyDown(JObject @params) + { + if (Keyboard.current == null) + return new ErrorResponse("No keyboard device found."); + var key = ResolveKey(@params["key"]?.ToString()); + if (key == null) return KeyError(@params["key"]?.ToString()); + QueueKeyState(key.Value, true); + return new SuccessResponse($"Key '{key.Value}' pressed (down)."); + } + + private static object HandleKeyUp(JObject @params) + { + if (Keyboard.current == null) + return new ErrorResponse("No keyboard device found."); + var key = ResolveKey(@params["key"]?.ToString()); + if (key == null) return KeyError(@params["key"]?.ToString()); + QueueKeyState(key.Value, false); + return new SuccessResponse($"Key '{key.Value}' released (up)."); + } + + private static object HandleTypeText(JObject @params) + { + if (Keyboard.current == null) + return new ErrorResponse("No keyboard device found."); + + var keysToken = @params["keys"]; + if (keysToken == null) + return new ErrorResponse("'keys' parameter is required for type_text."); + + var keyNames = new List(); + if (keysToken is JArray arr) + { + foreach (var item in arr) + keyNames.Add(item.ToString()); + } + else + { + keyNames.Add(keysToken.ToString()); + } + + var resolved = new List(); + foreach (var name in keyNames) + { + var key = ResolveKey(name); + if (key == null) return KeyError(name); + resolved.Add(key.Value); + } + + foreach (var key in resolved) + { + QueueKeyState(key, true); + QueueKeyState(key, false); + } + + return new SuccessResponse($"Typed {resolved.Count} key(s): [{string.Join(", ", resolved)}]"); + } + + // ==================================================================== + // Mouse button actions + // ==================================================================== + + private static object HandleMouseButtonDown(JObject @params) + { + if (Mouse.current == null) + return new ErrorResponse("No mouse device found."); + int button = ResolveMouseButton(@params["button"]?.ToString()); + if (button < 0) + return new ErrorResponse("'button' parameter is required (0/left, 1/right, 2/middle, 3/forward, 4/back)."); + QueueMouseButtonState(button, true); + return new SuccessResponse($"Mouse button {button} pressed (down)."); + } + + private static object HandleMouseButtonUp(JObject @params) + { + if (Mouse.current == null) + return new ErrorResponse("No mouse device found."); + int button = ResolveMouseButton(@params["button"]?.ToString()); + if (button < 0) + return new ErrorResponse("'button' parameter is required (0/left, 1/right, 2/middle, 3/forward, 4/back)."); + QueueMouseButtonState(button, false); + return new SuccessResponse($"Mouse button {button} released (up)."); + } + + // ==================================================================== + // Mouse movement + // ==================================================================== + + private static object HandleMouseMove(JObject @params) + { + var mouse = Mouse.current; + if (mouse == null) + return new ErrorResponse("No mouse device found."); + + float? dx = @params["delta_x"]?.ToObject(); + float? dy = @params["delta_y"]?.ToObject(); + float? x = @params["position_x"]?.ToObject(); + float? y = @params["position_y"]?.ToObject(); + + if (dx == null && dy == null && x == null && y == null) + return new ErrorResponse( + "At least one of 'delta_x'/'delta_y' (relative) or 'position_x'/'position_y' (absolute) is required."); + + // Focus Game View so Screen.width/height return correct values. + // UGUI's GraphicRaycaster depends on Screen dimensions for hit testing. + FocusGameView(); + + // Convert from screenshot/image coordinates (top-left origin) + // to InputSystem screen coordinates (bottom-left origin). + // Use Camera.pixelHeight which is always correct, unlike Screen.height + // which returns the focused Editor window's height (known Unity Editor bug). + if (x.HasValue || y.HasValue) + { + var cam = Camera.main; + if (cam != null && y.HasValue) + { + y = cam.pixelHeight - y.Value; + } + } + + using (StateEvent.From(mouse, out var eventPtr)) + { + if (dx.HasValue || dy.HasValue) + mouse.delta.WriteValueIntoEvent(new Vector2(dx ?? 0f, dy ?? 0f), eventPtr); + + if (x.HasValue || y.HasValue) + { + var currentPos = mouse.position.ReadValue(); + mouse.position.WriteValueIntoEvent( + new Vector2(x ?? currentPos.x, y ?? currentPos.y), eventPtr); + } + + InputState.Change(mouse, eventPtr); + } + + string details = ""; + if (dx.HasValue || dy.HasValue) details += $"delta=({dx ?? 0},{dy ?? 0}) "; + if (x.HasValue || y.HasValue) details += $"position=({x},{y})"; + return new SuccessResponse($"Mouse moved: {details.Trim()}"); + } + + // ==================================================================== + // Mouse scroll + // ==================================================================== + + private static object HandleMouseScroll(JObject @params) + { + var mouse = Mouse.current; + if (mouse == null) + return new ErrorResponse("No mouse device found."); + + float scrollX = @params["scroll_x"]?.ToObject() ?? 0f; + float scrollY = @params["scroll_y"]?.ToObject() ?? 0f; + + if (Math.Abs(scrollX) < 0.001f && Math.Abs(scrollY) < 0.001f) + return new ErrorResponse("At least one of 'scroll_x' or 'scroll_y' must be non-zero."); + + using (StateEvent.From(mouse, out var eventPtr)) + { + mouse.scroll.WriteValueIntoEvent(new Vector2(scrollX, scrollY), eventPtr); + InputState.Change(mouse, eventPtr); + } + + return new SuccessResponse($"Mouse scrolled: ({scrollX}, {scrollY})"); + } + + // ==================================================================== + // Click — convenience: move (optional) + press + release + // ==================================================================== + + private static object HandleClick(JObject @params) + { + var mouse = Mouse.current; + if (mouse == null) + return new ErrorResponse("No mouse device found."); + + string buttonRaw = @params["button"]?.ToString() ?? "left"; + int buttonIdx = ResolveMouseButton(buttonRaw); + if (buttonIdx < 0) + return new ErrorResponse($"Unknown mouse button: '{buttonRaw}'."); + + float? x = @params["position_x"]?.ToObject(); + float? y = @params["position_y"]?.ToObject(); + + if (x.HasValue || y.HasValue) + { + using (StateEvent.From(mouse, out var eventPtr)) + { + var currentPos = mouse.position.ReadValue(); + mouse.position.WriteValueIntoEvent( + new Vector2(x ?? currentPos.x, y ?? currentPos.y), eventPtr); + InputState.Change(mouse, eventPtr); + } + } + + QueueMouseButtonState(buttonIdx, true); + QueueMouseButtonState(buttonIdx, false); + + string pos = (x.HasValue || y.HasValue) ? $" at ({x}, {y})" : " at current position"; + return new SuccessResponse($"Clicked {buttonRaw} button{pos}."); + } + + // ==================================================================== + // Release all — safety valve + // ==================================================================== + + private static object HandleReleaseAll() + { + s_heldKeys.Clear(); + s_heldMouseButtons.Clear(); + int released = 0; + + var keyboard = Keyboard.current; + if (keyboard != null) + { + using (StateEvent.From(keyboard, out var eventPtr)) + { + foreach (Key key in Enum.GetValues(typeof(Key))) + { + if (key == Key.None || key == Key.IMESelected) continue; + try + { + keyboard[key].WriteValueIntoEvent(0f, eventPtr); + released++; + } + catch { /* Some enum values may not map to physical keys */ } + } + InputState.Change(keyboard, eventPtr); + } + } + + var mouse = Mouse.current; + if (mouse != null) + { + using (StateEvent.From(mouse, out var eventPtr)) + { + mouse.leftButton.WriteValueIntoEvent(0f, eventPtr); + mouse.rightButton.WriteValueIntoEvent(0f, eventPtr); + mouse.middleButton.WriteValueIntoEvent(0f, eventPtr); + mouse.forwardButton.WriteValueIntoEvent(0f, eventPtr); + mouse.backButton.WriteValueIntoEvent(0f, eventPtr); + InputState.Change(mouse, eventPtr); + released += 5; + } + } + + return new SuccessResponse($"Released all input devices ({released} controls zeroed)."); + } + + // ==================================================================== + // Low-level helpers + // ==================================================================== + + private static bool s_inputRouteConfigured; + + /// + /// Ensure InputSystem routes all device input to the Game View, + /// even when the Game View doesn't have editor focus. + /// Also focus the Game View window as a belt-and-suspenders measure. + /// + private static void EnsureInputRoutesToGameView() + { + if (!s_inputRouteConfigured) + { + var settings = InputSystem.settings; + if (settings != null) + { + settings.editorInputBehaviorInPlayMode = + InputSettings.EditorInputBehaviorInPlayMode.AllDeviceInputAlwaysGoesToGameView; + } + s_inputRouteConfigured = true; + } + + // NOTE: Do NOT call FocusGameView() here globally. + // It interferes with screenshot capture. Mouse actions call it explicitly. + } + + private static void FocusGameView() + { + var gameViewType = typeof(EditorWindow).Assembly.GetType("UnityEditor.GameView"); + if (gameViewType != null) + EditorWindow.FocusWindowIfItsOpen(gameViewType); + } + + private static void QueueKeyState(Key key, bool pressed) + { + var keyboard = Keyboard.current; + if (keyboard == null) return; + + EnsureNativeInputHook(); + + // Track held state so NativeInputInterceptor can re-inject every frame + if (pressed) + s_heldKeys.Add(key); + else + s_heldKeys.Remove(key); + + // Also apply immediately for the current frame + using (StateEvent.From(keyboard, out var eventPtr)) + { + keyboard[key].WriteValueIntoEvent(pressed ? 1f : 0f, eventPtr); + InputState.Change(keyboard, eventPtr); + } + } + + private static void QueueMouseButtonState(int button, bool pressed) + { + var mouse = Mouse.current; + var control = GetMouseButtonControl(mouse, button); + if (control == null) return; + + EnsureNativeInputHook(); + + if (pressed) + s_heldMouseButtons.Add(button); + else + s_heldMouseButtons.Remove(button); + + using (StateEvent.From(mouse, out var eventPtr)) + { + control.WriteValueIntoEvent(pressed ? 1f : 0f, eventPtr); + InputState.Change(mouse, eventPtr); + } + } + + private static ButtonControl GetMouseButtonControl(Mouse mouse, int button) + { + return button switch + { + 0 => mouse.leftButton, + 1 => mouse.rightButton, + 2 => mouse.middleButton, + 3 => mouse.forwardButton, + 4 => mouse.backButton, + _ => null, + }; + } + + // ==================================================================== + // Key resolution + // ==================================================================== + + private static Key? ResolveKey(string keyName) + { + if (string.IsNullOrEmpty(keyName)) return null; + if (KeyNameMap.TryGetValue(keyName, out var mapped)) return mapped; + if (Enum.TryParse(keyName, true, out var parsed) && parsed != Key.None) return parsed; + return null; + } + + private static int ResolveMouseButton(string raw) + { + if (string.IsNullOrEmpty(raw)) return -1; + if (MouseButtonMap.TryGetValue(raw, out int idx)) return idx; + if (int.TryParse(raw, out int num) && num >= 0 && num <= 4) return num; + return -1; + } + + private static ErrorResponse KeyError(string provided) + { + return new ErrorResponse( + $"Unknown key: '{provided ?? "(none)"}'. Use key names like " + + "'w', 'space', 'shift', 'ctrl', 'alt', 'escape', 'enter', 'tab', " + + "'up', 'down', 'left', 'right', 'f1'-'f12', '0'-'9', etc."); + } + + // ==================================================================== + // Key name aliases + // ==================================================================== + + private static Dictionary BuildKeyNameMap() + { + return new Dictionary(StringComparer.OrdinalIgnoreCase) + { + // Letters + { "a", Key.A }, { "b", Key.B }, { "c", Key.C }, { "d", Key.D }, + { "e", Key.E }, { "f", Key.F }, { "g", Key.G }, { "h", Key.H }, + { "i", Key.I }, { "j", Key.J }, { "k", Key.K }, { "l", Key.L }, + { "m", Key.M }, { "n", Key.N }, { "o", Key.O }, { "p", Key.P }, + { "q", Key.Q }, { "r", Key.R }, { "s", Key.S }, { "t", Key.T }, + { "u", Key.U }, { "v", Key.V }, { "w", Key.W }, { "x", Key.X }, + { "y", Key.Y }, { "z", Key.Z }, + + // Digits + { "0", Key.Digit0 }, { "1", Key.Digit1 }, { "2", Key.Digit2 }, + { "3", Key.Digit3 }, { "4", Key.Digit4 }, { "5", Key.Digit5 }, + { "6", Key.Digit6 }, { "7", Key.Digit7 }, { "8", Key.Digit8 }, + { "9", Key.Digit9 }, + + // Common aliases + { "space", Key.Space }, + { "spacebar", Key.Space }, + { "enter", Key.Enter }, + { "return", Key.Enter }, + { "esc", Key.Escape }, + { "escape", Key.Escape }, + { "tab", Key.Tab }, + { "backspace", Key.Backspace }, + { "delete", Key.Delete }, + { "del", Key.Delete }, + + // Modifiers + { "shift", Key.LeftShift }, + { "leftshift", Key.LeftShift }, + { "lshift", Key.LeftShift }, + { "rightshift", Key.RightShift }, + { "rshift", Key.RightShift }, + { "ctrl", Key.LeftCtrl }, + { "leftctrl", Key.LeftCtrl }, + { "lctrl", Key.LeftCtrl }, + { "rightctrl", Key.RightCtrl }, + { "rctrl", Key.RightCtrl }, + { "control", Key.LeftCtrl }, + { "alt", Key.LeftAlt }, + { "leftalt", Key.LeftAlt }, + { "lalt", Key.LeftAlt }, + { "rightalt", Key.RightAlt }, + { "ralt", Key.RightAlt }, + { "cmd", Key.LeftMeta }, + { "command", Key.LeftMeta }, + { "meta", Key.LeftMeta }, + { "leftmeta", Key.LeftMeta }, + { "rightmeta", Key.RightMeta }, + { "win", Key.LeftMeta }, + { "windows", Key.LeftMeta }, + { "capslock", Key.CapsLock }, + + // Arrow keys + { "up", Key.UpArrow }, + { "down", Key.DownArrow }, + { "left", Key.LeftArrow }, + { "right", Key.RightArrow }, + { "uparrow", Key.UpArrow }, + { "downarrow", Key.DownArrow }, + { "leftarrow", Key.LeftArrow }, + { "rightarrow", Key.RightArrow }, + { "arrowup", Key.UpArrow }, + { "arrowdown", Key.DownArrow }, + { "arrowleft", Key.LeftArrow }, + { "arrowright", Key.RightArrow }, + + // Function keys + { "f1", Key.F1 }, { "f2", Key.F2 }, { "f3", Key.F3 }, + { "f4", Key.F4 }, { "f5", Key.F5 }, { "f6", Key.F6 }, + { "f7", Key.F7 }, { "f8", Key.F8 }, { "f9", Key.F9 }, + { "f10", Key.F10 }, { "f11", Key.F11 }, { "f12", Key.F12 }, + + // Navigation + { "home", Key.Home }, + { "end", Key.End }, + { "pageup", Key.PageUp }, + { "pagedown", Key.PageDown }, + { "insert", Key.Insert }, + + // Punctuation / symbols + { "minus", Key.Minus }, + { "equals", Key.Equals }, + { "leftbracket", Key.LeftBracket }, + { "rightbracket", Key.RightBracket }, + { "backslash", Key.Backslash }, + { "semicolon", Key.Semicolon }, + { "quote", Key.Quote }, + { "comma", Key.Comma }, + { "period", Key.Period }, + { "slash", Key.Slash }, + { "backquote", Key.Backquote }, + { "tilde", Key.Backquote }, + + // Numpad + { "numpad0", Key.Numpad0 }, { "numpad1", Key.Numpad1 }, + { "numpad2", Key.Numpad2 }, { "numpad3", Key.Numpad3 }, + { "numpad4", Key.Numpad4 }, { "numpad5", Key.Numpad5 }, + { "numpad6", Key.Numpad6 }, { "numpad7", Key.Numpad7 }, + { "numpad8", Key.Numpad8 }, { "numpad9", Key.Numpad9 }, + { "numpadenter", Key.NumpadEnter }, + { "numpadplus", Key.NumpadPlus }, + { "numpadminus", Key.NumpadMinus }, + { "numpadtimes", Key.NumpadMultiply }, + { "numpaddivide", Key.NumpadDivide }, + { "numpadperiod", Key.NumpadPeriod }, + { "numlock", Key.NumLock }, + + // Misc + { "printscreen", Key.PrintScreen }, + { "scrolllock", Key.ScrollLock }, + { "pause", Key.Pause }, + }; + } + } +} +#endif diff --git a/Editor/Tools/Input/SimulateInput.cs.meta b/Editor/Tools/Input/SimulateInput.cs.meta new file mode 100644 index 0000000..32410a0 --- /dev/null +++ b/Editor/Tools/Input/SimulateInput.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: c7014f673b86e4b499f576f59b920aad \ No newline at end of file diff --git a/Editor/Tools/ManageScene.cs b/Editor/Tools/ManageScene.cs index 85c2461..e9854bb 100644 --- a/Editor/Tools/ManageScene.cs +++ b/Editor/Tools/ManageScene.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using NativeMcp.Editor.Helpers; using NativeMcp.Editor.Tools.Scene; using Newtonsoft.Json.Linq; @@ -49,8 +50,7 @@ public static object HandleCommand(JObject @params) try { var task = CommandRegistry.InvokeCommandAsync(toolName, forwarded); - // All scene tools are synchronous, so the task is already completed. - return task.Result; + return task.GetAwaiter().GetResult(); } catch (AggregateException ae) when (ae.InnerException != null) { @@ -66,9 +66,9 @@ public static object HandleCommand(JObject @params) /// Public API for screenshot capture. Delegates to SceneScreenshot. /// Kept for backward compatibility. /// - public static object ExecuteScreenshot(string fileName = null, int? superSize = null) + public static Task ExecuteScreenshot(string fileName = null, int? superSize = null) { - return SceneScreenshot.CaptureScreenshot(fileName, superSize); + return SceneScreenshot.CaptureScreenshotAsync(fileName, superSize); } } } diff --git a/Editor/Tools/Meta/UnityEditorControl.cs b/Editor/Tools/Meta/UnityEditorControl.cs index 0d0511f..03a25bc 100644 --- a/Editor/Tools/Meta/UnityEditorControl.cs +++ b/Editor/Tools/Meta/UnityEditorControl.cs @@ -20,7 +20,8 @@ namespace NativeMcp.Editor.Tools.Meta "- add_layer(layerName) — add a project layer\n" + "- remove_layer(layerName) — remove a project layer\n" + "- set_active_tool(toolName) — set editor tool (View, Move, Rotate, Scale, Rect, Transform)\n" + - "- refresh(mode?, scope?, compile?, wait_for_ready?) — refresh asset database")] + "- refresh(mode?, scope?, compile?, wait_for_ready?) — refresh asset database\n" + + "- execute_menu_item(menu_path) — execute any editor menu item by path")] public static class UnityEditorControl { private static readonly Dictionary ActionMap = new() @@ -37,11 +38,13 @@ public static class UnityEditorControl ["remove_layer"] = "editor_remove_layer", ["set_active_tool"] = "editor_set_active_tool", ["refresh"] = "refresh_unity", + ["execute_menu_item"] = "execute_menu_item", }; public class Parameters { - [ToolParameter("Action to perform: play, stop, pause, step_frame, play_for_frames, set_update_frequency, add_tag, remove_tag, add_layer, remove_layer, set_active_tool, refresh")] + [ToolParameter("Action to perform: play, stop, pause, step_frame, play_for_frames, set_update_frequency, " + + "add_tag, remove_tag, add_layer, remove_layer, set_active_tool, refresh, execute_menu_item")] public string action { get; set; } [ToolParameter("Number of frames to advance for play_for_frames (>= 1)", Required = false)] @@ -76,6 +79,9 @@ public class Parameters [ToolParameter("Wait for Unity ready state after refresh", Required = false)] public bool? wait_for_ready { get; set; } + + [ToolParameter("Menu item path for execute_menu_item (e.g. 'GameObject/3D Object/Cube')", Required = false)] + public string menu_path { get; set; } } public static async Task HandleCommand(JObject @params) diff --git a/Editor/Tools/Meta/UnityInput.cs b/Editor/Tools/Meta/UnityInput.cs new file mode 100644 index 0000000..1346297 --- /dev/null +++ b/Editor/Tools/Meta/UnityInput.cs @@ -0,0 +1,70 @@ +using NativeMcp.Editor.Helpers; +using Newtonsoft.Json.Linq; + +namespace NativeMcp.Editor.Tools.Meta +{ + [McpForUnityTool("unity_input", + Description = + "Simulate keyboard and mouse input during Play Mode via low-level InputSystem events.\n" + + "Actions:\n" + + "- key_down(key) — press keyboard key (persists until key_up)\n" + + "- key_up(key) — release keyboard key\n" + + "- type_text(keys) — press+release key sequence in one frame\n" + + "- mouse_button_down(button) — press mouse button\n" + + "- mouse_button_up(button) — release mouse button\n" + + "- mouse_move(delta_x?, delta_y?, position_x?, position_y?) — move mouse\n" + + "- mouse_scroll(scroll_x?, scroll_y?) — scroll wheel\n" + + "- click(button?, position_x?, position_y?) — move + click (single frame)\n" + + "- release_all() — release all keys and buttons\n" + + "NOTE: For UI clicks, 'click' may fail because UI needs pointer arrival before the press. " + + "Split into: mouse_move → step_frame → mouse_button_down → step_frame → mouse_button_up. " + + "Mouse coordinates are OS screen space, not Game View space.")] + public static class UnityInput + { + public class Parameters + { + [ToolParameter("Action: key_down, key_up, type_text, mouse_button_down, mouse_button_up, " + + "mouse_move, mouse_scroll, click, release_all")] + public string action { get; set; } + + [ToolParameter("Key name (e.g. 'w', 'space', 'shift', 'escape'). Case-insensitive.", Required = false)] + public string key { get; set; } + + [ToolParameter("Mouse button: 'left', 'right', 'middle' (or 0, 1, 2)", Required = false)] + public string button { get; set; } + + [ToolParameter("Relative mouse X movement in pixels", Required = false)] + public float? delta_x { get; set; } + + [ToolParameter("Relative mouse Y movement in pixels", Required = false)] + public float? delta_y { get; set; } + + [ToolParameter("Absolute mouse X position in screen pixels", Required = false)] + public float? position_x { get; set; } + + [ToolParameter("Absolute mouse Y position in screen pixels", Required = false)] + public float? position_y { get; set; } + + [ToolParameter("Horizontal scroll amount (120 = one notch)", Required = false)] + public float? scroll_x { get; set; } + + [ToolParameter("Vertical scroll amount (120 = one notch)", Required = false)] + public float? scroll_y { get; set; } + + [ToolParameter("Key name or JSON array of key names for type_text", Required = false)] + public string keys { get; set; } + } + + public static object HandleCommand(JObject @params) + { +#if !NATIVE_MCP_HAS_INPUT_SYSTEM + return new ErrorResponse( + "Input simulation requires the Unity Input System package (com.unity.inputsystem). " + + "Install it via Window > Package Manager."); +#else + return CommandRegistry.InvokeCommandAsync("simulate_input", @params ?? new JObject()) + .GetAwaiter().GetResult(); +#endif + } + } +} diff --git a/Editor/Tools/Meta/UnityInput.cs.meta b/Editor/Tools/Meta/UnityInput.cs.meta new file mode 100644 index 0000000..5ae01a9 --- /dev/null +++ b/Editor/Tools/Meta/UnityInput.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 04b7f733b0cfd425e91bf28caad6fa5c \ No newline at end of file diff --git a/Editor/Tools/Meta/UnityScene.cs b/Editor/Tools/Meta/UnityScene.cs index 11e8b65..361262a 100644 --- a/Editor/Tools/Meta/UnityScene.cs +++ b/Editor/Tools/Meta/UnityScene.cs @@ -17,7 +17,7 @@ namespace NativeMcp.Editor.Tools.Meta "— paged hierarchy listing\n" + "- get_active() — active scene info\n" + "- get_build_settings() — scenes in Build Settings\n" + - "- screenshot(fileName?, superSize?) — capture camera view as image")] + "- screenshot(fileName?, superSize?, mode?) — capture Game View as image (mode: auto/game_view/window/camera)")] public static class UnityScene { private static readonly Dictionary ActionMap = new() @@ -85,6 +85,9 @@ public class Parameters [ToolParameter("Screenshot supersampling multiplier", Required = false)] public int? superSize { get; set; } + + [ToolParameter("Screenshot capture mode: auto (default), game_view, window, camera", Required = false)] + public string mode { get; set; } } public static async Task HandleCommand(JObject @params) diff --git a/Editor/Tools/Scene/SceneScreenshot.cs b/Editor/Tools/Scene/SceneScreenshot.cs index 593c3e1..7c86f85 100644 --- a/Editor/Tools/Scene/SceneScreenshot.cs +++ b/Editor/Tools/Scene/SceneScreenshot.cs @@ -1,18 +1,31 @@ using System; +using System.Collections; using System.Collections.Generic; using System.IO; +using System.Reflection; +using System.Threading.Tasks; +using NativeMcp.Editor.Bridge; using NativeMcp.Editor.Helpers; +using NativeMcp.Editor.Protocol; using Newtonsoft.Json.Linq; +using UnityEditor; using UnityEngine; namespace NativeMcp.Editor.Tools.Scene { /// /// Capture a screenshot from the current camera view. + /// In Play Mode, focuses the Game View and uses a coroutine-based ScreenCapture for reliable capture + /// (including Screen Space - Overlay UI). Falls back to window pixel reading if needed. + /// In Edit Mode, renders via Camera.Render() into a RenderTexture. /// - [McpForUnityTool("scene_screenshot", Internal = true, Description = "Capture a screenshot from the current camera view.")] + [McpForUnityTool("scene_screenshot", Internal = true, + Description = "Capture a screenshot from the current camera view. " + + "In Play Mode captures the Game View including UI overlays.")] public static class SceneScreenshot { + private const int CoroutineTimeoutMs = 5000; + public class Parameters { [ToolParameter("Output file name (default: screenshot-.png)", Required = false)] @@ -20,133 +33,394 @@ public class Parameters [ToolParameter("Super-sampling multiplier for resolution (default 1)", Required = false)] public int? superSize { get; set; } + + [ToolParameter( + "Capture mode: 'auto' (default, tries coroutine then RT fallback), " + + "'game_view' (coroutine-based ScreenCapture only), " + + "'window' (Game View internal RenderTexture), " + + "'camera' (Camera.Render, no Overlay UI)", + Required = false)] + public string mode { get; set; } } - public static object HandleCommand(JObject @params) + public static async Task HandleCommand(JObject @params) { var p = @params ?? new JObject(); string fileName = SceneHelpers.ParseString(p, "fileName", "filename"); int? superSize = SceneHelpers.ParseInt(p, "superSize", "super_size", "supersize"); - return CaptureScreenshot(fileName, superSize); + string mode = SceneHelpers.ParseString(p, "mode") ?? "auto"; + return await CaptureScreenshotAsync(fileName, superSize, mode); } - /// - /// Core screenshot capture logic. Can be called directly for backward compatibility. - /// - internal static object CaptureScreenshot(string fileName, int? superSize) + internal static async Task CaptureScreenshotAsync(string fileName, int? superSize, string mode = "auto") { try { int size = (superSize.HasValue && superSize.Value > 0) ? superSize.Value : 1; - - Texture2D tex; + Texture2D tex = null; + string captureMethod = null; if (Application.isPlaying) { - tex = ScreenCapture.CaptureScreenshotAsTexture(size); - if (tex == null) - return new ErrorResponse("ScreenCapture.CaptureScreenshotAsTexture returned null."); + tex = await CapturePlayModeAsync(size, mode); + if (tex != null) + captureMethod = _lastCaptureMethod; } else { - Camera cam = Camera.main; + if (mode == "game_view" || mode == "window") + return new ErrorResponse($"Mode '{mode}' is only available in Play Mode."); - var urpCamDataType = System.Type.GetType( - "UnityEngine.Rendering.Universal.UniversalAdditionalCameraData, Unity.RenderPipelines.Universal.Runtime"); - var renderTypeProp = urpCamDataType?.GetProperty("renderType"); + tex = CaptureViaCamera(size); + captureMethod = "camera"; + } - if (cam == null || !cam.isActiveAndEnabled) - { - foreach (var c in UnityEngine.Object.FindObjectsByType(FindObjectsSortMode.None)) - { - if (!c.isActiveAndEnabled || c.targetTexture != null) continue; - - if (urpCamDataType != null && renderTypeProp != null) - { - var uacd = c.GetComponent(urpCamDataType); - if (uacd != null && renderTypeProp.GetValue(uacd)?.ToString() == "Overlay") continue; - } - - if (cam == null || c.depth > cam.depth) cam = c; - } - } + if (tex == null) + return new ErrorResponse("All capture methods failed. Ensure a valid Camera exists or the Game View is visible."); - if (cam == null) - return new ErrorResponse("No valid screen-rendering Camera found to capture screenshot."); + return BuildResult(tex, fileName, captureMethod); + } + catch (Exception e) + { + return new ErrorResponse($"Error capturing screenshot: {e.Message}"); + } + } - int width = Mathf.Max(1, cam.pixelWidth > 0 ? cam.pixelWidth : Screen.width) * size; - int height = Mathf.Max(1, cam.pixelHeight > 0 ? cam.pixelHeight : Screen.height) * size; + // Tracks which method succeeded for the response message + private static string _lastCaptureMethod; - var rt = RenderTexture.GetTemporary(width, height, 24, RenderTextureFormat.ARGB32); - tex = new Texture2D(width, height, TextureFormat.RGBA32, false); + private static async Task CapturePlayModeAsync(int size, string mode) + { + _lastCaptureMethod = null; - var prevRT = cam.targetTexture; - var prevActive = RenderTexture.active; - try + if (mode == "game_view" || mode == "auto") + { + try + { + var tex = await CaptureViaCoroutineAsync(size); + if (tex != null) { - cam.targetTexture = rt; - cam.Render(); - RenderTexture.active = rt; - tex.ReadPixels(new Rect(0, 0, width, height), 0, 0); - tex.Apply(); + _lastCaptureMethod = "coroutine"; + return tex; } - finally + } + catch (Exception e) + { + Debug.LogWarning($"[NativeMcp] Coroutine screenshot failed: {e.Message}"); + } + + if (mode == "game_view") + return null; + } + + if (mode == "window" || mode == "auto") + { + try + { + var tex = CaptureGameViewPixels(); + if (tex != null) { - cam.targetTexture = prevRT; - RenderTexture.active = prevActive; - RenderTexture.ReleaseTemporary(rt); + _lastCaptureMethod = "window"; + return tex; } } + catch (Exception e) + { + Debug.LogWarning($"[NativeMcp] Window pixel screenshot failed: {e.Message}"); + } - int texWidth = tex.width; - int texHeight = tex.height; - byte[] pngBytes = tex.EncodeToPNG(); - UnityEngine.Object.DestroyImmediate(tex); - - string resolvedName = string.IsNullOrWhiteSpace(fileName) - ? $"screenshot-{DateTime.Now:yyyyMMdd-HHmmss}.png" - : (fileName.EndsWith(".png", StringComparison.OrdinalIgnoreCase) ? fileName : fileName + ".png"); - string folder = Path.Combine(Application.dataPath, "Screenshots"); - Directory.CreateDirectory(folder); - string fullPath = Path.Combine(folder, resolvedName).Replace('\\', '/'); + if (mode == "window") + return null; + } - if (File.Exists(fullPath)) + if (mode == "camera" || mode == "auto") + { + try { - string dir = Path.GetDirectoryName(fullPath); - string baseName = Path.GetFileNameWithoutExtension(fullPath); - string ext = Path.GetExtension(fullPath); - int counter = 1; - do + var tex = CaptureViaCamera(size); + if (tex != null) { - fullPath = Path.Combine(dir, $"{baseName}-{counter}{ext}").Replace('\\', '/'); - counter++; - } while (File.Exists(fullPath)); + _lastCaptureMethod = "camera"; + return tex; + } + } + catch (Exception e) + { + Debug.LogWarning($"[NativeMcp] Camera screenshot failed: {e.Message}"); } - File.WriteAllBytes(fullPath, pngBytes); + } + + return null; + } + + #region Coroutine-based capture (ScreenCapture) + + private class ScreenshotCoroutineRunner : MonoBehaviour { } + + private static async Task CaptureViaCoroutineAsync(int size) + { + // Focus the Game View first + FocusGameView(); + + // Wait one editor frame for focus to settle + var focusTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + EditorApplication.delayCall += () => focusTcs.TrySetResult(true); + EditorNudge.BeginNudge(); + try + { + await focusTcs.Task; + } + finally + { + EditorNudge.EndNudge(); + } + + if (!Application.isPlaying) + return null; + + // Create temp GO to run coroutine + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var go = new GameObject("[NativeMcp] ScreenshotHelper") + { + hideFlags = HideFlags.HideAndDontSave + }; + + EditorNudge.BeginNudge(); + try + { + var runner = go.AddComponent(); + runner.StartCoroutine(CaptureCoroutine(size, tcs)); + + // Timeout + var timeoutTask = Task.Delay(CoroutineTimeoutMs); + var completed = await Task.WhenAny(tcs.Task, timeoutTask); + + if (completed == timeoutTask) + { + tcs.TrySetResult(null); + Debug.LogWarning("[NativeMcp] Coroutine screenshot timed out."); + return null; + } + + return tcs.Task.Result; + } + finally + { + EditorNudge.EndNudge(); + if (go != null) + UnityEngine.Object.DestroyImmediate(go); + } + } + + private static IEnumerator CaptureCoroutine(int size, TaskCompletionSource tcs) + { + yield return new WaitForEndOfFrame(); + try + { + var tex = ScreenCapture.CaptureScreenshotAsTexture(size); + tcs.TrySetResult(tex); + } + catch (Exception e) + { + Debug.LogWarning($"[NativeMcp] ScreenCapture.CaptureScreenshotAsTexture failed: {e.Message}"); + tcs.TrySetResult(null); + } + } + + #endregion - string projectRoot = Path.GetFullPath(Path.Combine(Application.dataPath, "..")).Replace('\\', '/'); - if (!projectRoot.EndsWith("/")) projectRoot += "/"; - string assetsRelPath = fullPath.StartsWith(projectRoot, StringComparison.OrdinalIgnoreCase) - ? fullPath.Substring(projectRoot.Length) - : fullPath; + #region Game View RenderTexture capture (fallback) - string base64 = System.Convert.ToBase64String(pngBytes); - string message = $"Screenshot captured to '{assetsRelPath}' ({texWidth}x{texHeight})."; + private static Texture2D CaptureGameViewPixels() + { + // Read the Game View's internal m_RenderTexture via reflection. + // This is the composited output that includes UI overlays. + // Safer than ReadScreenPixel which causes native crashes on macOS. + var gameView = FocusGameView(); + if (gameView == null) + { + Debug.LogWarning("[NativeMcp] Could not find Game View window."); + return null; + } + + gameView.Repaint(); + + var gameViewType = gameView.GetType(); + + // Try m_RenderTexture first (available in most Unity versions) + var rtField = gameViewType.GetField("m_RenderTexture", + BindingFlags.NonPublic | BindingFlags.Instance); + var rt = rtField?.GetValue(gameView) as RenderTexture; + + if (rt == null || rt.width <= 0 || rt.height <= 0) + { + Debug.LogWarning("[NativeMcp] Game View RenderTexture not available or empty."); + return null; + } + + var tex = new Texture2D(rt.width, rt.height, TextureFormat.RGBA32, false); + var prevActive = RenderTexture.active; + try + { + RenderTexture.active = rt; + tex.ReadPixels(new Rect(0, 0, rt.width, rt.height), 0, 0); + tex.Apply(); + } + finally + { + RenderTexture.active = prevActive; + } + + // Flip vertically — ReadPixels from RenderTexture yields Y-inverted data + // (RenderTexture origin is bottom-left, screen/PNG origin is top-left) + FlipTextureVertically(tex); + + return tex; + } - return new NativeMcp.Editor.Protocol.McpToolCallResult + #endregion + + #region Camera.Render capture (Edit Mode / fallback) + + private static Texture2D CaptureViaCamera(int size) + { + Camera cam = Camera.main; + + var urpCamDataType = Type.GetType( + "UnityEngine.Rendering.Universal.UniversalAdditionalCameraData, Unity.RenderPipelines.Universal.Runtime"); + var renderTypeProp = urpCamDataType?.GetProperty("renderType"); + + if (cam == null || !cam.isActiveAndEnabled) + { + foreach (var c in UnityEngine.Object.FindObjectsByType(FindObjectsSortMode.None)) { - IsError = false, - Content = new List + if (!c.isActiveAndEnabled || c.targetTexture != null) continue; + + if (urpCamDataType != null && renderTypeProp != null) { - new NativeMcp.Editor.Protocol.McpContentBlock { Type = "text", Text = message }, - new NativeMcp.Editor.Protocol.McpContentBlock { Type = "image", Data = base64, MimeType = "image/png" } + var uacd = c.GetComponent(urpCamDataType); + if (uacd != null && renderTypeProp.GetValue(uacd)?.ToString() == "Overlay") continue; } - }; + + if (cam == null || c.depth > cam.depth) cam = c; + } } - catch (Exception e) + + if (cam == null) + return null; + + int width = Mathf.Max(1, cam.pixelWidth > 0 ? cam.pixelWidth : Screen.width) * size; + int height = Mathf.Max(1, cam.pixelHeight > 0 ? cam.pixelHeight : Screen.height) * size; + + var rt = RenderTexture.GetTemporary(width, height, 24, RenderTextureFormat.ARGB32); + var tex = new Texture2D(width, height, TextureFormat.RGBA32, false); + + var prevRT = cam.targetTexture; + var prevActive = RenderTexture.active; + try { - return new ErrorResponse($"Error capturing screenshot: {e.Message}"); + cam.targetTexture = rt; + cam.Render(); + RenderTexture.active = rt; + tex.ReadPixels(new Rect(0, 0, width, height), 0, 0); + tex.Apply(); + } + finally + { + cam.targetTexture = prevRT; + RenderTexture.active = prevActive; + RenderTexture.ReleaseTemporary(rt); + } + + return tex; + } + + #endregion + + #region Helpers + + private static void FlipTextureVertically(Texture2D tex) + { + var pixels = tex.GetPixels(); + int w = tex.width, h = tex.height; + for (int y = 0; y < h / 2; y++) + { + int topRow = y * w; + int bottomRow = (h - 1 - y) * w; + for (int x = 0; x < w; x++) + { + (pixels[topRow + x], pixels[bottomRow + x]) = (pixels[bottomRow + x], pixels[topRow + x]); + } + } + tex.SetPixels(pixels); + tex.Apply(); + } + + private static EditorWindow FocusGameView() + { + try + { + var gameViewType = typeof(EditorWindow).Assembly.GetType("UnityEditor.GameView"); + if (gameViewType == null) return null; + var gameView = EditorWindow.GetWindow(gameViewType, false, null, true); + gameView?.Focus(); + return gameView; + } + catch + { + return null; } } + + private static object BuildResult(Texture2D tex, string fileName, string captureMethod) + { + int texWidth = tex.width; + int texHeight = tex.height; + byte[] pngBytes = tex.EncodeToPNG(); + UnityEngine.Object.DestroyImmediate(tex); + + string resolvedName = string.IsNullOrWhiteSpace(fileName) + ? $"screenshot-{DateTime.Now:yyyyMMdd-HHmmss}.png" + : (fileName.EndsWith(".png", StringComparison.OrdinalIgnoreCase) ? fileName : fileName + ".png"); + string folder = Path.Combine(Application.dataPath, "Screenshots"); + Directory.CreateDirectory(folder); + string fullPath = Path.Combine(folder, resolvedName).Replace('\\', '/'); + + if (File.Exists(fullPath)) + { + string dir = Path.GetDirectoryName(fullPath); + string baseName = Path.GetFileNameWithoutExtension(fullPath); + string ext = Path.GetExtension(fullPath); + int counter = 1; + do + { + fullPath = Path.Combine(dir, $"{baseName}-{counter}{ext}").Replace('\\', '/'); + counter++; + } while (File.Exists(fullPath)); + } + File.WriteAllBytes(fullPath, pngBytes); + + string projectRoot = Path.GetFullPath(Path.Combine(Application.dataPath, "..")).Replace('\\', '/'); + if (!projectRoot.EndsWith("/")) projectRoot += "/"; + string assetsRelPath = fullPath.StartsWith(projectRoot, StringComparison.OrdinalIgnoreCase) + ? fullPath.Substring(projectRoot.Length) + : fullPath; + + string base64 = Convert.ToBase64String(pngBytes); + string methodNote = captureMethod != null ? $" [method: {captureMethod}]" : ""; + string message = $"Screenshot captured to '{assetsRelPath}' ({texWidth}x{texHeight}).{methodNote}"; + + return new McpToolCallResult + { + IsError = false, + Content = new List + { + new McpContentBlock { Type = "text", Text = message }, + new McpContentBlock { Type = "image", Data = base64, MimeType = "image/png" } + } + }; + } + + #endregion } } diff --git a/Tests/Editor/InputToolTests.cs b/Tests/Editor/InputToolTests.cs new file mode 100644 index 0000000..6324ea1 --- /dev/null +++ b/Tests/Editor/InputToolTests.cs @@ -0,0 +1,184 @@ +using NativeMcp.Editor.Helpers; +using NativeMcp.Editor.Tools.EditorControl; +using Newtonsoft.Json.Linq; +using NUnit.Framework; + +#if NATIVE_MCP_HAS_INPUT_SYSTEM +using NativeMcp.Editor.Tools.Input; +#endif + +namespace NativeMcp.Editor.Tests +{ + [TestFixture] + public class InputToolTests + { +#if NATIVE_MCP_HAS_INPUT_SYSTEM + + // ── SimulateInput: Play Mode guard ── + + [Test] + public void KeyDown_NotInPlayMode_ReturnsError() + { + var result = SimulateInput.HandleCommand(new JObject + { + ["action"] = "key_down", + ["key"] = "w" + }); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("Play Mode")); + } + + [Test] + public void ReleaseAll_NotInPlayMode_ReturnsError() + { + var result = SimulateInput.HandleCommand(new JObject + { + ["action"] = "release_all" + }); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("Play Mode")); + } + + // ── SimulateInput: Parameter validation ── + + [Test] + public void KeyDown_MissingKey_ReturnsError() + { + // Without Play Mode this will hit the guard first, so we test + // that the action is recognized and the guard fires (not "unknown action") + var result = SimulateInput.HandleCommand(new JObject + { + ["action"] = "key_down" + }); + Assert.IsInstanceOf(result); + // Should be Play Mode error, not "unknown action" + Assert.That(((ErrorResponse)result).Error, Does.Contain("Play Mode")); + } + + [Test] + public void KeyDown_InvalidKey_ErrorNotUnknownAction() + { + // Since we're not in play mode, this hits the guard. + // We verify the action is dispatched correctly (not "unknown action") + var result = SimulateInput.HandleCommand(new JObject + { + ["action"] = "key_down", + ["key"] = "___invalid___" + }); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Not.Contain("Unknown action")); + } + + [Test] + public void MouseMove_NotInPlayMode_ReturnsError() + { + var result = SimulateInput.HandleCommand(new JObject + { + ["action"] = "mouse_move" + }); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("Play Mode")); + } + + [Test] + public void MouseScroll_NotInPlayMode_ReturnsError() + { + var result = SimulateInput.HandleCommand(new JObject + { + ["action"] = "mouse_scroll", + ["scroll_y"] = 120 + }); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("Play Mode")); + } + + [Test] + public void Click_NotInPlayMode_ReturnsError() + { + var result = SimulateInput.HandleCommand(new JObject + { + ["action"] = "click", + ["button"] = "left" + }); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("Play Mode")); + } + + [Test] + public void MissingAction_ReturnsError() + { + var result = SimulateInput.HandleCommand(new JObject()); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("Play Mode")); + } + + [Test] + public void UnknownAction_NotInPlayMode_ReturnsPlayModeError() + { + // Play Mode guard fires before action dispatch + var result = SimulateInput.HandleCommand(new JObject + { + ["action"] = "bogus_action" + }); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("Play Mode")); + } + +#endif + + // ── ExecuteMenuItem ── + + [Test] + public void ExecuteMenuItem_MissingPath_ReturnsError() + { + var result = ExecuteMenuItem.HandleCommand(new JObject()); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("menu_path")); + } + + [Test] + public void ExecuteMenuItem_BlacklistedPath_ReturnsError() + { + var result = ExecuteMenuItem.HandleCommand(new JObject + { + ["menu_path"] = "File/Quit" + }); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("blocked")); + } + + [Test] + public void ExecuteMenuItem_BlacklistedPath_CaseInsensitive() + { + var result = ExecuteMenuItem.HandleCommand(new JObject + { + ["menu_path"] = "file/quit" + }); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("blocked")); + } + + [Test] + public void ExecuteMenuItem_ValidPath_Succeeds() + { + // Window/General/Console is always available in the editor + var result = ExecuteMenuItem.HandleCommand(new JObject + { + ["menu_path"] = "Window/General/Console" + }); + Assert.IsInstanceOf(result); + Assert.That(((SuccessResponse)result).Message, Does.Contain("Executed")); + } + + [Test] + public void ExecuteMenuItem_InvalidPath_ReturnsError() + { + var result = ExecuteMenuItem.HandleCommand(new JObject + { + ["menu_path"] = "This/Menu/Does/Not/Exist/At/All" + }); + Assert.IsInstanceOf(result); + Assert.That(((ErrorResponse)result).Error, Does.Contain("Failed")); + } + } +} diff --git a/Tests/Editor/InputToolTests.cs.meta b/Tests/Editor/InputToolTests.cs.meta new file mode 100644 index 0000000..801df2f --- /dev/null +++ b/Tests/Editor/InputToolTests.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 7f58cfd3252a146b9bdbc8e012f007f7 \ No newline at end of file diff --git a/Tests/Editor/NativeMcp.Editor.Tests.asmdef b/Tests/Editor/NativeMcp.Editor.Tests.asmdef index 297fd55..6bad2fb 100644 --- a/Tests/Editor/NativeMcp.Editor.Tests.asmdef +++ b/Tests/Editor/NativeMcp.Editor.Tests.asmdef @@ -19,6 +19,12 @@ "defineConstraints": [ "UNITY_INCLUDE_TESTS" ], - "versionDefines": [], + "versionDefines": [ + { + "name": "com.unity.inputsystem", + "expression": "1.0.0", + "define": "NATIVE_MCP_HAS_INPUT_SYSTEM" + } + ], "noEngineReferences": false } From 7bc67fe705775697f98119cb58d2a660a4950a68 Mon Sep 17 00:00:00 2001 From: Shino Mailer Date: Sun, 5 Apr 2026 03:26:12 +0800 Subject: [PATCH 5/5] refactor: improve input simulation maintainability and robustness - Add [InitializeOnLoad] ReloadWatcher to reset static state on domain reload, preventing stale reflection hooks after recompilation - Replace Debug.Log/Warn/Error with McpLog to follow project convention - Extract ConvertScreenshotY() helper and apply Y-flip in HandleClick (was only done in HandleMouseMove, causing click coordinate mismatch) - Cache MethodInfo/target/args array in NativeInputInterceptor to avoid per-frame reflection extraction and object[] allocation - Narrow bare catch to ArgumentException in HandleReleaseAll - Add McpLog.Warn to SceneScreenshot.FocusGameView bare catch - Cache GameView type reflection lookup in static field Co-Authored-By: Claude Opus 4.6 (1M context) --- Editor/Tools/Input/SimulateInput.cs | 106 +++++++++++++++++--------- Editor/Tools/Scene/SceneScreenshot.cs | 3 +- 2 files changed, 74 insertions(+), 35 deletions(-) diff --git a/Editor/Tools/Input/SimulateInput.cs b/Editor/Tools/Input/SimulateInput.cs index 766dd2f..b2ed5af 100644 --- a/Editor/Tools/Input/SimulateInput.cs +++ b/Editor/Tools/Input/SimulateInput.cs @@ -17,19 +17,42 @@ namespace NativeMcp.Editor.Tools.Input public static class SimulateInput { // ── Native input interception ───────────────────────────────── - // The real problem: each frame, NativeInputRuntime delivers hardware - // events that overwrite any state we injected via InputState.Change. - // Solution: hook NativeInputRuntime.instance.onUpdate via reflection - // and discard native hardware events while MCP has synthetic input active. - // This is the same technique Unity's own InputTestFixture uses. + // Each frame, NativeInputRuntime delivers hardware events that overwrite + // any state we injected via InputState.Change. We hook onUpdate via + // reflection, let hardware events through, then overwrite keyboard/mouse + // state with our synthetic values. Same technique as InputTestFixture. private static readonly HashSet s_heldKeys = new HashSet(); private static readonly HashSet s_heldMouseButtons = new HashSet(); private static bool s_nativeHookInstalled; private static Delegate s_originalOnUpdate; + private static MethodInfo s_originalOnUpdateMethod; + private static object s_originalOnUpdateTarget; + private static readonly object[] s_interceptorArgs = new object[2]; private static PropertyInfo s_onUpdateProp; private static object s_nativeRuntimeInstance; + // ── Domain reload safety ───────────────────────────────────── + // Static fields survive domain reload but become stale (reflection + // targets point to dead instances). Reset everything so the hook + // is lazily re-installed on the next unity_input call. + [InitializeOnLoad] + private static class ReloadWatcher + { + static ReloadWatcher() + { + s_heldKeys.Clear(); + s_heldMouseButtons.Clear(); + s_nativeHookInstalled = false; + s_originalOnUpdate = null; + s_originalOnUpdateMethod = null; + s_originalOnUpdateTarget = null; + s_onUpdateProp = null; + s_nativeRuntimeInstance = null; + s_inputRouteConfigured = false; + } + } + private static bool HasSyntheticInput => s_heldKeys.Count > 0 || s_heldMouseButtons.Count > 0; /// @@ -51,7 +74,7 @@ private static void EnsureNativeInputHook() "UnityEngine.InputSystem.LowLevel.NativeInputRuntime"); if (nativeRuntimeType == null) { - Debug.LogWarning("[SimulateInput] Cannot find NativeInputRuntime type."); + McpLog.Warn("[SimulateInput] Cannot find NativeInputRuntime type."); return; } @@ -60,7 +83,7 @@ private static void EnsureNativeInputHook() s_nativeRuntimeInstance = instanceField?.GetValue(null); if (s_nativeRuntimeInstance == null) { - Debug.LogWarning("[SimulateInput] NativeInputRuntime.instance is null."); + McpLog.Warn("[SimulateInput] NativeInputRuntime.instance is null."); return; } @@ -69,7 +92,7 @@ private static void EnsureNativeInputHook() BindingFlags.Public | BindingFlags.Instance | BindingFlags.NonPublic); if (s_onUpdateProp == null) { - Debug.LogWarning("[SimulateInput] Cannot find onUpdate property."); + McpLog.Warn("[SimulateInput] Cannot find onUpdate property."); return; } @@ -82,7 +105,7 @@ private static void EnsureNativeInputHook() "UnityEngine.InputSystem.LowLevel.InputUpdateDelegate"); if (delegateType == null) { - Debug.LogWarning("[SimulateInput] Cannot find InputUpdateDelegate type."); + McpLog.Warn("[SimulateInput] Cannot find InputUpdateDelegate type."); return; } @@ -92,14 +115,18 @@ private static void EnsureNativeInputHook() s_onUpdateProp.SetValue(s_nativeRuntimeInstance, interceptDelegate); + // Cache method/target for fast invocation in the per-frame interceptor + s_originalOnUpdateMethod = s_originalOnUpdate?.Method; + s_originalOnUpdateTarget = s_originalOnUpdate?.Target; + EditorApplication.playModeStateChanged += OnPlayModeChanged; s_nativeHookInstalled = true; - Debug.Log("[SimulateInput] Native input hook installed successfully."); + McpLog.Info("[SimulateInput] Native input hook installed successfully."); } catch (Exception ex) { - Debug.LogError($"[SimulateInput] Failed to install native input hook: {ex}"); + McpLog.Error($"[SimulateInput] Failed to install native input hook: {ex}"); } } @@ -116,14 +143,14 @@ private static void NativeInputInterceptor(InputUpdateType updateType, ref Input // Instead, let all events through normally, then overwrite keyboard/mouse // state with our synthetic values AFTER the update processes them. - // Call the original handler (triggers InputSystem.Update internally) - if (s_originalOnUpdate != null) + // Call the original handler (triggers InputSystem.Update internally). + // Uses cached MethodInfo/target to avoid per-frame reflection overhead. + if (s_originalOnUpdateMethod != null) { - var method = s_originalOnUpdate.Method; - var target = s_originalOnUpdate.Target; - var args = new object[] { updateType, eventBuffer }; - method.Invoke(target, args); - eventBuffer = (InputEventBuffer)args[1]; + s_interceptorArgs[0] = updateType; + s_interceptorArgs[1] = eventBuffer; + s_originalOnUpdateMethod.Invoke(s_originalOnUpdateTarget, s_interceptorArgs); + eventBuffer = (InputEventBuffer)s_interceptorArgs[1]; } if (HasSyntheticInput) @@ -382,18 +409,10 @@ private static object HandleMouseMove(JObject @params) // UGUI's GraphicRaycaster depends on Screen dimensions for hit testing. FocusGameView(); - // Convert from screenshot/image coordinates (top-left origin) - // to InputSystem screen coordinates (bottom-left origin). - // Use Camera.pixelHeight which is always correct, unlike Screen.height - // which returns the focused Editor window's height (known Unity Editor bug). - if (x.HasValue || y.HasValue) - { - var cam = Camera.main; - if (cam != null && y.HasValue) - { - y = cam.pixelHeight - y.Value; - } - } + // Convert Y from screenshot space (top-left origin) to + // InputSystem screen space (bottom-left origin). + if (y.HasValue) + y = ConvertScreenshotY(y.Value); using (StateEvent.From(mouse, out var eventPtr)) { @@ -461,6 +480,10 @@ private static object HandleClick(JObject @params) if (x.HasValue || y.HasValue) { + FocusGameView(); + if (y.HasValue) + y = ConvertScreenshotY(y.Value); + using (StateEvent.From(mouse, out var eventPtr)) { var currentPos = mouse.position.ReadValue(); @@ -500,7 +523,7 @@ private static object HandleReleaseAll() keyboard[key].WriteValueIntoEvent(0f, eventPtr); released++; } - catch { /* Some enum values may not map to physical keys */ } + catch (ArgumentException) { /* Some enum values may not map to physical keys */ } } InputState.Change(keyboard, eventPtr); } @@ -552,11 +575,26 @@ private static void EnsureInputRoutesToGameView() // It interferes with screenshot capture. Mouse actions call it explicitly. } + private static Type s_gameViewType; + private static Type GameViewType => s_gameViewType ??= + typeof(EditorWindow).Assembly.GetType("UnityEditor.GameView"); + private static void FocusGameView() { - var gameViewType = typeof(EditorWindow).Assembly.GetType("UnityEditor.GameView"); - if (gameViewType != null) - EditorWindow.FocusWindowIfItsOpen(gameViewType); + if (GameViewType != null) + EditorWindow.FocusWindowIfItsOpen(GameViewType); + } + + /// + /// Convert Y from screenshot space (top-left origin) to + /// InputSystem screen space (bottom-left origin). + /// Uses Camera.pixelHeight which is always correct, unlike Screen.height + /// which returns the focused Editor window's height (known Unity Editor bug). + /// + private static float ConvertScreenshotY(float y) + { + var cam = Camera.main; + return cam != null ? cam.pixelHeight - y : y; } private static void QueueKeyState(Key key, bool pressed) diff --git a/Editor/Tools/Scene/SceneScreenshot.cs b/Editor/Tools/Scene/SceneScreenshot.cs index 7c86f85..485fb0c 100644 --- a/Editor/Tools/Scene/SceneScreenshot.cs +++ b/Editor/Tools/Scene/SceneScreenshot.cs @@ -366,8 +366,9 @@ private static EditorWindow FocusGameView() gameView?.Focus(); return gameView; } - catch + catch (Exception ex) { + McpLog.Warn($"[SceneScreenshot] FocusGameView failed: {ex.Message}"); return null; } }