From bc36ff70830e43f3eb2033b20381bdc0d1f6a166 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 18 Jul 2026 12:30:06 -0700 Subject: [PATCH] =?UTF-8?q?bench:=20Unreal-scale=20live=20benchmark=20?= =?UTF-8?q?=E2=80=94=20Epic's=20real=20830=20UE=205.8=20schemas=20replayed?= =?UTF-8?q?=20(Opus=204.8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replays the actual tool schemas captured from Epic's UE 5.8 ModelContextProtocol + AllToolsets plugins (830 tools / 52 toolsets) as live registry tools with mocked editor responses, then benchmarks eager vs bare-bridge vs bridge+listing at two scales (62-tool editor subset, full 830) on Claude Opus 4.8 (1M ctx; eager at 830 does not fit any 200K model — first call requests ~266K tokens). Headline (full 830, mean per task, rescored): eager 8/8 at 810,578 input tokens ($4.05); bare bridge 16/16 at 160,844 ($0.80); listing 16/16 at 257,264 ($1.29). Frontier model erases the accuracy gap in every mode; cost is the differentiator. At 62 tools eager wins on cost — consistent with the auto-threshold design. Also parameterizes livetest harness model + listing_max_tokens via env/args (TS_UE_MODEL, TS_UE_SCALE, TS_UE_MODES, TS_UE_LISTING_MAX). --- mcp-research-data/ue_bench_rows.json | 1576 +++++++++++++++++++++++ mcp-research-data/ue_bench_summary.json | 50 + scripts/tool_search_livetest.py | 7 +- scripts/tool_search_livetest_ue.py | 295 +++++ 4 files changed, 1926 insertions(+), 2 deletions(-) create mode 100644 mcp-research-data/ue_bench_rows.json create mode 100644 mcp-research-data/ue_bench_summary.json create mode 100644 scripts/tool_search_livetest_ue.py diff --git a/mcp-research-data/ue_bench_rows.json b/mcp-research-data/ue_bench_rows.json new file mode 100644 index 000000000000..1d8941fa836a --- /dev/null +++ b/mcp-research-data/ue_bench_rows.json @@ -0,0 +1,1576 @@ +[ + { + "scenario_id": "U1_spawn_named", + "mode": "listing", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 24.85, + "api_calls": 5, + "prompt_tokens_total": 159590, + "completion_tokens_total": 1182, + "underlying_tools_called": [ + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U1_spawn_named", + "mode": "listing", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 56.03, + "api_calls": 13, + "prompt_tokens_total": 433445, + "completion_tokens_total": 2504, + "underlying_tools_called": [ + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_remove_from_scene", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_remove_from_scene", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_remove_from_scene", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_remove_from_scene", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U1_spawn_named", + "mode": "bridge", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 13.6, + "api_calls": 5, + "prompt_tokens_total": 148004, + "completion_tokens_total": 406, + "underlying_tools_called": [ + "skills_list", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U1_spawn_named", + "mode": "bridge", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 14.24, + "api_calls": 5, + "prompt_tokens_total": 148041, + "completion_tokens_total": 432, + "underlying_tools_called": [ + "skill_manage", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U1_spawn_named", + "mode": "eager", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 8.59, + "api_calls": 2, + "prompt_tokens_total": 107981, + "completion_tokens_total": 216, + "underlying_tools_called": [ + "skill_manage", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "skills_list" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U1_spawn_named", + "mode": "eager", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 7.78, + "api_calls": 2, + "prompt_tokens_total": 107995, + "completion_tokens_total": 216, + "underlying_tools_called": [ + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U2_viewport_shot", + "mode": "listing", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 7.59, + "api_calls": 2, + "prompt_tokens_total": 62771, + "completion_tokens_total": 140, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_CaptureViewport", + "skill_manage", + "EditorToolset_EditorAppToolset_CaptureViewport" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U2_viewport_shot", + "mode": "listing", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 6.41, + "api_calls": 2, + "prompt_tokens_total": 62751, + "completion_tokens_total": 126, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_CaptureViewport", + "skill_manage", + "EditorToolset_EditorAppToolset_CaptureViewport" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U2_viewport_shot", + "mode": "bridge", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 17.94, + "api_calls": 5, + "prompt_tokens_total": 144355, + "completion_tokens_total": 335, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_CaptureEditorImage", + "EditorToolset_EditorAppToolset_CaptureEditorImage" + ], + "success": false, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U2_viewport_shot", + "mode": "bridge", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 13.06, + "api_calls": 4, + "prompt_tokens_total": 113753, + "completion_tokens_total": 267, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_CaptureEditorImage", + "EditorToolset_EditorAppToolset_CaptureEditorImage" + ], + "success": false, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U2_viewport_shot", + "mode": "eager", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 8.75, + "api_calls": 2, + "prompt_tokens_total": 107831, + "completion_tokens_total": 76, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_CaptureViewport" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U2_viewport_shot", + "mode": "eager", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 9.59, + "api_calls": 2, + "prompt_tokens_total": 107819, + "completion_tokens_total": 76, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_CaptureViewport" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U3_play_mode", + "mode": "listing", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 9.71, + "api_calls": 3, + "prompt_tokens_total": 94354, + "completion_tokens_total": 208, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE", + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U3_play_mode", + "mode": "listing", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 10.64, + "api_calls": 3, + "prompt_tokens_total": 94335, + "completion_tokens_total": 203, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE", + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U3_play_mode", + "mode": "bridge", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 13.72, + "api_calls": 4, + "prompt_tokens_total": 114079, + "completion_tokens_total": 273, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE", + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U3_play_mode", + "mode": "bridge", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 10.96, + "api_calls": 4, + "prompt_tokens_total": 114165, + "completion_tokens_total": 304, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE", + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U3_play_mode", + "mode": "eager", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 10.69, + "api_calls": 3, + "prompt_tokens_total": 162003, + "completion_tokens_total": 167, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U3_play_mode", + "mode": "eager", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 11.23, + "api_calls": 3, + "prompt_tokens_total": 162012, + "completion_tokens_total": 167, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U4_selection_para", + "mode": "listing", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 7.01, + "api_calls": 2, + "prompt_tokens_total": 62755, + "completion_tokens_total": 157, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_GetSelectedActors", + "EditorToolset_EditorAppToolset_GetSelectedActors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U4_selection_para", + "mode": "listing", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 7.01, + "api_calls": 2, + "prompt_tokens_total": 62755, + "completion_tokens_total": 157, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_GetSelectedActors", + "EditorToolset_EditorAppToolset_GetSelectedActors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U4_selection_para", + "mode": "bridge", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 9.4, + "api_calls": 3, + "prompt_tokens_total": 84959, + "completion_tokens_total": 212, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_GetSelectedActors", + "EditorToolset_EditorAppToolset_GetSelectedActors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U4_selection_para", + "mode": "bridge", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 9.2, + "api_calls": 3, + "prompt_tokens_total": 84959, + "completion_tokens_total": 212, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_GetSelectedActors", + "EditorToolset_EditorAppToolset_GetSelectedActors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U4_selection_para", + "mode": "eager", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 7.26, + "api_calls": 2, + "prompt_tokens_total": 107829, + "completion_tokens_total": 77, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_GetSelectedActors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U4_selection_para", + "mode": "eager", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 8.01, + "api_calls": 2, + "prompt_tokens_total": 107835, + "completion_tokens_total": 77, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_GetSelectedActors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U5_shape_chain", + "mode": "listing", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 66.62, + "api_calls": 15, + "prompt_tokens_total": 507680, + "completion_tokens_total": 3428, + "underlying_tools_called": [ + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_set_label", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "EditorToolset_EditorAppToolset_FocusOnActors", + "EditorToolset_EditorAppToolset_CaptureViewport", + "vision_analyze", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_set_label", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "EditorToolset_EditorAppToolset_FocusOnActors", + "EditorToolset_EditorAppToolset_CaptureViewport" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U5_shape_chain", + "mode": "listing", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 51.89, + "api_calls": 11, + "prompt_tokens_total": 360889, + "completion_tokens_total": 2522, + "underlying_tools_called": [ + "skills_list", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "skill_manage", + "editor_toolset_toolsets_actor_ActorTools_get_label", + "EditorToolset_EditorAppToolset_SelectActors", + "EditorToolset_EditorAppToolset_GetSelectedActors", + "skill_manage", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "EditorToolset_EditorAppToolset_GetSelectedActors", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_get_label", + "EditorToolset_EditorAppToolset_SelectActors", + "EditorToolset_EditorAppToolset_GetSelectedActors", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "EditorToolset_EditorAppToolset_GetSelectedActors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U5_shape_chain", + "mode": "bridge", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 82.52, + "api_calls": 15, + "prompt_tokens_total": 558229, + "completion_tokens_total": 3085, + "underlying_tools_called": [ + "skills_list", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "skill_manage", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "skill_manage", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U5_shape_chain", + "mode": "bridge", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 31.74, + "api_calls": 6, + "prompt_tokens_total": 197409, + "completion_tokens_total": 1057, + "underlying_tools_called": [ + "skills_list", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "skill_manage", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U5_shape_chain", + "mode": "eager", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 14.31, + "api_calls": 3, + "prompt_tokens_total": 162344, + "completion_tokens_total": 363, + "underlying_tools_called": [ + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "skill_manage" + ], + "success": false, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U5_shape_chain", + "mode": "eager", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 13.66, + "api_calls": 3, + "prompt_tokens_total": 162337, + "completion_tokens_total": 370, + "underlying_tools_called": [ + "skills_list", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube" + ], + "success": false, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U6_impossible", + "mode": "listing", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 4.44, + "api_calls": 1, + "prompt_tokens_total": 31316, + "completion_tokens_total": 68, + "underlying_tools_called": [ + "skill_manage" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U6_impossible", + "mode": "listing", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 5.63, + "api_calls": 1, + "prompt_tokens_total": 31316, + "completion_tokens_total": 55, + "underlying_tools_called": [], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U6_impossible", + "mode": "bridge", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 3.89, + "api_calls": 1, + "prompt_tokens_total": 27902, + "completion_tokens_total": 43, + "underlying_tools_called": [], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U6_impossible", + "mode": "bridge", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 4.38, + "api_calls": 1, + "prompt_tokens_total": 27896, + "completion_tokens_total": 66, + "underlying_tools_called": [ + "skill_manage" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U6_impossible", + "mode": "eager", + "scale": "editor", + "rep": 1, + "n_tools_registered": 62, + "elapsed_seconds": 4.25, + "api_calls": 1, + "prompt_tokens_total": 53877, + "completion_tokens_total": 41, + "underlying_tools_called": [], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U6_impossible", + "mode": "eager", + "scale": "editor", + "rep": 2, + "n_tools_registered": 62, + "elapsed_seconds": 4.29, + "api_calls": 1, + "prompt_tokens_total": 53886, + "completion_tokens_total": 37, + "underlying_tools_called": [], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U1_spawn_named", + "mode": "listing", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 13.72, + "api_calls": 2, + "prompt_tokens_total": 109282, + "completion_tokens_total": 174, + "underlying_tools_called": [ + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U1_spawn_named", + "mode": "listing", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 8.08, + "api_calls": 2, + "prompt_tokens_total": 109275, + "completion_tokens_total": 172, + "underlying_tools_called": [ + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U1_spawn_named", + "mode": "bridge", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 16.0, + "api_calls": 5, + "prompt_tokens_total": 148999, + "completion_tokens_total": 425, + "underlying_tools_called": [ + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U1_spawn_named", + "mode": "bridge", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 39.72, + "api_calls": 8, + "prompt_tokens_total": 248221, + "completion_tokens_total": 1310, + "underlying_tools_called": [ + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U2_viewport_shot", + "mode": "listing", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 7.99, + "api_calls": 2, + "prompt_tokens_total": 109237, + "completion_tokens_total": 138, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_CaptureViewport", + "EditorToolset_EditorAppToolset_CaptureViewport" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U2_viewport_shot", + "mode": "listing", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 8.69, + "api_calls": 2, + "prompt_tokens_total": 109231, + "completion_tokens_total": 141, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_CaptureViewport", + "EditorToolset_EditorAppToolset_CaptureViewport" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U2_viewport_shot", + "mode": "bridge", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 22.04, + "api_calls": 7, + "prompt_tokens_total": 210837, + "completion_tokens_total": 629, + "underlying_tools_called": [ + "browser_navigate", + "EditorToolset_EditorAppToolset_CaptureViewport", + "EditorToolset_EditorAppToolset_CaptureViewport" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U2_viewport_shot", + "mode": "bridge", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 18.61, + "api_calls": 6, + "prompt_tokens_total": 180004, + "completion_tokens_total": 531, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_CaptureViewport", + "EditorToolset_EditorAppToolset_CaptureViewport" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U3_play_mode", + "mode": "listing", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 11.41, + "api_calls": 3, + "prompt_tokens_total": 164049, + "completion_tokens_total": 203, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE", + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U3_play_mode", + "mode": "listing", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 9.06, + "api_calls": 3, + "prompt_tokens_total": 164040, + "completion_tokens_total": 203, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE", + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U3_play_mode", + "mode": "bridge", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 26.44, + "api_calls": 5, + "prompt_tokens_total": 147589, + "completion_tokens_total": 459, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE", + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U3_play_mode", + "mode": "bridge", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 16.56, + "api_calls": 5, + "prompt_tokens_total": 147040, + "completion_tokens_total": 410, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE", + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U4_selection_para", + "mode": "listing", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 6.76, + "api_calls": 2, + "prompt_tokens_total": 109227, + "completion_tokens_total": 139, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_GetSelectedActors", + "EditorToolset_EditorAppToolset_GetSelectedActors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U4_selection_para", + "mode": "listing", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 8.56, + "api_calls": 2, + "prompt_tokens_total": 109237, + "completion_tokens_total": 145, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_GetSelectedActors", + "EditorToolset_EditorAppToolset_GetSelectedActors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U4_selection_para", + "mode": "bridge", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 9.54, + "api_calls": 3, + "prompt_tokens_total": 85070, + "completion_tokens_total": 224, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_GetSelectedActors", + "EditorToolset_EditorAppToolset_GetSelectedActors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U4_selection_para", + "mode": "bridge", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 9.13, + "api_calls": 3, + "prompt_tokens_total": 85047, + "completion_tokens_total": 211, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_GetSelectedActors", + "EditorToolset_EditorAppToolset_GetSelectedActors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U5_shape_chain", + "mode": "listing", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 67.39, + "api_calls": 13, + "prompt_tokens_total": 734288, + "completion_tokens_total": 3180, + "underlying_tools_called": [ + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_asset_AssetTools_exists", + "editor_toolset_toolsets_actor_ActorTools_add_component", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_object_ObjectTools_list_properties", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_object_ObjectTools_get_properties", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_object_ObjectTools_set_properties", + "EditorToolset_EditorAppToolset_FocusOnActors", + "EditorToolset_EditorAppToolset_CaptureViewport", + "vision_analyze", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_asset_AssetTools_exists", + "editor_toolset_toolsets_actor_ActorTools_add_component", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_object_ObjectTools_list_properties", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_object_ObjectTools_get_properties", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_object_ObjectTools_set_properties", + "EditorToolset_EditorAppToolset_FocusOnActors", + "EditorToolset_EditorAppToolset_CaptureViewport" + ], + "success": false, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U5_shape_chain", + "mode": "listing", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 90.57, + "api_calls": 15, + "prompt_tokens_total": 878311, + "completion_tokens_total": 3566, + "underlying_tools_called": [ + "skills_list", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_actor_ActorTools_add_component", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "skill_manage", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "EditorToolset_EditorAppToolset_SelectActors", + "editor_toolset_toolsets_object_ObjectTools_get_properties", + "EditorToolset_EditorAppToolset_GetSelectedActors", + "editor_toolset_toolsets_object_ObjectTools_get_properties", + "EditorToolset_EditorAppToolset_FocusOnActors", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_actor_ActorTools_add_component", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "EditorToolset_EditorAppToolset_SelectActors", + "editor_toolset_toolsets_object_ObjectTools_get_properties", + "EditorToolset_EditorAppToolset_GetSelectedActors", + "editor_toolset_toolsets_object_ObjectTools_get_properties", + "EditorToolset_EditorAppToolset_FocusOnActors" + ], + "success": false, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U5_shape_chain", + "mode": "bridge", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 30.59, + "api_calls": 6, + "prompt_tokens_total": 208197, + "completion_tokens_total": 1121, + "underlying_tools_called": [ + "skills_list", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U5_shape_chain", + "mode": "bridge", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 34.15, + "api_calls": 7, + "prompt_tokens_total": 236442, + "completion_tokens_total": 1154, + "underlying_tools_called": [ + "skill_manage", + "skills_list", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", + "skill_manage", + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", + "editor_toolset_toolsets_actor_ActorTools_set_actor_transform" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U6_impossible", + "mode": "listing", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 4.47, + "api_calls": 1, + "prompt_tokens_total": 54554, + "completion_tokens_total": 55, + "underlying_tools_called": [], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U6_impossible", + "mode": "listing", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 4.37, + "api_calls": 1, + "prompt_tokens_total": 54554, + "completion_tokens_total": 60, + "underlying_tools_called": [], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U6_impossible", + "mode": "bridge", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 4.61, + "api_calls": 1, + "prompt_tokens_total": 27902, + "completion_tokens_total": 70, + "underlying_tools_called": [ + "skill_manage" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U6_impossible", + "mode": "bridge", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 4.34, + "api_calls": 1, + "prompt_tokens_total": 27899, + "completion_tokens_total": 61, + "underlying_tools_called": [], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U7_deep_cut", + "mode": "listing", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 33.57, + "api_calls": 5, + "prompt_tokens_total": 275130, + "completion_tokens_total": 1330, + "underlying_tools_called": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetSystemData", + "editor_toolset_toolsets_asset_AssetTools_exists", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetSystemData", + "editor_toolset_toolsets_asset_AssetTools_exists" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U7_deep_cut", + "mode": "listing", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 28.07, + "api_calls": 6, + "prompt_tokens_total": 347886, + "completion_tokens_total": 1103, + "underlying_tools_called": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetSystemData", + "NiagaraToolsets_NiagaraToolset_System_GetSystemSummary", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetSystemData", + "NiagaraToolsets_NiagaraToolset_System_GetSystemSummary" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U7_deep_cut", + "mode": "bridge", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 18.36, + "api_calls": 4, + "prompt_tokens_total": 124101, + "completion_tokens_total": 628, + "underlying_tools_called": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U7_deep_cut", + "mode": "bridge", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 24.2, + "api_calls": 6, + "prompt_tokens_total": 190469, + "completion_tokens_total": 1085, + "underlying_tools_called": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U8_console_trap", + "mode": "listing", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 32.83, + "api_calls": 6, + "prompt_tokens_total": 335905, + "completion_tokens_total": 1229, + "underlying_tools_called": [ + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U8_console_trap", + "mode": "listing", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 45.94, + "api_calls": 8, + "prompt_tokens_total": 452020, + "completion_tokens_total": 1745, + "underlying_tools_called": [ + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U8_console_trap", + "mode": "bridge", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 35.71, + "api_calls": 7, + "prompt_tokens_total": 216937, + "completion_tokens_total": 1520, + "underlying_tools_called": [ + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U8_console_trap", + "mode": "bridge", + "scale": "full", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 49.24, + "api_calls": 9, + "prompt_tokens_total": 288757, + "completion_tokens_total": 1996, + "underlying_tools_called": [ + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", + "EditorToolset_EditorAppToolset_SearchCVars", + "EditorToolset_EditorAppToolset_GetCameraTransform", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", + "EditorToolset_EditorAppToolset_SearchCVars", + "EditorToolset_EditorAppToolset_GetCameraTransform" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U1_spawn_named", + "mode": "eager", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 15.89, + "api_calls": 2, + "prompt_tokens_total": 648323, + "completion_tokens_total": 215, + "underlying_tools_called": [ + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U2_viewport_shot", + "mode": "eager", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 14.7, + "api_calls": 2, + "prompt_tokens_total": 648161, + "completion_tokens_total": 88, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_CaptureViewport" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U3_play_mode", + "mode": "eager", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 20.53, + "api_calls": 3, + "prompt_tokens_total": 972457, + "completion_tokens_total": 151, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_StartPIE", + "EditorToolset_EditorAppToolset_StopPIE" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U4_selection_para", + "mode": "eager", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 17.38, + "api_calls": 2, + "prompt_tokens_total": 648169, + "completion_tokens_total": 104, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_GetSelectedActors" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U5_shape_chain", + "mode": "eager", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 21.85, + "api_calls": 3, + "prompt_tokens_total": 972831, + "completion_tokens_total": 373, + "underlying_tools_called": [ + "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", + "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube" + ], + "success": false, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U6_impossible", + "mode": "eager", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 7.93, + "api_calls": 1, + "prompt_tokens_total": 324045, + "completion_tokens_total": 45, + "underlying_tools_called": [], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U7_deep_cut", + "mode": "eager", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 33.64, + "api_calls": 5, + "prompt_tokens_total": 1622364, + "completion_tokens_total": 767, + "underlying_tools_called": [ + "editor_toolset_toolsets_asset_AssetTools_load_asset", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "editor_toolset_toolsets_asset_AssetTools_save_assets" + ], + "success": true, + "error": null, + "success_rescored": true + }, + { + "scenario_id": "U8_console_trap", + "mode": "eager", + "scale": "full", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 19.59, + "api_calls": 2, + "prompt_tokens_total": 648270, + "completion_tokens_total": 437, + "underlying_tools_called": [ + "EditorToolset_EditorAppToolset_SearchCVars" + ], + "success": true, + "error": null, + "success_rescored": true + } +] \ No newline at end of file diff --git a/mcp-research-data/ue_bench_summary.json b/mcp-research-data/ue_bench_summary.json new file mode 100644 index 000000000000..bf7eaff55f85 --- /dev/null +++ b/mcp-research-data/ue_bench_summary.json @@ -0,0 +1,50 @@ +{ + "editor|eager": { + "succ": 12, + "n": 12, + "api": 2.2, + "itok": 116979, + "usd": 0.58, + "sec": 9.0 + }, + "editor|bridge": { + "succ": 12, + "n": 12, + "api": 4.7, + "itok": 146979, + "usd": 0.73, + "sec": 18.7 + }, + "editor|listing": { + "succ": 12, + "n": 12, + "api": 5, + "itok": 163663, + "usd": 0.82, + "sec": 21.5 + }, + "full|eager": { + "succ": 8, + "n": 8, + "api": 2.5, + "itok": 810578, + "usd": 4.05, + "sec": 18.9 + }, + "full|bridge": { + "succ": 16, + "n": 16, + "api": 5.2, + "itok": 160844, + "usd": 0.8, + "sec": 22.5 + }, + "full|listing": { + "succ": 16, + "n": 16, + "api": 4.6, + "itok": 257264, + "usd": 1.29, + "sec": 23.8 + } +} \ No newline at end of file diff --git a/scripts/tool_search_livetest.py b/scripts/tool_search_livetest.py index d6a9bbeb3a64..1df22ff8c7f4 100644 --- a/scripts/tool_search_livetest.py +++ b/scripts/tool_search_livetest.py @@ -248,7 +248,9 @@ SCENARIOS: List[Dict[str, Any]] = [ # --------------------------------------------------------------------------- -def setup_isolated_home(enabled: bool, listing: str = "off") -> Path: +def setup_isolated_home(enabled: bool, listing: str = "off", + listing_max_tokens: int = 4000, + model: str = "anthropic/claude-haiku-4.5") -> Path: """Create a fresh ~/.hermes/ for one test, copying minimal credentials. Also reads OPENROUTER_API_KEY from the user's real ``~/.hermes/.env`` so @@ -278,7 +280,7 @@ def setup_isolated_home(enabled: bool, listing: str = "off") -> Path: cfg = { "model": { "provider": "openrouter", - "model": "anthropic/claude-haiku-4.5", + "model": model, }, "tools": { "tool_search": { @@ -287,6 +289,7 @@ def setup_isolated_home(enabled: bool, listing: str = "off") -> Path: "search_default_limit": 5, "max_search_limit": 20, "listing": listing, + "listing_max_tokens": listing_max_tokens, }, }, "logging": {"level": "WARNING"}, diff --git a/scripts/tool_search_livetest_ue.py b/scripts/tool_search_livetest_ue.py new file mode 100644 index 000000000000..36199e699dc1 --- /dev/null +++ b/scripts/tool_search_livetest_ue.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""Live benchmark v3: Epic Unreal Engine 5.8 MCP surface (830 REAL schemas), replayed. + +Registers the actual tool schemas captured live from Epic's UE 5.8 +ModelContextProtocol + AllToolsets plugins (probe_raw_5.8.0_alltoolsets.json, +probe date 2026-07-02) into the Hermes tool registry with mock handlers, +then runs UE-realistic scenarios in three modes: + + eager — all schemas in the tools array (at 830 tools: ~165K tokens) + bridge — tool_search bridge, no listing (old behavior) + listing — bridge + skills-style catalog listing (PR #67034) + +Catalog scale is controlled by TS_UE_SCALE: + "editor" — EditorApp + Scene + Primitive + Actor toolsets (~65 tools) + "full" — all 52 toolsets / 830 tools + +Env: TS_BENCH_REPS (default 2), TS_UE_MODES, TS_UE_SCALE, TS_UE_SUMMARY. +""" +from __future__ import annotations + +import json, os, re, shutil, sys, time, traceback +from pathlib import Path +from typing import Any, Dict, List + +_THIS_DIR = Path(__file__).resolve().parent +_WORKTREE_ROOT = _THIS_DIR.parent +sys.path.insert(0, str(_WORKTREE_ROOT)) +sys.path.insert(0, str(_THIS_DIR)) + +import tool_search_livetest as base + +PROBE = "/tmp/ue-bridge-probe/docs/epic_mcp/probe_raw_5.8.0_alltoolsets.json" +N_REPS = int(os.environ.get("TS_BENCH_REPS", "2")) + +EDITOR_TOOLSETS = ( + "EditorToolset.EditorAppToolset", + "editor_toolset.toolsets.scene.SceneTools", + "editor_toolset.toolsets.primitive.PrimitiveTools", + "editor_toolset.toolsets.actor.ActorTools", +) + +_SANITIZE = re.compile(r"[^A-Za-z0-9_]") + + +def _mock_result(tool_name: str) -> str: + """Plausible success payload keyed on verb-ish name shape.""" + short = tool_name.rsplit("_", 1)[-1].lower() + if any(v in tool_name.lower() for v in ("get", "list", "find", "search", "query", "is_", "can_", "checked")): + return json.dumps({"result": [{"name": "Cube_1", "path": "/Game/Level:PersistentLevel.Cube_1", + "class": "StaticMeshActor", "location": [0, 0, 100]}]}) + if "screenshot" in tool_name.lower() or "capture" in tool_name.lower(): + return json.dumps({"result": {"image_path": "/tmp/ue_viewport_0001.png", "width": 1280, "height": 720}}) + return json.dumps({"result": {"ok": True, "op": short, "actor": "/Game/Level:PersistentLevel.Cube_1"}}) + + +def load_epic_tools(scale: str) -> List[Dict[str, Any]]: + raw = json.load(open(PROBE)) + out = [] + for ts_name, ts in raw["toolsets"].items(): + if not isinstance(ts, dict) or not ts.get("tools"): + continue + if scale == "editor" and ts_name not in EDITOR_TOOLSETS: + continue + for t in ts["tools"]: + name = _SANITIZE.sub("_", t.get("name", "")) + if not name: + continue + out.append({ + "name": name, + "description": t.get("description", "") or "", + "parameters": t.get("inputSchema") or {"type": "object", "properties": {}}, + }) + return out + + +def register_epic_tools(scale: str) -> int: + from tools.registry import registry + tools = load_epic_tools(scale) + for tdef in tools: + def make_handler(nm): + def _h(*a, **kw): + return _mock_result(nm) + return _h + registry.register( + name=tdef["name"], + toolset="mcp-unreal", + schema={"name": tdef["name"], "description": tdef["description"], + "parameters": tdef["parameters"]}, + handler=make_handler(tdef["name"]), + ) + return len(tools) + + +# Expected tools use SUBSTRING match against sanitized names (full names are +# long dotted paths, e.g. editor_toolset_toolsets_scene_SceneTools_..._add_to_scene_from_class). +SCENARIOS: List[Dict[str, Any]] = [ + { + "id": "U1_spawn_named", + "description": "Direct ask naming the operation (spawn actor)", + "prompt": ("Spawn a PointLight actor in the level at location x=0 y=0 z=300. " + "Then tell me you're done. Don't do anything else."), + "expected_any": ["add_to_scene_from_class", "spawn"], + }, + { + "id": "U2_viewport_shot", + "description": "Paraphrased capability (viewport capture)", + "prompt": ("Show me what the level currently looks like — grab an image of the " + "editor view and tell me the file path. Nothing else."), + "expected_any": ["CaptureViewport", "Screenshot", "screenshot"], + }, + { + "id": "U3_play_mode", + "description": "Start then stop play-in-editor (2-step, same toolset)", + "prompt": ("Start a play-in-editor session, then immediately stop it, then say done."), + "expected_any": ["StartPIE"], + "expected_any_2": ["StopPIE"], + }, + { + "id": "U4_selection_para", + "description": "Paraphrase, no tool words ('what am I working with')", + "prompt": ("What actors do I currently have selected in the editor? Just list them."), + "expected_any": ["GetSelectedActors", "get_selected"], + }, + { + "id": "U5_shape_chain", + "description": "Multi-step: spawn actor + attach cube shape + move it", + "prompt": ("Create an empty StaticMeshActor called Crate, attach a cube-shaped mesh " + "component to it, and move the actor to x=100 y=200 z=0. Then say done."), + "expected_any": ["add_cube"], + "expected_any_2": ["set_actor_transform", "transform"], + }, + { + "id": "U6_impossible", + "description": "Capability that does NOT exist (honesty check)", + "prompt": ("Order a pepperoni pizza to be delivered to my studio. If you truly can't, " + "reply 'CANNOT: ' plus a one-line reason."), + "expected_any": [], + }, + { + "id": "U7_deep_cut", + "description": "Rarely-used tool buried deep in the catalog (niagara user variable)", + "prompt": ("On the Niagara system asset at /Game/FX/NS_Sparks, add a user-exposed float " + "variable named SpawnRateScale. Then say done."), + "expected_any": ["AddUserVariables", "user_variable", "UserParameter"], + "full_only": True, + }, + { + "id": "U8_console_trap", + "description": "Plausible-but-absent tool (no console-exec exists in Epic's 830)", + "prompt": ("Run the console command 'stat fps' in the editor and tell me what it says. " + "If there is genuinely no way to run console commands, reply 'CANNOT: ' plus why."), + "expected_any": [], + "full_only": True, + }, +] + + +def run_one(scenario, mode, scale, rep, out_dir: Path): + enabled = mode in ("bridge", "listing") + model = os.environ.get("TS_UE_MODEL", "anthropic/claude-opus-4.8") + # 830-tool catalogs need headroom: full listing ~ names+descs won't fit 4K, + # so give the full scale a real budget (names+descs ~ 26K est; names-only ~8K). + lmax = int(os.environ.get("TS_UE_LISTING_MAX", "30000" if scale == "full" else "4000")) + hermes_home = base.setup_isolated_home( + enabled, listing=("auto" if mode == "listing" else "off"), + listing_max_tokens=lmax, model=model) + os.environ["HERMES_HOME"] = str(hermes_home) + base.reset_module_state() + n_registered = register_epic_tools(scale) + + from tools.registry import registry + original_dispatch = registry.dispatch + tool_call_log: List[str] = [] + + def logging_dispatch(name, args, **kw): + tool_call_log.append(name) + return original_dispatch(name, args, **kw) + registry.dispatch = logging_dispatch + + usage_log: List[Dict[str, Any]] = [] + + started = time.time() + error = None + final_response = "" + messages_out: List[Dict[str, Any]] = [] + pm = None + _orig_norm = None + try: + from run_agent import AIAgent + agent = AIAgent( + provider="openrouter", model=model, + quiet_mode=True, save_trajectories=False, + skip_context_files=True, skip_memory=True, + platform="cli", max_iterations=15, + ) + import agent.conversation_loop as _cl + _orig_norm = _cl.normalize_usage + def _norm_spy(raw, **kw): + cu = _orig_norm(raw, **kw) + try: + usage_log.append({"prompt_tokens": cu.prompt_tokens, + "completion_tokens": getattr(cu, "output_tokens", 0) or 0, + "cached_tokens": getattr(cu, "cache_read_tokens", 0) or 0}) + except Exception: + pass + return cu + _cl.normalize_usage = _norm_spy + result = agent.run_conversation( + user_message=scenario["prompt"], + system_message=("You are controlling a live Unreal Engine 5.8 editor. The editor is " + "already running and connected through your Unreal (mcp-unreal) tools — " + "do not try to locate or launch the editor process yourself. " + "Complete the task with the available tools. Be concise."), + ) + if isinstance(result, dict): + final_response = result.get("final_response") or "" + messages_out = result.get("messages") or [] + else: + final_response = str(result) + except Exception: + error = traceback.format_exc() + finally: + registry.dispatch = original_dispatch + if _orig_norm is not None: + try: + import agent.conversation_loop as _cl2 + _cl2.normalize_usage = _orig_norm + except Exception: + pass + + elapsed = time.time() - started + bridge_call_log = base._extract_bridge_calls(messages_out) + called = list(tool_call_log) + for b in bridge_call_log: + if b.get("name") == "tool_call": + inner = (b.get("args") or {}).get("name") + if inner: + called.append(inner) + + def hit(subs): + return any(any(s.lower() in n.lower() for s in subs) for n in called) + + exp1 = scenario.get("expected_any") or [] + exp2 = scenario.get("expected_any_2") + if not exp1: + # honesty scenarios: success = no hallucinated UE tool call claiming to do it + success = (error is None) and ("CANNOT" in (final_response or "").upper() + or "can't" in (final_response or "").lower() + or "cannot" in (final_response or "").lower()) + else: + success = hit(exp1) and (hit(exp2) if exp2 else True) + + rec = { + "scenario_id": scenario["id"], "mode": mode, "scale": scale, "rep": rep, + "n_tools_registered": n_registered, + "elapsed_seconds": round(elapsed, 2), + "api_calls": len(usage_log), + "prompt_tokens_total": sum(u["prompt_tokens"] or 0 for u in usage_log), + "completion_tokens_total": sum(u["completion_tokens"] or 0 for u in usage_log), + "per_call_usage": usage_log, + "bridge_calls": bridge_call_log, + "underlying_tools_called": called[:40], + "success": bool(success), "error": error, + "final_response": base._redact_secrets(final_response)[:400], + } + (out_dir / f"{scenario['id']}__{mode}__{scale}__rep{rep}.json").write_text(json.dumps(rec, indent=1), encoding="utf-8") + shutil.rmtree(Path(os.environ["HERMES_HOME"]).parent, ignore_errors=True) + return rec + + +def main(): + out_dir = _THIS_DIR / "out_ue" + out_dir.mkdir(exist_ok=True) + scale = os.environ.get("TS_UE_SCALE", "full") + modes = [m for m in os.environ.get("TS_UE_MODES", "listing,bridge,eager").split(",") if m] + rows = [] + for scenario in SCENARIOS: + if scenario.get("full_only") and scale != "full": + continue + for mode in modes: + for rep in range(1, N_REPS + 1): + rec = run_one(scenario, mode, scale, rep, out_dir) + print(f"{scenario['id']:18} {mode:8} {scale:6} rep{rep}: api={rec['api_calls']} " + f"in={rec['prompt_tokens_total']:>8,} t={rec['elapsed_seconds']:>6}s " + f"ok={rec['success']} err={bool(rec['error'])}", flush=True) + rows.append(rec) + name = os.environ.get("TS_UE_SUMMARY", f"_ue_bench_{scale}.json") + (out_dir / name).write_text(json.dumps( + [{k: v for k, v in r.items() if k not in ("per_call_usage", "bridge_calls", "final_response")} for r in rows], + indent=1), encoding="utf-8") + print("done ->", out_dir / name) + + +if __name__ == "__main__": + main()