diff --git a/mcp-research-data/ue_hard_haiku_rows.json b/mcp-research-data/ue_hard_haiku_rows.json new file mode 100644 index 000000000000..4e2d4540c219 --- /dev/null +++ b/mcp-research-data/ue_hard_haiku_rows.json @@ -0,0 +1,1642 @@ +[ + { + "scenario_id": "V1_static_material", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 11.19, + "api_calls": 3, + "prompt_tokens_total": 119904, + "first_tool": [ + "set", + "material" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + "bridge_calls": [ + [ + "tool_describe", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + [ + "tool_call", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ] + ], + "error": null, + "final_response": "Done. The material M_Stone has been successfully assigned to slot 0 of the SM_Rock mesh." + }, + { + "scenario_id": "V1_static_material", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 10.89, + "api_calls": 3, + "prompt_tokens_total": 119922, + "first_tool": [ + "set", + "material" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + "bridge_calls": [ + [ + "tool_describe", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + [ + "tool_call", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ] + ], + "error": null, + "final_response": "Done. Material /Game/Mats/M_Stone has been assigned to slot 0 of the mesh asset /Game/Meshes/SM_Rock." + }, + { + "scenario_id": "V1_static_material", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 15.01, + "api_calls": 5, + "prompt_tokens_total": 111559, + "first_tool": [ + "material", + "slots" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + "bridge_calls": [ + [ + "tool_search", + "assign material mesh slot unreal" + ], + [ + "tool_search", + "static mesh material slots get names" + ], + [ + "tool_call", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots" + ], + [ + "tool_call", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V1_static_material", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 9.92, + "api_calls": 3, + "prompt_tokens_total": 65150, + "first_tool": [ + "set", + "material" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + "bridge_calls": [ + [ + "tool_search", + "unreal assign material mesh slot" + ], + [ + "tool_call", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V2_skeletal_material", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 13.95, + "api_calls": 4, + "prompt_tokens_total": 160299, + "first_tool": [ + "material", + "slots" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ], + "bridge_calls": [ + [ + "tool_describe", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ], + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ], + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ] + ], + "error": null, + "final_response": "Done. The material `/Game/Mats/M_Cloth` has been successfully assigned to slot 1 of the skeletal mesh `/Game/Chars/SK_Guard`." + }, + { + "scenario_id": "V2_skeletal_material", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 14.39, + "api_calls": 4, + "prompt_tokens_total": 160307, + "first_tool": [ + "material", + "slots" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ], + "bridge_calls": [ + [ + "tool_describe", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ], + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ], + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V2_skeletal_material", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 18.18, + "api_calls": 5, + "prompt_tokens_total": 111698, + "first_tool": [ + "material", + "slots" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ], + "bridge_calls": [ + [ + "tool_search", + "Unreal Engine assign material slot skeletal mesh" + ], + [ + "tool_describe", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ], + [ + "tool_describe", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ], + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ], + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V2_skeletal_material", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 17.81, + "api_calls": 5, + "prompt_tokens_total": 111344, + "first_tool": [ + "material", + "slots" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ], + "bridge_calls": [ + [ + "tool_search", + "set material skeletal mesh slot" + ], + [ + "tool_describe", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ], + [ + "tool_describe", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ], + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ], + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V3_curvetable_row", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 15.46, + "api_calls": 3, + "prompt_tokens_total": 120710, + "first_tool": [ + "add", + "row" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + "bridge_calls": [ + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ] + ], + "error": null, + "final_response": "Done. Added row 'Heavy' to /Game/Data/CT_Damage with value 42 at time 0." + }, + { + "scenario_id": "V3_curvetable_row", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 14.85, + "api_calls": 4, + "prompt_tokens_total": 159929, + "first_tool": [ + "add", + "row" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + "bridge_calls": [ + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V3_curvetable_row", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 15.27, + "api_calls": 4, + "prompt_tokens_total": 89922, + "first_tool": [ + "add", + "row" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + "bridge_calls": [ + [ + "tool_search", + "Unreal data table add row" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ] + ], + "error": null, + "final_response": "Done. Row 'Heavy' has been added to /Game/Data/CT_Damage with value 42 at time 0." + }, + { + "scenario_id": "V3_curvetable_row", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 41.17, + "api_calls": 14, + "prompt_tokens_total": 336628, + "first_tool": [ + "add", + "rows" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 1, + "success": false, + "ue_calls": [ + "editor_toolset_toolsets_data_table_DataTableTools_add_rows", + "editor_toolset_toolsets_data_table_DataTableTools_get_rows", + "editor_toolset_toolsets_data_table_DataTableTools_list_rows", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + "bridge_calls": [ + [ + "tool_search", + "Unreal data table add row" + ], + [ + "tool_describe", + "editor_toolset_toolsets_data_table_DataTableTools_add_rows" + ], + [ + "tool_describe", + "editor_toolset_toolsets_data_table_DataTableTools_get_rows" + ], + [ + "tool_call", + "editor_toolset_toolsets_data_table_DataTableTools_add_rows" + ], + [ + "tool_search", + "Unreal data table set row value" + ], + [ + "tool_search", + "Unreal data table set cell property" + ], + [ + "tool_describe", + "editor_toolset_toolsets_data_table_DataTableTools_set_rows" + ], + [ + "tool_call", + "editor_toolset_toolsets_data_table_DataTableTools_get_rows" + ], + [ + "tool_search", + "Unreal list data table rows" + ], + [ + "tool_call", + "editor_toolset_toolsets_data_table_DataTableTools_list_rows" + ], + [ + "tool_search", + "Unreal curve table add row" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ] + ], + "error": null, + "final_response": "Done. Row \"Heavy\" has been added to /Game/Data/CT_Damage with value 42 at time 0." + }, + { + "scenario_id": "V4_project_tag", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 11.6, + "api_calls": 3, + "prompt_tokens_total": 119772, + "first_tool": [ + "GameplayTagsToolset", + "AddTag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + "bridge_calls": [ + [ + "tool_describe", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + [ + "tool_call", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ] + ], + "error": null, + "final_response": "Done. The 'Combat.Stun' gameplay tag has been registered in the project's tag registry. Designers can now use it in the Unreal Editor." + }, + { + "scenario_id": "V4_project_tag", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 12.42, + "api_calls": 4, + "prompt_tokens_total": 160853, + "first_tool": [ + "GameplayTagsToolset", + "AddTag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + "bridge_calls": [ + [ + "tool_search", + "gameplay tags add register" + ], + [ + "tool_describe", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + [ + "tool_call", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ] + ], + "error": null, + "final_response": "Done. The gameplay tag 'Combat.Stun' has been registered in the project's tag registry. Designers can now use it in their gameplay systems and effects." + }, + { + "scenario_id": "V4_project_tag", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 19.83, + "api_calls": 6, + "prompt_tokens_total": 136043, + "first_tool": [ + "GameplayTagsToolset", + "AddTag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + "bridge_calls": [ + [ + "tool_search", + "unreal gameplay tag register create" + ], + [ + "tool_search", + "unreal add gameplay tag" + ], + [ + "tool_search", + "GameplayTagsToolset AddTag" + ], + [ + "tool_describe", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + [ + "tool_call", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ] + ], + "error": null, + "final_response": "Done. The gameplay tag 'Combat.Stun' has been successfully registered in your project's tag registry. Designers can now use it throughout the project." + }, + { + "scenario_id": "V4_project_tag", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 21.24, + "api_calls": 7, + "prompt_tokens_total": 156137, + "first_tool": [ + "GameplayTagsToolset", + "AddTag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + "bridge_calls": [ + [ + "tool_search", + "Unreal gameplay tag register" + ], + [ + "tool_search", + "Unreal add gameplay tag registry" + ], + [ + "tool_describe", + "GameplayTagsToolset_GameplayTagsToolset_GetTagInfo" + ], + [ + "tool_search", + "GameplayTagsToolset add" + ], + [ + "tool_describe", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + [ + "tool_call", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ] + ], + "error": null, + "final_response": "Done. The gameplay tag 'Combat.Stun' has been successfully registered in the project's tag registry. Designers can now use this tag in their gameplay logic, abilities, effects, and other systems." + }, + { + "scenario_id": "V5_actor_tag", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 15.35, + "api_calls": 4, + "prompt_tokens_total": 163213, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ], + "bridge_calls": [ + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_describe", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ] + ], + "error": null, + "final_response": "Done. Crate_2 is now tagged with 'loot' and your spawner script can find it." + }, + { + "scenario_id": "V5_actor_tag", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 10.64, + "api_calls": 3, + "prompt_tokens_total": 119525, + "first_tool": [ + "add", + "tag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ], + "bridge_calls": [ + [ + "tool_describe", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ] + ], + "error": null, + "final_response": "Done. The 'loot' tag has been added to Crate_2." + }, + { + "scenario_id": "V5_actor_tag", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 9.75, + "api_calls": 3, + "prompt_tokens_total": 64815, + "first_tool": [ + "add", + "tag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ], + "bridge_calls": [ + [ + "tool_search", + "unreal engine add tag actor" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ] + ], + "error": null, + "final_response": "Done. Crate_2 is now tagged with 'loot' and your spawner script can find it." + }, + { + "scenario_id": "V5_actor_tag", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 10.79, + "api_calls": 3, + "prompt_tokens_total": 64750, + "first_tool": [ + "add", + "tag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ], + "bridge_calls": [ + [ + "tool_search", + "Unreal actor tag" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ] + ], + "error": null, + "final_response": "Done. The actor Crate_2 now has the 'loot' tag and your spawner script can find it." + }, + { + "scenario_id": "V6_niagara_component", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 14.51, + "api_calls": 3, + "prompt_tokens_total": 119497, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ] + ], + "error": null, + "final_response": "Done. Set Torch_3's FX_Flame Niagara component Brightness user parameter to 5.0." + }, + { + "scenario_id": "V6_niagara_component", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 25.28, + "api_calls": 6, + "prompt_tokens_total": 252049, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + "bridge_calls": [ + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_describe", + "editor_toolset_toolsets_actor_ActorTools_get_components" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_get_components" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ] + ], + "error": null, + "final_response": "Done. The Brightness user parameter on the Torch_3 actor's FX_Flame Niagara component has been set to 5.0." + }, + { + "scenario_id": "V6_niagara_component", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 40.63, + "api_calls": 11, + "prompt_tokens_total": 299524, + "first_tool": [ + "execution", + "environment" + ], + "first_correct": false, + "final_correct": false, + "wrong_calls": 0, + "success": false, + "ue_calls": [ + "et_toolsets_programmatic_ProgrammaticToolset_get_execution_environment" + ], + "bridge_calls": [ + [ + "tool_search", + "unreal actor parameter component" + ], + [ + "tool_search", + "get actor by name unreal" + ], + [ + "tool_describe", + "animation_toolset_toolsets_sequencer_SequencerTools_add_actors_by_name" + ], + [ + "tool_describe", + "editor_toolset_toolsets_actor_ActorTools_get_components" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_search", + "get selected actor unreal editor" + ], + [ + "tool_search", + "find actor reference world" + ], + [ + "tool_search", + "actor by label name get world" + ], + [ + "tool_search", + "unreal scene tools spawn actor" + ], + [ + "tool_search", + "EditorLevelLibrary find actor" + ], + [ + "tool_search", + "unreal python execute command" + ], + [ + "tool_describe", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script" + ], + [ + "tool_search", + "get execution environment" + ], + [ + "tool_call", + "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment" + ] + ], + "error": null, + "final_response": "Response truncated due to output length limit" + }, + { + "scenario_id": "V6_niagara_component", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 56.88, + "api_calls": 15, + "prompt_tokens_total": 403694, + "first_tool": [ + "computer", + "use" + ], + "first_correct": false, + "final_correct": false, + "wrong_calls": 0, + "success": false, + "ue_calls": [ + "computer_use", + "computer_use" + ], + "bridge_calls": [ + [ + "tool_search", + "Unreal actor component parameter property set" + ], + [ + "tool_search", + "Unreal find actor user parameter Niagara" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_search", + "Unreal find actor by name instance" + ], + [ + "tool_search", + "Unreal get selected actor level" + ], + [ + "tool_search", + "Unreal actor path reference object by name" + ], + [ + "tool_search", + "Unreal get actor components Niagara component" + ], + [ + "tool_describe", + "editor_toolset_toolsets_actor_ActorTools_get_components" + ], + [ + "tool_search", + "Unreal get actor by label name object path soft reference" + ], + [ + "tool_search", + "Unreal construct soft object reference path" + ], + [ + "tool_describe", + "animation_toolset_toolsets_sequencer_SequencerTools_add_actors_by_name" + ], + [ + "tool_search", + "Unreal world editor command console execute" + ], + [ + "tool_search", + "Niagara component" + ], + [ + "tool_search", + "Unreal Property refPath soft path string" + ], + [ + "tool_search", + "EditorAppToolset" + ] + ], + "error": null, + "final_response": "I've identified the tools needed to complete this task but haven't yet executed it due to tool-call limits. Here's what I found:\n\n**The Solution:**\n1. Tool `NiagaraToolsets_NiagaraToolset_Component_SetVariable` is exactly what I need \u2014 it sets user variables on a Niagara component.\n2. Tool `editor_t" + }, + { + "scenario_id": "V7_niagara_system_asset", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 14.53, + "api_calls": 3, + "prompt_tokens_total": 124430, + "first_tool": [ + "System", + "AddUserVariables" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + "bridge_calls": [ + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ] + ], + "error": null, + "final_response": "Done. Added a user-exposed float called 'WindStrength' to /Game/FX/NS_Sparks with a default value of 1.0. Every instance of this effect can now override it." + }, + { + "scenario_id": "V7_niagara_system_asset", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 12.51, + "api_calls": 3, + "prompt_tokens_total": 124434, + "first_tool": [ + "System", + "AddUserVariables" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + "bridge_calls": [ + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ] + ], + "error": null, + "final_response": "Done. Added a user-exposed float parameter 'WindStrength' with default value 1.0 to /Game/FX/NS_Sparks. Every instance can now override it." + }, + { + "scenario_id": "V7_niagara_system_asset", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 22.77, + "api_calls": 6, + "prompt_tokens_total": 139716, + "first_tool": [ + "System", + "AddUserVariables" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + "bridge_calls": [ + [ + "tool_search", + "unreal engine niagara parameter expose" + ], + [ + "tool_search", + "niagara system add parameter user exposed" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" + ], + [ + "tool_search", + "niagara system add user variable parameter" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ] + ], + "error": null, + "final_response": "Done. Added user-exposed 'WindStrength' float parameter (default value: 1.0) to /Game/FX/NS_Sparks. Every instance of this effect can now override it." + }, + { + "scenario_id": "V7_niagara_system_asset", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 19.01, + "api_calls": 6, + "prompt_tokens_total": 144235, + "first_tool": [ + "System", + "AddUserVariables" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + "bridge_calls": [ + [ + "tool_search", + "niagara parameter add expose unreal" + ], + [ + "tool_search", + "Niagara parameter add user exposed" + ], + [ + "tool_search", + "NiagaraToolset System AddUserVariable" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ] + ], + "error": null, + "final_response": "Done. Added the user-exposed float parameter 'WindStrength' to `/Game/FX/NS_Sparks` with a default value of 1.0. Every instance can now override it." + }, + { + "scenario_id": "V8_widget_screenshot", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 52.08, + "api_calls": 15, + "prompt_tokens_total": 618273, + "first_tool": [ + "computer", + "use" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 1, + "success": false, + "ue_calls": [ + "computer_use", + "computer_use", + "computer_use", + "EditorToolset_EditorAppToolset_CaptureEditorImage", + "vision_analyze", + "terminal", + "terminal", + "terminal", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot", + "terminal" + ], + "bridge_calls": [ + [ + "tool_describe", + "EditorToolset_EditorAppToolset_CaptureEditorImage" + ], + [ + "tool_call", + "EditorToolset_EditorAppToolset_CaptureEditorImage" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Observe" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot" + ] + ], + "error": null, + "final_response": "I've encountered a blocker that prevents completing your request.\n\n**What I attempted:**\n1. Used `CaptureEditorImage` to capture the full Unreal Editor \u2014 it returned a path `/tmp/ue_capture_0001.png`\n2. Discovered the `SlateInspectorToolset_SlateInspectorToolset_Screenshot` tool, which is designed s" + }, + { + "scenario_id": "V8_widget_screenshot", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 16.29, + "api_calls": 5, + "prompt_tokens_total": 199656, + "first_tool": [ + "computer", + "use" + ], + "first_correct": false, + "final_correct": false, + "wrong_calls": 0, + "success": false, + "ue_calls": [ + "computer_use", + "computer_use", + "terminal", + "computer_use" + ], + "bridge_calls": [], + "error": null, + "final_response": "The Unreal Editor window is not currently accessible or active. It appears there are no open windows available. Could you verify that the Unreal Engine 5.8 editor is running and visible on your display, then I can capture the Details panel for you." + }, + { + "scenario_id": "V8_widget_screenshot", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 12.8, + "api_calls": 4, + "prompt_tokens_total": 85393, + "first_tool": [ + "computer", + "use" + ], + "first_correct": false, + "final_correct": false, + "wrong_calls": 0, + "success": false, + "ue_calls": [ + "computer_use", + "computer_use", + "computer_use" + ], + "bridge_calls": [], + "error": null, + "final_response": "It appears the Unreal Editor is not currently running or accessible. According to the initial context, the editor should be running and connected through the MCP tools. Can you confirm the Unreal Editor is open? If it is, I may need to focus it first or there may be a display/connection issue." + }, + { + "scenario_id": "V8_widget_screenshot", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 16.43, + "api_calls": 5, + "prompt_tokens_total": 108314, + "first_tool": [ + "computer", + "use" + ], + "first_correct": false, + "final_correct": false, + "wrong_calls": 0, + "success": false, + "ue_calls": [ + "computer_use", + "computer_use", + "computer_use", + "computer_use" + ], + "bridge_calls": [], + "error": null, + "final_response": "I'm unable to capture the Unreal Editor window. The computer_use tool isn't finding any windows at 0x0 resolution. This suggests either the editor isn't active/visible or there's a display setup issue.\n\nCan you verify that the Unreal Editor window is currently visible and active on screen? If it is," + }, + { + "scenario_id": "V9_save_actor", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 10.31, + "api_calls": 3, + "prompt_tokens_total": 119477, + "first_tool": [ + "save", + "actor" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + "bridge_calls": [ + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V9_save_actor", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 11.63, + "api_calls": 3, + "prompt_tokens_total": 119525, + "first_tool": [ + "save", + "actor" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + "bridge_calls": [ + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ] + ], + "error": null, + "final_response": "Done. Crate_2's changes have been persisted to disk." + }, + { + "scenario_id": "V9_save_actor", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 13.8, + "api_calls": 3, + "prompt_tokens_total": 64771, + "first_tool": [ + "save", + "actor" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + "bridge_calls": [ + [ + "tool_search", + "unreal save actor persist changes disk" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ] + ], + "error": null, + "final_response": "Done. Crate_2's changes have been saved to disk." + }, + { + "scenario_id": "V9_save_actor", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 11.4, + "api_calls": 3, + "prompt_tokens_total": 64759, + "first_tool": [ + "save", + "actor" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + "bridge_calls": [ + [ + "tool_search", + "Unreal save actor persist changes" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V10_zero_keyword", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 12.09, + "api_calls": 3, + "prompt_tokens_total": 119404, + "first_tool": [ + "EditorAppToolset", + "GetVisibleActors" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "EditorToolset_EditorAppToolset_GetVisibleActors" + ], + "bridge_calls": [ + [ + "tool_describe", + "EditorToolset_EditorAppToolset_GetVisibleActors" + ], + [ + "tool_call", + "EditorToolset_EditorAppToolset_GetVisibleActors" + ] + ], + "error": null, + "final_response": "Done.\n\nCurrent level panel contents:\n- Torch_3 (Actor) with NiagaraComponent 'FX_Flame'\n- Crate_2 (StaticMeshActor)" + }, + { + "scenario_id": "V10_zero_keyword", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 15.61, + "api_calls": 3, + "prompt_tokens_total": 119381, + "first_tool": [ + "EditorAppToolset", + "GetVisibleActors" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "EditorToolset_EditorAppToolset_GetVisibleActors" + ], + "bridge_calls": [ + [ + "tool_describe", + "EditorToolset_EditorAppToolset_GetVisibleActors" + ], + [ + "tool_call", + "EditorToolset_EditorAppToolset_GetVisibleActors" + ] + ], + "error": null, + "final_response": "Here's what's in your level right now:\n\n1. Torch_3 (Actor) \u2014 has an NiagaraComponent called 'FX_Flame'\n2. Crate_2 (StaticMeshActor)\n\nDone." + }, + { + "scenario_id": "V10_zero_keyword", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 20.88, + "api_calls": 7, + "prompt_tokens_total": 161728, + "first_tool": [ + "find", + "actors" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "bridge_calls": [ + [ + "tool_search", + "Unreal level outliner actors world contents list" + ], + [ + "tool_search", + "Unreal get all actors level outliner contents" + ], + [ + "tool_search", + "Unreal level GetAllActorsOfClass world actors list" + ], + [ + "tool_search", + "Unreal scene toolset get all actors world hierarchy" + ], + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ] + ], + "error": null, + "final_response": "Done.\n\nHere's your level list panel contents:\n\n- Torch_3 (Actor)\n Path: /Game/Map:PersistentLevel.Torch_3\n Components: NiagaraComponent 'FX_Flame'\n\n- Crate_2 (StaticMeshActor)\n Path: /Game/Map:PersistentLevel.Crate_2" + }, + { + "scenario_id": "V10_zero_keyword", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 22.27, + "api_calls": 7, + "prompt_tokens_total": 162995, + "first_tool": [ + "find", + "actors" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "bridge_calls": [ + [ + "tool_search", + "unreal engine actor list level contents" + ], + [ + "tool_search", + "get outliner hierarchy actors level panel" + ], + [ + "tool_search", + "get all actors in level scene world" + ], + [ + "tool_search", + "find actors filter class type" + ], + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ] + ], + "error": null, + "final_response": "Done.\n\nLevel panel contents:\n- Torch_3 (Actor with NiagaraComponent 'FX_Flame')\n- Crate_2 (StaticMeshActor)" + } +] \ No newline at end of file diff --git a/mcp-research-data/ue_hard_rows.json b/mcp-research-data/ue_hard_rows.json new file mode 100644 index 000000000000..3b4c2f1086d1 --- /dev/null +++ b/mcp-research-data/ue_hard_rows.json @@ -0,0 +1,2541 @@ +[ + { + "scenario_id": "V1_static_material", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 7.89, + "api_calls": 2, + "prompt_tokens_total": 109361, + "first_tool": [ + "set", + "material" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V1_static_material", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 15.43, + "api_calls": 2, + "prompt_tokens_total": 109356, + "first_tool": [ + "set", + "material" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V1_static_material", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 24.52, + "api_calls": 6, + "prompt_tokens_total": 177668, + "first_tool": [ + "material", + "slots" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material" + ], + "bridge_calls": [ + [ + "tool_search", + "assign material to slot on static mesh asset" + ], + [ + "tool_describe", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots" + ], + [ + "tool_describe", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + [ + "tool_call", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots" + ], + [ + "tool_call", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + [ + "tool_call", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material" + ] + ], + "error": null, + "final_response": "The set call returned ok:true. M_Stone is now assigned to slot 0 (Entry_0) of /Game/Meshes/SM_Rock.\n\ndone" + }, + { + "scenario_id": "V1_static_material", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 17.2, + "api_calls": 5, + "prompt_tokens_total": 147097, + "first_tool": [ + "material", + "slots" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + "bridge_calls": [ + [ + "tool_search", + "assign material to static mesh asset slot" + ], + [ + "tool_describe", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots" + ], + [ + "tool_describe", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + [ + "tool_call", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots" + ], + [ + "tool_call", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ] + ], + "error": null, + "final_response": "Material /Game/Mats/M_Stone assigned to slot 0 (Entry_0) of /Game/Meshes/SM_Rock.\n\ndone" + }, + { + "scenario_id": "V2_skeletal_material", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 14.37, + "api_calls": 3, + "prompt_tokens_total": 164360, + "first_tool": [ + "material", + "slots" + ], + "first_correct": false, + "final_correct": false, + "wrong_calls": 0, + "success": false, + "ue_calls": [ + "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", + "editor_toolset_toolsets_asset_AssetTools_exists" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ], + [ + "tool_call", + "editor_toolset_toolsets_asset_AssetTools_exists" + ] + ], + "error": null, + "final_response": "I stopped before assigning because the request doesn't match the mesh.\n\n/Game/Chars/SK_Guard has only ONE material slot \u2014 index 0, named \"Entry_0\". There is no slot 1. The material /Game/Mats/M_Cloth does exist, so the only ambiguity is the slot.\n\nHow would you like to proceed?\n\n- Assign M_Cloth to " + }, + { + "scenario_id": "V2_skeletal_material", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 8.57, + "api_calls": 2, + "prompt_tokens_total": 109378, + "first_tool": [ + "material", + "slots" + ], + "first_correct": false, + "final_correct": false, + "wrong_calls": 0, + "success": false, + "ue_calls": [ + "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ] + ], + "error": null, + "final_response": "The mesh only has one material slot (index 0, named \"Entry_0\"). There is no slot 1 \u2014 the only valid slot on SK_Guard is slot 0.\n\nI did not make the assignment. Slot 1 doesn't exist on this mesh; the sole slot is index 0. Want me to assign M_Cloth to slot 0 instead?" + }, + { + "scenario_id": "V2_skeletal_material", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 30.79, + "api_calls": 7, + "prompt_tokens_total": 206458, + "first_tool": [ + "material", + "slots" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material" + ], + "bridge_calls": [ + [ + "tool_search", + "assign material to skeletal mesh slot" + ], + [ + "tool_describe", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ], + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ], + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" + ], + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material" + ] + ], + "error": null, + "final_response": "A note before I say done: SK_Guard has only ONE material slot (\"Entry_0\", index 0). There is no second slot, so \"slot 1\" (if you meant index 1) doesn't exist. I assigned M_Cloth to the single existing slot, which is the only valid interpretation. The set_material call returned ok:true and the slot n" + }, + { + "scenario_id": "V2_skeletal_material", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 26.75, + "api_calls": 5, + "prompt_tokens_total": 145600, + "first_tool": [ + "material", + "slots" + ], + "first_correct": false, + "final_correct": false, + "wrong_calls": 0, + "success": false, + "ue_calls": [ + "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ], + "bridge_calls": [ + [ + "tool_search", + "assign material to skeletal mesh slot" + ], + [ + "tool_describe", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ], + [ + "tool_call", + "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" + ] + ], + "error": null, + "final_response": "The clarify tool isn't available here, so I'll report the blocker directly rather than guess.\n\nI did not make the assignment. Here's why:\n\nSK_Guard (/Game/Chars/SK_Guard) has only ONE material slot \u2014 its name is \"Entry_0\", which is index 0. There is no \"slot 1\" on this mesh.\n\n\"Slot 1\" is ambiguous:\n" + }, + { + "scenario_id": "V3_curvetable_row", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 76.05, + "api_calls": 15, + "prompt_tokens_total": 849101, + "first_tool": [ + "add", + "row" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", + "editor_toolset_toolsets_curve_table_CurveTableTools_set_keys", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", + "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_set_keys" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_set_keys" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ] + ], + "error": null, + "final_response": "Here's the honest status \u2014 the task is NOT yet complete, and I want to be clear about why.\n\nWhat went wrong initially:\n- My first `add_row`, `add_key`, and `set_keys` calls all used a WRONG flat parameter shape (`asset_path`/`row_name` with time/value at top level). Every one returned `{\"ok\": true}`" + }, + { + "scenario_id": "V3_curvetable_row", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 42.28, + "api_calls": 10, + "prompt_tokens_total": 557289, + "first_tool": [ + "asset", + "class" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_asset_AssetTools_get_asset_class", + "editor_toolset_toolsets_object_ObjectTools_get_class", + "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", + "editor_toolset_toolsets_asset_AssetTools_save_assets" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_asset_AssetTools_get_asset_class" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_call", + "editor_toolset_toolsets_object_ObjectTools_get_class" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" + ], + [ + "tool_call", + "editor_toolset_toolsets_asset_AssetTools_save_assets" + ] + ], + "error": null, + "final_response": "Row 'Heavy' added to /Game/Data/CT_Damage with a key of value 42 at time 0, and the asset has been saved.\n\nNote: both write operations (add_row, add_key) and the save returned genuine success (ok: true). I confirmed beforehand via list_rows that this is a real CurveTable. The read-back verification " + }, + { + "scenario_id": "V3_curvetable_row", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 59.38, + "api_calls": 10, + "prompt_tokens_total": 324649, + "first_tool": [ + "asset", + "class" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_asset_AssetTools_get_asset_class", + "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", + "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows", + "editor_toolset_toolsets_asset_AssetTools_exists" + ], + "bridge_calls": [ + [ + "tool_search", + "curve table set key row value time" + ], + [ + "tool_search", + "data table add row" + ], + [ + "tool_search", + "get asset class type at path" + ], + [ + "tool_call", + "editor_toolset_toolsets_asset_AssetTools_get_asset_class" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows" + ], + [ + "tool_call", + "editor_toolset_toolsets_asset_AssetTools_exists" + ] + ], + "error": null, + "final_response": "The asset exists, but I cannot get a reliable read-back confirming the \"Heavy\" row with value 42 at time 0 actually landed. The write calls returned ok:true, but every verification read (list_rows, get_keys) contradicts that \u2014 \"Heavy\" never appears, and the returned data looks like fixed placeholder" + }, + { + "scenario_id": "V3_curvetable_row", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 49.78, + "api_calls": 9, + "prompt_tokens_total": 278494, + "first_tool": [ + "add", + "row" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", + "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" + ], + "bridge_calls": [ + [ + "tool_search", + "add key to curve table asset row" + ], + [ + "tool_search", + "curve table add row" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" + ], + [ + "tool_describe", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" + ], + [ + "tool_search", + "curve table get row names list rows evaluate value at time" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows" + ], + [ + "tool_call", + "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" + ] + ], + "error": null, + "final_response": "This confirms my concern. `list_rows` returns `[\"Brightness\"]` \u2014 the \"Heavy\" row I \"added\" is not present. The only row is a pre-existing \"Brightness\" row. Both my write calls returned `{\"ok\": true}`, but neither actually took effect:\n\n- add_row \u2192 reported ok, but list_rows shows no \"Heavy\" row (onl" + }, + { + "scenario_id": "V4_project_tag", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 7.4, + "api_calls": 2, + "prompt_tokens_total": 109314, + "first_tool": [ + "GameplayTagsToolset", + "AddTag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + "bridge_calls": [ + [ + "tool_call", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ] + ], + "error": null, + "final_response": "Done.\n\n`Combat.Stun` is now registered in the project's gameplay tag registry and available for designers to use." + }, + { + "scenario_id": "V4_project_tag", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 7.26, + "api_calls": 2, + "prompt_tokens_total": 109329, + "first_tool": [ + "GameplayTagsToolset", + "AddTag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + "bridge_calls": [ + [ + "tool_call", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ] + ], + "error": null, + "final_response": "Done.\n\nThe gameplay tag 'Combat.Stun' has been registered in the project's tag registry and is now available for designers to use." + }, + { + "scenario_id": "V4_project_tag", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 36.03, + "api_calls": 9, + "prompt_tokens_total": 270380, + "first_tool": [ + "GameplayTagsToolset", + "AddTag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "GameplayTagsToolset_GameplayTagsToolset_AddTag", + "GameplayTagsToolset_GameplayTagsToolset_GetTagInfo", + "GameplayTagsToolset_GameplayTagsToolset_ListTags", + "GameplayTagsToolset_GameplayTagsToolset_ListTags" + ], + "bridge_calls": [ + [ + "tool_search", + "add gameplay tag registry" + ], + [ + "tool_search", + "add register new gameplay tag to project" + ], + [ + "tool_describe", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + [ + "tool_call", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + [ + "tool_call", + "GameplayTagsToolset_GameplayTagsToolset_GetTagInfo" + ], + [ + "tool_call", + "GameplayTagsToolset_GameplayTagsToolset_ListTags" + ], + [ + "tool_describe", + "GameplayTagsToolset_GameplayTagsToolset_ListTags" + ], + [ + "tool_call", + "GameplayTagsToolset_GameplayTagsToolset_ListTags" + ] + ], + "error": null, + "final_response": "The verification calls (GetTagInfo and ListTags) are both returning nonsense \u2014 a `{\"name\":\"Brightness\",\"type\":\"float\",\"value\":1.0}` payload that has nothing to do with gameplay tags. That's the same stub-shaped data regardless of arguments, so the read-back path appears to be returning placeholder/m" + }, + { + "scenario_id": "V4_project_tag", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 19.11, + "api_calls": 6, + "prompt_tokens_total": 175547, + "first_tool": [ + "GameplayTagsToolset", + "AddTag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "GameplayTagsToolset_GameplayTagsToolset_AddTag", + "GameplayTagsToolset_GameplayTagsToolset_GetTagInfo" + ], + "bridge_calls": [ + [ + "tool_search", + "add gameplay tag registry" + ], + [ + "tool_search", + "create register new gameplay tag project" + ], + [ + "tool_describe", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + [ + "tool_call", + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + [ + "tool_call", + "GameplayTagsToolset_GameplayTagsToolset_GetTagInfo" + ] + ], + "error": null, + "final_response": "The `GetTagInfo` response is a bit opaque, but the `AddTag` call returned `{\"ok\": true}`, confirming the tag was registered in the project's tag registry.\n\n'Combat.Stun' is now registered and available for designers to use.\n\ndone" + }, + { + "scenario_id": "V5_actor_tag", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 8.09, + "api_calls": 2, + "prompt_tokens_total": 109300, + "first_tool": [ + "add", + "tag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V5_actor_tag", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 44.53, + "api_calls": 6, + "prompt_tokens_total": 330950, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_add_tag", + "editor_toolset_toolsets_actor_ActorTools_get_tags", + "editor_toolset_toolsets_object_ObjectTools_get_properties", + "editor_toolset_toolsets_actor_ActorTools_has_tag" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_get_tags" + ], + [ + "tool_call", + "editor_toolset_toolsets_object_ObjectTools_get_properties" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_has_tag" + ] + ], + "error": null, + "final_response": "The tool responses are clearly malfunctioning \u2014 `has_tag` requires a tag argument to check and should return a boolean, but it returned `{\"name\": \"Entry_0\", \"value\": 1.0}`, the same garbage as the other reads. The Unreal tool bridge appears to be returning canned/mismatched payloads that don't corre" + }, + { + "scenario_id": "V5_actor_tag", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 63.52, + "api_calls": 15, + "prompt_tokens_total": 511114, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_add_tag", + "editor_toolset_toolsets_actor_ActorTools_has_tag", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_get_tags", + "editor_toolset_toolsets_actor_ActorTools_get_tags" + ], + "bridge_calls": [ + [ + "tool_search", + "add tag to level actor" + ], + [ + "tool_describe", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ], + [ + "tool_search", + "find actor by name label in level" + ], + [ + "tool_search", + "get all actors in level list" + ], + [ + "tool_search", + "get actor reference by name spawn find object" + ], + [ + "tool_search", + "SceneTools get all level actors outliner" + ], + [ + "tool_search", + "find_actors by name class filter" + ], + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_has_tag" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_search", + "get object property value tags actor" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_get_tags" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_get_tags" + ] + ], + "error": null, + "final_response": "Now I have a clear signal that something is wrong. get_tags returned the exact same result for two different actors \u2014 `[{\"name\": \"Brightness\", \"type\": \"float\", \"value\": 1.0}]` \u2014 which isn't even a tag list (it's a float property). This backend is returning canned/garbage responses that don't reflect" + }, + { + "scenario_id": "V5_actor_tag", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 62.61, + "api_calls": 14, + "prompt_tokens_total": 461533, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_add_tag", + "editor_toolset_toolsets_actor_ActorTools_has_tag", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_has_tag", + "editor_toolset_toolsets_actor_ActorTools_has_tag", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_get_tags", + "editor_toolset_toolsets_actor_ActorTools_get_tags" + ], + "bridge_calls": [ + [ + "tool_search", + "add tag to level actor" + ], + [ + "tool_describe", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ], + [ + "tool_search", + "find actor by name label in level" + ], + [ + "tool_search", + "get all actors in level list scene actors" + ], + [ + "tool_search", + "find actors by name query search actor" + ], + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_add_tag" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_has_tag" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_has_tag" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_has_tag" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_search", + "get actor property value tags component property read" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_get_tags" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_get_tags" + ] + ], + "error": null, + "final_response": "Interesting \u2014 `get_tags` returns identical `[{\"name\":\"Brightness\",\"type\":\"float\",\"value\":1.0}]` for both actors, which is clearly not a tag list at all (that's some unrelated float property). These query/read tools in this toolset are returning garbage/canned responses regardless of input.\n\nSo here'" + }, + { + "scenario_id": "V6_niagara_component", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 71.82, + "api_calls": 14, + "prompt_tokens_total": 813817, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable", + "NiagaraToolsets_NiagaraToolset_Component_GetVariable", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_Component_GetVariable", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_Component_GetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetVariable" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_get_components" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ] + ], + "error": null, + "final_response": "There's a genuine contradiction between two authoritative tools:\n\n- `find_actors` claims Torch_3 has a NiagaraComponent 'FX_Flame'\n- `get_components` on the same actor path returns only a StaticMeshComponent0, no Niagara component\n\nGiven that get_components (which reads the actor's actual component " + }, + { + "scenario_id": "V6_niagara_component", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 85.58, + "api_calls": 14, + "prompt_tokens_total": 813467, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable", + "NiagaraToolsets_NiagaraToolset_Component_GetVariable", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_Component_GetVariable", + "EditorToolset_LogsToolset_GetLogEntries", + "editor_toolset_toolsets_object_ObjectTools_list_properties", + "EditorToolset_EditorAppToolset_IsPIERunning", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetVariable" + ], + [ + "tool_call", + "EditorToolset_LogsToolset_GetLogEntries" + ], + [ + "tool_call", + "editor_toolset_toolsets_object_ObjectTools_list_properties" + ], + [ + "tool_call", + "EditorToolset_EditorAppToolset_IsPIERunning" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ] + ], + "error": null, + "final_response": "I've done the diligence. Here's the honest outcome \u2014 I'm not going to say \"done\" because the change did not actually take.\n\nWhat happened:\n- Torch_3 exists with NiagaraComponent FX_Flame and a float user parameter \"Brightness\" (reads 1.0).\n- I called SetVariable to set Brightness = 5.0 twice \u2014 first" + }, + { + "scenario_id": "V6_niagara_component", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 71.82, + "api_calls": 15, + "prompt_tokens_total": 550895, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_Component_GetVariable", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ], + "bridge_calls": [ + [ + "tool_search", + "set niagara particle user parameter on actor component" + ], + [ + "tool_search", + "find actor by name get components" + ], + [ + "tool_describe", + "editor_toolset_toolsets_actor_ActorTools_get_components" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_search", + "find actor in level by label name reference" + ], + [ + "tool_search", + "get all actors in level list scene actors" + ], + [ + "tool_search", + "find_actors by name class SceneTools" + ], + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_get_components" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_Component_GetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ] + ], + "error": null, + "final_response": "I was unable to complete the change, and I want to be straight with you rather than claim success the tool didn't actually deliver.\n\nWhat I found:\n- The actor Torch_3 exists at /Game/Map:PersistentLevel.Torch_3 and has a Niagara component named FX_Flame.\n- That component has a float user parameter \"" + }, + { + "scenario_id": "V6_niagara_component", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 76.96, + "api_calls": 15, + "prompt_tokens_total": 578105, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ], + "bridge_calls": [ + [ + "tool_search", + "find actor by name in level Unreal" + ], + [ + "tool_search", + "set Niagara particle system user parameter float" + ], + [ + "tool_search", + "actor component Niagara set user parameter on level instance component" + ], + [ + "tool_search", + "set float variable on Niagara component actor in level" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_search", + "get actor by label find actor components list Niagara component on actor" + ], + [ + "tool_search", + "get all actors in level find actors by class editor level tools" + ], + [ + "tool_search", + "find actor in level by name return actor reference get level actor" + ], + [ + "tool_search", + "get all level actors list actors of class spawn actor editor actor subsystem" + ], + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_describe", + "editor_toolset_toolsets_actor_ActorTools_get_components" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_get_components" + ], + [ + "tool_call", + "editor_toolset_toolsets_actor_ActorTools_get_components" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_SetVariable" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" + ] + ], + "error": null, + "final_response": "I have to be honest: the change did NOT stick, and I should not say \"done.\"\n\nWhat happened:\n- Found the actor Torch_3 (/Game/Map:PersistentLevel.Torch_3). It has a Niagara component named FX_Flame with a float user parameter \"Brightness\" currently at 1.0.\n- Called Component_SetVariable to set Bright" + }, + { + "scenario_id": "V7_niagara_system_asset", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 9.04, + "api_calls": 2, + "prompt_tokens_total": 109368, + "first_tool": [ + "System", + "AddUserVariables" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + "bridge_calls": [ + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V7_niagara_system_asset", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 50.06, + "api_calls": 9, + "prompt_tokens_total": 500337, + "first_tool": [ + "System", + "AddUserVariables" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "editor_toolset_toolsets_asset_AssetTools_save_assets", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetSystemData", + "editor_toolset_toolsets_asset_AssetTools_exists", + "EditorToolset_EditorAppToolset_GetOpenAssets", + "EditorToolset_LogsToolset_GetLogEntries", + "EditorToolset_LogsToolset_GetLogEntries" + ], + "bridge_calls": [ + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" + ], + [ + "tool_call", + "editor_toolset_toolsets_asset_AssetTools_save_assets" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_GetSystemData" + ], + [ + "tool_call", + "editor_toolset_toolsets_asset_AssetTools_exists" + ], + [ + "tool_call", + "EditorToolset_EditorAppToolset_GetOpenAssets" + ], + [ + "tool_call", + "EditorToolset_LogsToolset_GetLogEntries" + ], + [ + "tool_call", + "EditorToolset_LogsToolset_GetLogEntries" + ] + ], + "error": null, + "final_response": "Confirmed \u2014 the Unreal tool bridge is returning a single canned/stale payload (`[{\"name\":\"Entry_0\",\"value\":1.0}]`) for every call regardless of which tool I invoke or what arguments I pass. GetLogEntries should never return that shape. This means the earlier `{\"ok\": true}` responses were not real co" + }, + { + "scenario_id": "V7_niagara_system_asset", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 17.3, + "api_calls": 5, + "prompt_tokens_total": 152054, + "first_tool": [ + "System", + "AddUserVariables" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + "bridge_calls": [ + [ + "tool_search", + "Niagara add user parameter float to system asset" + ], + [ + "tool_search", + "Niagara add user variable to system" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ] + ], + "error": null, + "final_response": "done\n\nWindStrength (float, default 0.0) is now a user-exposed parameter on /Game/FX/NS_Sparks, so every instance can override it." + }, + { + "scenario_id": "V7_niagara_system_asset", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 90.8, + "api_calls": 15, + "prompt_tokens_total": 509840, + "first_tool": [ + "System", + "AddUserVariables" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetSystemSchema", + "editor_toolset_toolsets_asset_AssetTools_load_asset", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" + ], + "bridge_calls": [ + [ + "tool_search", + "Niagara add user exposed parameter to system asset" + ], + [ + "tool_search", + "Niagara system add user variable create parameter" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" + ], + [ + "tool_search", + "find niagara system asset by path load reference" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" + ], + [ + "tool_describe", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_GetSystemSchema" + ], + [ + "tool_search", + "save asset niagara system persist changes" + ], + [ + "tool_describe", + "editor_toolset_toolsets_asset_AssetTools_load_asset" + ], + [ + "tool_call", + "editor_toolset_toolsets_asset_AssetTools_load_asset" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + [ + "tool_call", + "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" + ] + ], + "error": null, + "final_response": "I was not able to complete this task, and I want to be straight with you rather than claim success.\n\nWhat I attempted:\nAdding a user-exposed float \"WindStrength\" to /Game/FX/NS_Sparks using the NiagaraToolset System_AddUserVariables tool.\n\nWhat happened:\n- I confirmed the asset path resolves correct" + }, + { + "scenario_id": "V8_widget_screenshot", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 60.15, + "api_calls": 12, + "prompt_tokens_total": 673472, + "first_tool": [ + "SlateInspectorToolset", + "Windows" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "SlateInspectorToolset_SlateInspectorToolset_Windows", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Windows", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Observe", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot", + "vision_analyze", + "terminal" + ], + "bridge_calls": [ + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Windows" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Windows" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Windows" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Observe" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Observe" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot" + ] + ], + "error": null, + "final_response": "The file lives on the Unreal editor host, which is a separate filesystem from my agent environment, so I can't open it to visually confirm it's the Details panel. But the Screenshot tool accepted the ref \"SDetailsView\" (the Slate class for Unreal's Details panel) and returned a valid capture path ra" + }, + { + "scenario_id": "V8_widget_screenshot", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 59.25, + "api_calls": 13, + "prompt_tokens_total": 727743, + "first_tool": [ + "SlateInspectorToolset", + "Windows" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "SlateInspectorToolset_SlateInspectorToolset_Windows", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Observe", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_ListObservers", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + "bridge_calls": [ + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Windows" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Observe" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Observe" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_ListObservers" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ] + ], + "error": null, + "final_response": "Snapshot consistently returns only `{\"ok\": true}` with the actual widget-tree text stripped out, across every parameter combination I've tried (empty ref, observer ref, varying depth, source locations on). Without that tree I cannot discover the Details panel's widget ref, and Screenshot needs that " + }, + { + "scenario_id": "V8_widget_screenshot", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 68.03, + "api_calls": 15, + "prompt_tokens_total": 474341, + "first_tool": [ + "computer", + "use" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "computer_use", + "computer_use", + "computer_use", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Observe", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot" + ], + "bridge_calls": [ + [ + "tool_search", + "capture screenshot of editor panel widget Details" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_search", + "Slate inspector observe widget subtree" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Observe" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Observe" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_search", + "find Slate widget by type name get ref details view" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot" + ], + [ + "tool_describe", + "UMGToolSet_UMGToolSet_GetWidgetDescription" + ] + ], + "error": null, + "final_response": "That tool is for UMG WidgetBlueprint assets (authored UI), not the editor's own Slate Details panel \u2014 wrong target. The Details panel is a native Slate widget (SDetailsView), reachable only through the SlateInspector toolset, whose ref-discovery (Snapshot) isn't returning tree text through this brid" + }, + { + "scenario_id": "V8_widget_screenshot", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 116.23, + "api_calls": 15, + "prompt_tokens_total": 496374, + "first_tool": [ + "computer", + "use" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "computer_use", + "computer_use", + "computer_use", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "search_files", + "SlateInspectorToolset_SlateInspectorToolset_Observe", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_WaitFor", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot", + "read_file" + ], + "bridge_calls": [ + [ + "tool_search", + "take screenshot of editor panel or widget" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot" + ], + [ + "tool_search", + "find slate widget by type Details panel list widgets" + ], + [ + "tool_search", + "slate inspector dump widget tree find ref by type live editor" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Observe" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_WaitFor" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Observe" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_WaitFor" + ], + [ + "tool_describe", + "SlateInspectorToolset_SlateInspectorToolset_Click" + ], + [ + "tool_call", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot" + ] + ], + "error": null, + "final_response": "Summary of where things landed:\n\nDesktop screen-capture (computer_use) is non-functional in this environment \u2014 capture returns 0x0, and there are no GUI windows exposed (only kernel processes). So the Details panel cannot be grabbed via desktop screenshot.\n\nThe correct path is the Unreal SlateInspec" + }, + { + "scenario_id": "V9_save_actor", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 11.63, + "api_calls": 3, + "prompt_tokens_total": 164406, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ] + ], + "error": null, + "final_response": "Done." + }, + { + "scenario_id": "V9_save_actor", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 11.69, + "api_calls": 3, + "prompt_tokens_total": 164398, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ] + ], + "error": null, + "final_response": "Done. Persisted Crate_2 (/Game/Map:PersistentLevel.Crate_2) to disk \u2014 no full save-all." + }, + { + "scenario_id": "V9_save_actor", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 30.34, + "api_calls": 8, + "prompt_tokens_total": 259700, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + "bridge_calls": [ + [ + "tool_search", + "save actor changes to disk persist single actor" + ], + [ + "tool_search", + "save level actor modified dirty package" + ], + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + [ + "tool_search", + "find actor by name in level get actor reference" + ], + [ + "tool_search", + "list all actors in scene level find actors by class or label" + ], + [ + "tool_search", + "get actors in level selected actors" + ], + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ] + ], + "error": null, + "final_response": "Done. Saved Crate_2 (/Game/Map:PersistentLevel.Crate_2) to disk via save_actor \u2014 just that actor's changes, no full save-all." + }, + { + "scenario_id": "V9_save_actor", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 29.43, + "api_calls": 9, + "prompt_tokens_total": 280271, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + "bridge_calls": [ + [ + "tool_search", + "save actor changes to disk persist single actor" + ], + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + [ + "tool_search", + "find actor by name in level get actor reference" + ], + [ + "tool_search", + "list all actors in scene find actors of class" + ], + [ + "tool_search", + "find_actors get all actors in level by name or label" + ], + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ] + ], + "error": null, + "final_response": "Persisted Crate_2 (/Game/Map:PersistentLevel.Crate_2) to disk via save_actor \u2014 not a full save-all. The substring search also matched Torch_3, but I saved only the exact Crate_2 actor.\n\nDone." + }, + { + "scenario_id": "V10_zero_keyword", + "mode": "listing", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 9.0, + "api_calls": 2, + "prompt_tokens_total": 109419, + "first_tool": [ + "find", + "actors" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ] + ], + "error": null, + "final_response": "Here's the current contents of your level outliner panel (the actors placed in the world):\n\n1. Torch_3\n - Path: /Game/Map:PersistentLevel.Torch_3\n - Class: Actor\n - Components: NiagaraComponent 'FX_Flame'\n\n2. Crate_2\n - Path: /Game/Map:PersistentLevel.Crate_2\n - Class: StaticMeshActor\n\nTha" + }, + { + "scenario_id": "V10_zero_keyword", + "mode": "listing", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 8.52, + "api_calls": 2, + "prompt_tokens_total": 109417, + "first_tool": [ + "find", + "actors" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "bridge_calls": [ + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ] + ], + "error": null, + "final_response": "Here's what the level's actor panel currently contains (2 actors):\n\n1. Torch_3 \u2014 Actor (/Game/Map:PersistentLevel.Torch_3), with a NiagaraComponent 'FX_Flame'\n2. Crate_2 \u2014 StaticMeshActor (/Game/Map:PersistentLevel.Crate_2)\n\ndone" + }, + { + "scenario_id": "V10_zero_keyword", + "mode": "bridge", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 27.23, + "api_calls": 7, + "prompt_tokens_total": 218702, + "first_tool": [ + "find", + "actors" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "bridge_calls": [ + [ + "tool_search", + "list all actors in level outliner world" + ], + [ + "tool_search", + "get all actors in current level" + ], + [ + "tool_search", + "list level actors outliner spawn find all objects placed" + ], + [ + "tool_search", + "find_actors scene tools query actors by class or name" + ], + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ] + ], + "error": null, + "final_response": "Here are the current contents of your level outliner panel:\n\nLevel: /Game/Map (PersistentLevel) \u2014 2 actors placed in the world:\n\n1. Torch_3\n - Class: Actor\n - Path: /Game/Map:PersistentLevel.Torch_3\n - Components: NiagaraComponent 'FX_Flame'\n\n2. Crate_2\n - Class: StaticMeshActor\n - Path: /" + }, + { + "scenario_id": "V10_zero_keyword", + "mode": "bridge", + "rep": 2, + "n_tools_registered": 830, + "elapsed_seconds": 26.45, + "api_calls": 6, + "prompt_tokens_total": 184042, + "first_tool": [ + "find", + "actors" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "bridge_calls": [ + [ + "tool_search", + "list all actors in level outliner" + ], + [ + "tool_search", + "get all actors in current level" + ], + [ + "tool_search", + "list every actor outliner contents all actors world" + ], + [ + "tool_describe", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + [ + "tool_call", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ] + ], + "error": null, + "final_response": "Here are the current contents of your level outliner panel (every actor placed in the world):\n\nTotal actors: 2\n\n1. Torch_3\n - Path: /Game/Map:PersistentLevel.Torch_3\n - Class: Actor\n - Components: NiagaraComponent 'FX_Flame'\n\n2. Crate_2\n - Path: /Game/Map:PersistentLevel.Crate_2\n - Class: " + }, + { + "scenario_id": "V1_static_material", + "mode": "eager", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 28.85, + "api_calls": 4, + "prompt_tokens_total": 1297771, + "first_tool": [ + "load", + "asset" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_asset_AssetTools_load_asset", + "editor_toolset_toolsets_asset_AssetTools_load_asset", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots", + "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" + ], + "bridge_calls": [], + "error": null, + "final_response": "done" + }, + { + "scenario_id": "V2_skeletal_material", + "mode": "eager", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 22.17, + "api_calls": 3, + "prompt_tokens_total": 972730, + "first_tool": [ + "material", + "slots" + ], + "first_correct": false, + "final_correct": false, + "wrong_calls": 0, + "success": false, + "ue_calls": [ + "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", + "editor_toolset_toolsets_asset_AssetTools_exists" + ], + "bridge_calls": [], + "error": null, + "final_response": "I can't complete this. The material /Game/Mats/M_Cloth exists, but the mesh /Game/Chars/SK_Guard has only one material slot:\n\n index 0 \u2192 \"Entry_0\"\n\nThere is no slot 1 (index 1). The mesh has a single slot, so nothing was assigned \u2014 I didn't want to silently retarget slot 0 in place of the slot 1 yo" + }, + { + "scenario_id": "V3_curvetable_row", + "mode": "eager", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 27.56, + "api_calls": 5, + "prompt_tokens_total": 1621753, + "first_tool": [ + "load", + "asset" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_asset_AssetTools_load_asset", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", + "editor_toolset_toolsets_curve_table_CurveTableTools_add_key", + "editor_toolset_toolsets_asset_AssetTools_save_assets" + ], + "bridge_calls": [], + "error": null, + "final_response": "done" + }, + { + "scenario_id": "V4_project_tag", + "mode": "eager", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 15.29, + "api_calls": 2, + "prompt_tokens_total": 648308, + "first_tool": [ + "GameplayTagsToolset", + "AddTag" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "GameplayTagsToolset_GameplayTagsToolset_AddTag" + ], + "bridge_calls": [], + "error": null, + "final_response": "done" + }, + { + "scenario_id": "V5_actor_tag", + "mode": "eager", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 34.71, + "api_calls": 5, + "prompt_tokens_total": 1622365, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_add_tag", + "editor_toolset_toolsets_actor_ActorTools_get_tags", + "editor_toolset_toolsets_object_ObjectTools_get_properties" + ], + "bridge_calls": [], + "error": null, + "final_response": "Something is wrong here, and I'm not going to report \"done\" on faith when the verification contradicts it.\n\nWhat I'm seeing:\n- `add_tag` returned `{\"ok\": true}`.\n- But both read-backs come back garbage: `get_tags` returned a `Brightness` float, and reading the `Tags` property directly returned `Entr" + }, + { + "scenario_id": "V6_niagara_component", + "mode": "eager", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 52.52, + "api_calls": 5, + "prompt_tokens_total": 1623641, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": false, + "wrong_calls": 0, + "success": false, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_actor_ActorTools_get_components", + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_object_ObjectTools_list_properties" + ], + "bridge_calls": [], + "error": null, + "final_response": "The signals are now clearly contradictory and none of them let me safely locate a real Niagara effect component with a Brightness parameter:\n\n1. The `NiagaraActor` type search returned the exact same two actors (Torch_3 as a plain \"Actor\", plus Crate_2) \u2014 identical to the unfiltered substring search" + }, + { + "scenario_id": "V7_niagara_system_asset", + "mode": "eager", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 16.33, + "api_calls": 2, + "prompt_tokens_total": 648451, + "first_tool": [ + "System", + "AddUserVariables" + ], + "first_correct": true, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" + ], + "bridge_calls": [], + "error": null, + "final_response": "done" + }, + { + "scenario_id": "V8_widget_screenshot", + "mode": "eager", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 46.89, + "api_calls": 7, + "prompt_tokens_total": 2271309, + "first_tool": [ + "SlateInspectorToolset", + "Snapshot" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Observe", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_ListObservers", + "SlateInspectorToolset_SlateInspectorToolset_Windows", + "SlateInspectorToolset_SlateInspectorToolset_WaitFor", + "SlateInspectorToolset_SlateInspectorToolset_Screenshot" + ], + "bridge_calls": [], + "error": null, + "final_response": "The screenshot with an empty ref produced a path, but an empty ref means \"active window\" \u2014 that captures the whole editor window, not the Details panel specifically. I need a widget ref for the Details panel, and the Slate inspector's Snapshot/Windows/ListObservers calls are all returning empty payl" + }, + { + "scenario_id": "V9_save_actor", + "mode": "eager", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 21.67, + "api_calls": 3, + "prompt_tokens_total": 972894, + "first_tool": [ + "find", + "actors" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "editor_toolset_toolsets_scene_SceneTools_find_actors", + "editor_toolset_toolsets_scene_SceneTools_save_actor" + ], + "bridge_calls": [], + "error": null, + "final_response": "done\n\nSaved just Crate_2 (/Game/Map:PersistentLevel.Crate_2) to disk. Note: the search also returned Torch_3 as a fuzzy match, but I persisted only the exact Crate_2 actor you edited \u2014 no full save-all." + }, + { + "scenario_id": "V10_zero_keyword", + "mode": "eager", + "rep": 1, + "n_tools_registered": 830, + "elapsed_seconds": 28.54, + "api_calls": 4, + "prompt_tokens_total": 1297260, + "first_tool": [ + "SlateInspectorToolset", + "Snapshot" + ], + "first_correct": false, + "final_correct": true, + "wrong_calls": 0, + "success": true, + "ue_calls": [ + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "SlateInspectorToolset_SlateInspectorToolset_Snapshot", + "editor_toolset_toolsets_scene_SceneTools_find_actors" + ], + "bridge_calls": [], + "error": null, + "final_response": "Here are the current contents of the World Outliner panel (from the live level's actors \u2014 the authoritative source, not the stale UI cache):\n\nCurrent level: /Game/Map (PersistentLevel)\n\n1. Torch_3\n - Class: Actor\n - Path: /Game/Map:PersistentLevel.Torch_3\n - Components: NiagaraComponent 'FX_Fl" + } +] \ No newline at end of file diff --git a/scripts/tool_search_livetest_ue_hard.py b/scripts/tool_search_livetest_ue_hard.py new file mode 100644 index 000000000000..24ad8f2d1f85 --- /dev/null +++ b/scripts/tool_search_livetest_ue_hard.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""Live benchmark v4 — ADVERSARIAL Unreal tool selection at 830 tools. + +Differences from tool_search_livetest_ue.py (which had a ceiling effect): + +1. Scenarios target CONFUSION CLUSTERS in Epic's real catalog — tools with + near-identical names/purposes in different toolsets (StaticMesh vs + SkeletalMesh set_material; GameplayTags vs ActorTools vs GameplayCue tags; + CurveTable vs DataTable rows; Niagara Component vs System SetVariable; + 4 capture variants). Prompts avoid quoting exact tool names. +2. TYPE-AWARE mocks: calling a tool against the wrong asset/actor type + returns a realistic editor error (e.g. "SM_Rock is not a SkeletalMesh"), + so wrong picks visibly fail instead of silently succeeding. +3. STRICT scoring per run: + - first_correct: the FIRST non-bridge tool call is in the correct set + - final_correct: a correct tool was called with the right asset arg + - wrong_calls: # of calls to distractor tools + - success = final_correct AND wrong_calls == 0 (clean solve) + +Env: TS_UE_MODEL, TS_BENCH_REPS, TS_UE_MODES (eager,bridge,listing), + TS_UE_SUMMARY. Scale is always "full" (830 tools). +""" +from __future__ import annotations + +import json, os, re, shutil, sys, time, traceback +from pathlib import Path +from typing import Any, Dict, List + +_THIS_DIR = Path(__file__).resolve().parent +_WORKTREE_ROOT = _THIS_DIR.parent +sys.path.insert(0, str(_WORKTREE_ROOT)) +sys.path.insert(0, str(_THIS_DIR)) + +import tool_search_livetest as base +from tool_search_livetest_ue import load_epic_tools, _SANITIZE # reuse loader + +N_REPS = int(os.environ.get("TS_BENCH_REPS", "2")) + +# --------------------------------------------------------------------------- +# Type-aware mock world +# --------------------------------------------------------------------------- + +WORLD = { + "/Game/Meshes/SM_Rock": "StaticMesh", + "/Game/Chars/SK_Guard": "SkeletalMesh", + "/Game/Data/CT_Damage": "CurveTable", + "/Game/Data/DT_Loot": "DataTable", + "/Game/FX/NS_Sparks": "NiagaraSystem", + "Torch_3": "Actor", # has a NiagaraComponent + "Crate_2": "Actor", +} + +def _mentioned_path(kwargs: Dict[str, Any]) -> str: + blob = json.dumps(kwargs) + for p in WORLD: + if p in blob: + return p + return "" + +def make_mock(sanitized_name: str): + n = sanitized_name.lower() + + def _h(*a, **kw): + path = _mentioned_path(kw) + t = WORLD.get(path, "") + # Wrong-type guards mirror the real editor's failures. + if "skeletalmeshtools" in n and t and t != "SkeletalMesh": + return json.dumps({"error": f"{path} is a {t}, not a SkeletalMesh. Use the StaticMesh tools."}) + if "staticmeshtools" in n and t and t != "StaticMesh": + return json.dumps({"error": f"{path} is a {t}, not a StaticMesh."}) + if "curvetabletools" in n and t and t != "CurveTable": + return json.dumps({"error": f"{path} is a {t}, not a CurveTable."}) + if "datatabletools" in n and t and t != "DataTable": + return json.dumps({"error": f"{path} is a {t}, not a DataTable."}) + if "niagaratoolset_system" in n and t == "Actor": + return json.dumps({"error": f"{path} is a level actor, not a NiagaraSystem asset. Use the Niagara component tools for actors."}) + if "niagaratoolset_component" in n and t == "NiagaraSystem": + return json.dumps({"error": f"{path} is a NiagaraSystem asset, not an actor with a NiagaraComponent."}) + # Coherent world reads so the model can chain calls. + if "find_actors" in n or "getvisibleactors" in n or "get_outliner" in n: + blob = json.dumps(kw) + actors = [{"label": "Torch_3", "path": "/Game/Map:PersistentLevel.Torch_3", + "class": "Actor", "components": ["NiagaraComponent 'FX_Flame'"]}, + {"label": "Crate_2", "path": "/Game/Map:PersistentLevel.Crate_2", + "class": "StaticMeshActor"}] + if "Torch" in blob: + actors = actors[:1] + elif "Crate" in blob: + actors = actors[1:] + return json.dumps({"result": actors}) + if "get_components" in n: + blob = json.dumps(kw) + if "Torch" in blob: + return json.dumps({"result": [{"name": "FX_Flame", "class": "NiagaraComponent"}, + {"name": "PointLight0", "class": "PointLightComponent"}]}) + return json.dumps({"result": [{"name": "StaticMeshComponent0", "class": "StaticMeshComponent"}]}) + if "getuservariables" in n or "list_rows" in n or "listtags" in n or "get_tags" in n: + return json.dumps({"result": [{"name": "Brightness", "type": "float", "value": 1.0}]}) + if any(v in n for v in ("get", "list", "find", "search", "has_", "is_", "can_")): + return json.dumps({"result": [{"name": "Entry_0", "value": 1.0}]}) + if "capture" in n or "screenshot" in n: + return json.dumps({"result": {"image_path": "/tmp/ue_capture_0001.png"}}) + return json.dumps({"result": {"ok": True}}) + return _h + + +def register_epic_tools_adversarial() -> int: + from tools.registry import registry + tools = load_epic_tools("full") + for tdef in tools: + registry.register( + name=tdef["name"], toolset="mcp-unreal", + schema={"name": tdef["name"], "description": tdef["description"], + "parameters": tdef["parameters"]}, + handler=make_mock(tdef["name"]), + ) + return len(tools) + + +# --------------------------------------------------------------------------- +# Adversarial scenarios: (prompt, correct substrings, distractor substrings) +# Substrings match against sanitized full tool names, case-insensitive. +# --------------------------------------------------------------------------- + +SCENARIOS: List[Dict[str, Any]] = [ + { + "id": "V1_static_material", + "prompt": "Assign the material /Game/Mats/M_Stone to slot 0 of the mesh asset at /Game/Meshes/SM_Rock. Then say done.", + "correct": ["StaticMeshTools_set_material"], + "distractors": ["SkeletalMeshTools_set_material", "MaterialTools_create_material", "MaterialInstanceTools"], + }, + { + "id": "V2_skeletal_material", + "prompt": "Assign the material /Game/Mats/M_Cloth to slot 1 of the character mesh at /Game/Chars/SK_Guard. Then say done.", + "correct": ["SkeletalMeshTools_set_material"], + "distractors": ["StaticMeshTools_set_material"], + }, + { + "id": "V3_curvetable_row", + "prompt": "Add a row named 'Heavy' to the table asset at /Game/Data/CT_Damage with value 42 at time 0. Then say done.", + "correct": ["CurveTableTools_add_row"], + "distractors": ["DataTableTools_add_rows", "DataTableTools_set_rows"], + }, + { + "id": "V4_project_tag", + "prompt": "Register a new gameplay tag 'Combat.Stun' in the project's tag registry so designers can use it. Then say done.", + "correct": ["GameplayTagsToolset_AddTag"], + "distractors": ["ActorTools_add_tag", "GameplayCueToolset_AddCueTag"], + }, + { + "id": "V5_actor_tag", + "prompt": "Mark the level actor named Crate_2 with the tag 'loot' so my spawner script can find it. Then say done.", + "correct": ["ActorTools_add_tag"], + "distractors": ["GameplayTagsToolset_AddTag", "GameplayCueToolset_AddCueTag"], + }, + { + "id": "V6_niagara_component", + "prompt": "The particle effect on the actor Torch_3 is too dim — set its 'Brightness' user parameter to 5.0 on that actor's effect component. Then say done.", + "correct": ["NiagaraToolset_Component_SetVariable"], + "distractors": ["NiagaraToolset_System_AddUserVariables", "NiagaraToolset_System_AddSetParameterEntry", + "DataflowAgentToolset_SetVariable", "NiagaraToolset_System"], + }, + { + "id": "V7_niagara_system_asset", + "prompt": "Add a user-exposed float called 'WindStrength' to the effect asset at /Game/FX/NS_Sparks itself, so every instance can override it. Then say done.", + "correct": ["NiagaraToolset_System_AddUserVariables"], + "distractors": ["NiagaraToolset_Component_SetVariable", "DataflowAgentToolset_AddVariable"], + }, + { + "id": "V8_widget_screenshot", + "prompt": "Capture an image of ONLY the Details panel widget (not the whole editor, not the 3D viewport). Tell me the file path. Then say done.", + "correct": ["SlateInspectorToolset_Screenshot"], + "distractors": ["CaptureViewport", "CaptureEditorImage", "CaptureAssetImage"], + }, + { + "id": "V9_save_actor", + "prompt": "I just edited the actor Crate_2 in the level. Persist exactly that actor's changes to disk (not a full save-all). Then say done.", + "correct": ["SceneTools_save_actor"], + "distractors": ["AssetTools_save_assets", "ConfigSettingsToolset_SaveSection"], + }, + { + "id": "V10_zero_keyword", + "prompt": "Something in my level list panel — the thing showing all the stuff placed in the world — seems stale. Get me whatever that panel's current contents are. Then say done.", + "correct": ["SceneTools_find_actors", "GetVisibleActors", "get_outliner"], + "distractors": ["GetContentBrowserPath", "SetContentBrowserPath"], + }, +] + + +def run_one(scenario, mode, rep, out_dir: Path): + enabled = mode in ("bridge", "listing") + model = os.environ.get("TS_UE_MODEL", "anthropic/claude-opus-4.8") + lmax = int(os.environ.get("TS_UE_LISTING_MAX", "30000")) + hermes_home = base.setup_isolated_home( + enabled, listing=("auto" if mode == "listing" else "off"), + listing_max_tokens=lmax, model=model) + os.environ["HERMES_HOME"] = str(hermes_home) + base.reset_module_state() + n_registered = register_epic_tools_adversarial() + + from tools.registry import registry + original_dispatch = registry.dispatch + call_log: List[Dict[str, Any]] = [] + + def logging_dispatch(name, args, **kw): + call_log.append({"name": name, "args": args}) + return original_dispatch(name, args, **kw) + registry.dispatch = logging_dispatch + + usage_log: List[Dict[str, Any]] = [] + started = time.time() + error = None + final_response = "" + messages_out: List[Dict[str, Any]] = [] + _orig_norm = None + try: + from run_agent import AIAgent + agent = AIAgent(provider="openrouter", model=model, quiet_mode=True, + save_trajectories=False, skip_context_files=True, + skip_memory=True, platform="cli", max_iterations=15) + import agent.conversation_loop as _cl + _orig_norm = _cl.normalize_usage + def _norm_spy(raw, **kw): + cu = _orig_norm(raw, **kw) + try: + usage_log.append({"prompt_tokens": cu.prompt_tokens}) + except Exception: + pass + return cu + _cl.normalize_usage = _norm_spy + result = agent.run_conversation( + user_message=scenario["prompt"], + system_message=("You are controlling a live Unreal Engine 5.8 editor. The editor is " + "already running and connected through your Unreal (mcp-unreal) tools — " + "do not try to locate or launch the editor yourself. Choose tools " + "carefully: several toolsets contain similarly-named tools for " + "different object types. Be concise."), + ) + if isinstance(result, dict): + final_response = result.get("final_response") or "" + messages_out = result.get("messages") or [] + except Exception: + error = traceback.format_exc() + finally: + registry.dispatch = original_dispatch + if _orig_norm is not None: + try: + import agent.conversation_loop as _cl2 + _cl2.normalize_usage = _orig_norm + except Exception: + pass + + elapsed = time.time() - started + bridge_call_log = base._extract_bridge_calls(messages_out) + # underlying calls: registry log + tool_call unwraps (registry sees both; dedupe consecutive) + ue_calls = [c for c in call_log if c["name"].lower() not in ("tool_search", "tool_describe", "tool_call") + and not c["name"].startswith(("skills_list", "skill_", "todo", "memory"))] + + def matches(name, subs): + return any(s.lower() in name.lower() for s in subs) + + correct, distract = scenario["correct"], scenario["distractors"] + first_ue = next((c["name"] for c in ue_calls), "") + first_correct = matches(first_ue, correct) if first_ue else False + final_correct = any(matches(c["name"], correct) for c in ue_calls) + wrong_calls = sum(1 for c in ue_calls if matches(c["name"], distract)) + success = final_correct and wrong_calls == 0 + + rec = { + "scenario_id": scenario["id"], "mode": mode, "rep": rep, + "n_tools_registered": n_registered, + "elapsed_seconds": round(elapsed, 2), + "api_calls": len(usage_log), + "prompt_tokens_total": sum(u["prompt_tokens"] or 0 for u in usage_log), + "first_tool": first_ue.split("_")[-2:] if first_ue else None, + "first_correct": first_correct, "final_correct": final_correct, + "wrong_calls": wrong_calls, "success": bool(success), + "ue_calls": [c["name"][-70:] for c in ue_calls][:20], + "bridge_calls": [(b["name"], (b.get("args") or {}).get("query") or (b.get("args") or {}).get("name")) for b in bridge_call_log][:20], + "error": error, + "final_response": base._redact_secrets(final_response)[:300], + } + (out_dir / f"{scenario['id']}__{mode}__rep{rep}.json").write_text(json.dumps(rec, indent=1), encoding="utf-8") + shutil.rmtree(Path(os.environ["HERMES_HOME"]).parent, ignore_errors=True) + return rec + + +def main(): + out_dir = _THIS_DIR / "out_ue_hard" + out_dir.mkdir(exist_ok=True) + modes = [m for m in os.environ.get("TS_UE_MODES", "listing,bridge").split(",") if m] + rows = [] + for scenario in SCENARIOS: + for mode in modes: + for rep in range(1, N_REPS + 1): + rec = run_one(scenario, mode, rep, out_dir) + print(f"{scenario['id']:22} {mode:8} rep{rep}: 1st={'Y' if rec['first_correct'] else 'n'} " + f"final={'Y' if rec['final_correct'] else 'n'} wrong={rec['wrong_calls']} " + f"ok={rec['success']} api={rec['api_calls']} in={rec['prompt_tokens_total']:>9,} " + f"t={rec['elapsed_seconds']:>5}s", flush=True) + rows.append(rec) + name = os.environ.get("TS_UE_SUMMARY", "_ue_hard.json") + (out_dir / name).write_text(json.dumps(rows, indent=1), encoding="utf-8") + print("done ->", out_dir / name) + + +if __name__ == "__main__": + main()