mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-27 17:58:07 +00:00
3a1bbfac61
Replays the actual tool schemas captured from Epic's UE 5.8 ModelContextProtocol + AllToolsets plugins (830 tools / 52 toolsets) as live registry tools with mocked editor responses, then benchmarks eager vs bare-bridge vs bridge+listing at two scales (62-tool editor subset, full 830) on Claude Opus 4.8 (1M ctx; eager at 830 does not fit any 200K model — first call requests ~266K tokens). Headline (full 830, mean per task, rescored): eager 8/8 at 810,578 input tokens ($4.05); bare bridge 16/16 at 160,844 ($0.80); listing 16/16 at 257,264 ($1.29). Frontier model erases the accuracy gap in every mode; cost is the differentiator. At 62 tools eager wins on cost — consistent with the auto-threshold design. Also parameterizes livetest harness model + listing_max_tokens via env/args (TS_UE_MODEL, TS_UE_SCALE, TS_UE_MODES, TS_UE_LISTING_MAX).
1576 lines
No EOL
46 KiB
JSON
1576 lines
No EOL
46 KiB
JSON
[
|
|
{
|
|
"scenario_id": "U1_spawn_named",
|
|
"mode": "listing",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 24.85,
|
|
"api_calls": 5,
|
|
"prompt_tokens_total": 159590,
|
|
"completion_tokens_total": 1182,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U1_spawn_named",
|
|
"mode": "listing",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 56.03,
|
|
"api_calls": 13,
|
|
"prompt_tokens_total": 433445,
|
|
"completion_tokens_total": 2504,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_remove_from_scene",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_remove_from_scene",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_remove_from_scene",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_remove_from_scene",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U1_spawn_named",
|
|
"mode": "bridge",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 13.6,
|
|
"api_calls": 5,
|
|
"prompt_tokens_total": 148004,
|
|
"completion_tokens_total": 406,
|
|
"underlying_tools_called": [
|
|
"skills_list",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U1_spawn_named",
|
|
"mode": "bridge",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 14.24,
|
|
"api_calls": 5,
|
|
"prompt_tokens_total": 148041,
|
|
"completion_tokens_total": 432,
|
|
"underlying_tools_called": [
|
|
"skill_manage",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U1_spawn_named",
|
|
"mode": "eager",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 8.59,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 107981,
|
|
"completion_tokens_total": 216,
|
|
"underlying_tools_called": [
|
|
"skill_manage",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"skills_list"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U1_spawn_named",
|
|
"mode": "eager",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 7.78,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 107995,
|
|
"completion_tokens_total": 216,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U2_viewport_shot",
|
|
"mode": "listing",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 7.59,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 62771,
|
|
"completion_tokens_total": 140,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_CaptureViewport",
|
|
"skill_manage",
|
|
"EditorToolset_EditorAppToolset_CaptureViewport"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U2_viewport_shot",
|
|
"mode": "listing",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 6.41,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 62751,
|
|
"completion_tokens_total": 126,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_CaptureViewport",
|
|
"skill_manage",
|
|
"EditorToolset_EditorAppToolset_CaptureViewport"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U2_viewport_shot",
|
|
"mode": "bridge",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 17.94,
|
|
"api_calls": 5,
|
|
"prompt_tokens_total": 144355,
|
|
"completion_tokens_total": 335,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_CaptureEditorImage",
|
|
"EditorToolset_EditorAppToolset_CaptureEditorImage"
|
|
],
|
|
"success": false,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U2_viewport_shot",
|
|
"mode": "bridge",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 13.06,
|
|
"api_calls": 4,
|
|
"prompt_tokens_total": 113753,
|
|
"completion_tokens_total": 267,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_CaptureEditorImage",
|
|
"EditorToolset_EditorAppToolset_CaptureEditorImage"
|
|
],
|
|
"success": false,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U2_viewport_shot",
|
|
"mode": "eager",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 8.75,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 107831,
|
|
"completion_tokens_total": 76,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_CaptureViewport"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U2_viewport_shot",
|
|
"mode": "eager",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 9.59,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 107819,
|
|
"completion_tokens_total": 76,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_CaptureViewport"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U3_play_mode",
|
|
"mode": "listing",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 9.71,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 94354,
|
|
"completion_tokens_total": 208,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE",
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U3_play_mode",
|
|
"mode": "listing",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 10.64,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 94335,
|
|
"completion_tokens_total": 203,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE",
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U3_play_mode",
|
|
"mode": "bridge",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 13.72,
|
|
"api_calls": 4,
|
|
"prompt_tokens_total": 114079,
|
|
"completion_tokens_total": 273,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE",
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U3_play_mode",
|
|
"mode": "bridge",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 10.96,
|
|
"api_calls": 4,
|
|
"prompt_tokens_total": 114165,
|
|
"completion_tokens_total": 304,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE",
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U3_play_mode",
|
|
"mode": "eager",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 10.69,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 162003,
|
|
"completion_tokens_total": 167,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U3_play_mode",
|
|
"mode": "eager",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 11.23,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 162012,
|
|
"completion_tokens_total": 167,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U4_selection_para",
|
|
"mode": "listing",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 7.01,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 62755,
|
|
"completion_tokens_total": 157,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U4_selection_para",
|
|
"mode": "listing",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 7.01,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 62755,
|
|
"completion_tokens_total": 157,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U4_selection_para",
|
|
"mode": "bridge",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 9.4,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 84959,
|
|
"completion_tokens_total": 212,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U4_selection_para",
|
|
"mode": "bridge",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 9.2,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 84959,
|
|
"completion_tokens_total": 212,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U4_selection_para",
|
|
"mode": "eager",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 7.26,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 107829,
|
|
"completion_tokens_total": 77,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U4_selection_para",
|
|
"mode": "eager",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 8.01,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 107835,
|
|
"completion_tokens_total": 77,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U5_shape_chain",
|
|
"mode": "listing",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 66.62,
|
|
"api_calls": 15,
|
|
"prompt_tokens_total": 507680,
|
|
"completion_tokens_total": 3428,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_label",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"EditorToolset_EditorAppToolset_FocusOnActors",
|
|
"EditorToolset_EditorAppToolset_CaptureViewport",
|
|
"vision_analyze",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_label",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"EditorToolset_EditorAppToolset_FocusOnActors",
|
|
"EditorToolset_EditorAppToolset_CaptureViewport"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U5_shape_chain",
|
|
"mode": "listing",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 51.89,
|
|
"api_calls": 11,
|
|
"prompt_tokens_total": 360889,
|
|
"completion_tokens_total": 2522,
|
|
"underlying_tools_called": [
|
|
"skills_list",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"skill_manage",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_label",
|
|
"EditorToolset_EditorAppToolset_SelectActors",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"skill_manage",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_label",
|
|
"EditorToolset_EditorAppToolset_SelectActors",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U5_shape_chain",
|
|
"mode": "bridge",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 82.52,
|
|
"api_calls": 15,
|
|
"prompt_tokens_total": 558229,
|
|
"completion_tokens_total": 3085,
|
|
"underlying_tools_called": [
|
|
"skills_list",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"skill_manage",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"skill_manage",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U5_shape_chain",
|
|
"mode": "bridge",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 31.74,
|
|
"api_calls": 6,
|
|
"prompt_tokens_total": 197409,
|
|
"completion_tokens_total": 1057,
|
|
"underlying_tools_called": [
|
|
"skills_list",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"skill_manage",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U5_shape_chain",
|
|
"mode": "eager",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 14.31,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 162344,
|
|
"completion_tokens_total": 363,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"skill_manage"
|
|
],
|
|
"success": false,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U5_shape_chain",
|
|
"mode": "eager",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 13.66,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 162337,
|
|
"completion_tokens_total": 370,
|
|
"underlying_tools_called": [
|
|
"skills_list",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube"
|
|
],
|
|
"success": false,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U6_impossible",
|
|
"mode": "listing",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 4.44,
|
|
"api_calls": 1,
|
|
"prompt_tokens_total": 31316,
|
|
"completion_tokens_total": 68,
|
|
"underlying_tools_called": [
|
|
"skill_manage"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U6_impossible",
|
|
"mode": "listing",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 5.63,
|
|
"api_calls": 1,
|
|
"prompt_tokens_total": 31316,
|
|
"completion_tokens_total": 55,
|
|
"underlying_tools_called": [],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U6_impossible",
|
|
"mode": "bridge",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 3.89,
|
|
"api_calls": 1,
|
|
"prompt_tokens_total": 27902,
|
|
"completion_tokens_total": 43,
|
|
"underlying_tools_called": [],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U6_impossible",
|
|
"mode": "bridge",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 4.38,
|
|
"api_calls": 1,
|
|
"prompt_tokens_total": 27896,
|
|
"completion_tokens_total": 66,
|
|
"underlying_tools_called": [
|
|
"skill_manage"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U6_impossible",
|
|
"mode": "eager",
|
|
"scale": "editor",
|
|
"rep": 1,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 4.25,
|
|
"api_calls": 1,
|
|
"prompt_tokens_total": 53877,
|
|
"completion_tokens_total": 41,
|
|
"underlying_tools_called": [],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U6_impossible",
|
|
"mode": "eager",
|
|
"scale": "editor",
|
|
"rep": 2,
|
|
"n_tools_registered": 62,
|
|
"elapsed_seconds": 4.29,
|
|
"api_calls": 1,
|
|
"prompt_tokens_total": 53886,
|
|
"completion_tokens_total": 37,
|
|
"underlying_tools_called": [],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U1_spawn_named",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 13.72,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 109282,
|
|
"completion_tokens_total": 174,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U1_spawn_named",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 8.08,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 109275,
|
|
"completion_tokens_total": 172,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U1_spawn_named",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 16.0,
|
|
"api_calls": 5,
|
|
"prompt_tokens_total": 148999,
|
|
"completion_tokens_total": 425,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U1_spawn_named",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 39.72,
|
|
"api_calls": 8,
|
|
"prompt_tokens_total": 248221,
|
|
"completion_tokens_total": 1310,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U2_viewport_shot",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 7.99,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 109237,
|
|
"completion_tokens_total": 138,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_CaptureViewport",
|
|
"EditorToolset_EditorAppToolset_CaptureViewport"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U2_viewport_shot",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 8.69,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 109231,
|
|
"completion_tokens_total": 141,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_CaptureViewport",
|
|
"EditorToolset_EditorAppToolset_CaptureViewport"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U2_viewport_shot",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 22.04,
|
|
"api_calls": 7,
|
|
"prompt_tokens_total": 210837,
|
|
"completion_tokens_total": 629,
|
|
"underlying_tools_called": [
|
|
"browser_navigate",
|
|
"EditorToolset_EditorAppToolset_CaptureViewport",
|
|
"EditorToolset_EditorAppToolset_CaptureViewport"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U2_viewport_shot",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 18.61,
|
|
"api_calls": 6,
|
|
"prompt_tokens_total": 180004,
|
|
"completion_tokens_total": 531,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_CaptureViewport",
|
|
"EditorToolset_EditorAppToolset_CaptureViewport"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U3_play_mode",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 11.41,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 164049,
|
|
"completion_tokens_total": 203,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE",
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U3_play_mode",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 9.06,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 164040,
|
|
"completion_tokens_total": 203,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE",
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U3_play_mode",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 26.44,
|
|
"api_calls": 5,
|
|
"prompt_tokens_total": 147589,
|
|
"completion_tokens_total": 459,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE",
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U3_play_mode",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 16.56,
|
|
"api_calls": 5,
|
|
"prompt_tokens_total": 147040,
|
|
"completion_tokens_total": 410,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE",
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U4_selection_para",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 6.76,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 109227,
|
|
"completion_tokens_total": 139,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U4_selection_para",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 8.56,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 109237,
|
|
"completion_tokens_total": 145,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U4_selection_para",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 9.54,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 85070,
|
|
"completion_tokens_total": 224,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U4_selection_para",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 9.13,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 85047,
|
|
"completion_tokens_total": 211,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U5_shape_chain",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 67.39,
|
|
"api_calls": 13,
|
|
"prompt_tokens_total": 734288,
|
|
"completion_tokens_total": 3180,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_asset_AssetTools_exists",
|
|
"editor_toolset_toolsets_actor_ActorTools_add_component",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_object_ObjectTools_list_properties",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_object_ObjectTools_get_properties",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_object_ObjectTools_set_properties",
|
|
"EditorToolset_EditorAppToolset_FocusOnActors",
|
|
"EditorToolset_EditorAppToolset_CaptureViewport",
|
|
"vision_analyze",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_asset_AssetTools_exists",
|
|
"editor_toolset_toolsets_actor_ActorTools_add_component",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_object_ObjectTools_list_properties",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_object_ObjectTools_get_properties",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_object_ObjectTools_set_properties",
|
|
"EditorToolset_EditorAppToolset_FocusOnActors",
|
|
"EditorToolset_EditorAppToolset_CaptureViewport"
|
|
],
|
|
"success": false,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U5_shape_chain",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 90.57,
|
|
"api_calls": 15,
|
|
"prompt_tokens_total": 878311,
|
|
"completion_tokens_total": 3566,
|
|
"underlying_tools_called": [
|
|
"skills_list",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_actor_ActorTools_add_component",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"skill_manage",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"EditorToolset_EditorAppToolset_SelectActors",
|
|
"editor_toolset_toolsets_object_ObjectTools_get_properties",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"editor_toolset_toolsets_object_ObjectTools_get_properties",
|
|
"EditorToolset_EditorAppToolset_FocusOnActors",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_actor_ActorTools_add_component",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_find_actors",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_actor_transform",
|
|
"editor_toolset_toolsets_actor_ActorTools_get_components",
|
|
"EditorToolset_EditorAppToolset_SelectActors",
|
|
"editor_toolset_toolsets_object_ObjectTools_get_properties",
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors",
|
|
"editor_toolset_toolsets_object_ObjectTools_get_properties",
|
|
"EditorToolset_EditorAppToolset_FocusOnActors"
|
|
],
|
|
"success": false,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U5_shape_chain",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 30.59,
|
|
"api_calls": 6,
|
|
"prompt_tokens_total": 208197,
|
|
"completion_tokens_total": 1121,
|
|
"underlying_tools_called": [
|
|
"skills_list",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U5_shape_chain",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 34.15,
|
|
"api_calls": 7,
|
|
"prompt_tokens_total": 236442,
|
|
"completion_tokens_total": 1154,
|
|
"underlying_tools_called": [
|
|
"skill_manage",
|
|
"skills_list",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform",
|
|
"skill_manage",
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube",
|
|
"editor_toolset_toolsets_actor_ActorTools_set_actor_transform"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U6_impossible",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 4.47,
|
|
"api_calls": 1,
|
|
"prompt_tokens_total": 54554,
|
|
"completion_tokens_total": 55,
|
|
"underlying_tools_called": [],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U6_impossible",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 4.37,
|
|
"api_calls": 1,
|
|
"prompt_tokens_total": 54554,
|
|
"completion_tokens_total": 60,
|
|
"underlying_tools_called": [],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U6_impossible",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 4.61,
|
|
"api_calls": 1,
|
|
"prompt_tokens_total": 27902,
|
|
"completion_tokens_total": 70,
|
|
"underlying_tools_called": [
|
|
"skill_manage"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U6_impossible",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 4.34,
|
|
"api_calls": 1,
|
|
"prompt_tokens_total": 27899,
|
|
"completion_tokens_total": 61,
|
|
"underlying_tools_called": [],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U7_deep_cut",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 33.57,
|
|
"api_calls": 5,
|
|
"prompt_tokens_total": 275130,
|
|
"completion_tokens_total": 1330,
|
|
"underlying_tools_called": [
|
|
"NiagaraToolsets_NiagaraToolset_System_AddUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetSystemData",
|
|
"editor_toolset_toolsets_asset_AssetTools_exists",
|
|
"NiagaraToolsets_NiagaraToolset_System_AddUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetSystemData",
|
|
"editor_toolset_toolsets_asset_AssetTools_exists"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U7_deep_cut",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 28.07,
|
|
"api_calls": 6,
|
|
"prompt_tokens_total": 347886,
|
|
"completion_tokens_total": 1103,
|
|
"underlying_tools_called": [
|
|
"NiagaraToolsets_NiagaraToolset_System_AddUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetSystemData",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetSystemSummary",
|
|
"NiagaraToolsets_NiagaraToolset_System_AddUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetSystemData",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetSystemSummary"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U7_deep_cut",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 18.36,
|
|
"api_calls": 4,
|
|
"prompt_tokens_total": 124101,
|
|
"completion_tokens_total": 628,
|
|
"underlying_tools_called": [
|
|
"NiagaraToolsets_NiagaraToolset_System_AddUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_AddUserVariables"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U7_deep_cut",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 24.2,
|
|
"api_calls": 6,
|
|
"prompt_tokens_total": 190469,
|
|
"completion_tokens_total": 1085,
|
|
"underlying_tools_called": [
|
|
"NiagaraToolsets_NiagaraToolset_System_AddUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_AddUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetUserVariables"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U8_console_trap",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 32.83,
|
|
"api_calls": 6,
|
|
"prompt_tokens_total": 335905,
|
|
"completion_tokens_total": 1229,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U8_console_trap",
|
|
"mode": "listing",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 45.94,
|
|
"api_calls": 8,
|
|
"prompt_tokens_total": 452020,
|
|
"completion_tokens_total": 1745,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U8_console_trap",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 35.71,
|
|
"api_calls": 7,
|
|
"prompt_tokens_total": 216937,
|
|
"completion_tokens_total": 1520,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U8_console_trap",
|
|
"mode": "bridge",
|
|
"scale": "full",
|
|
"rep": 2,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 49.24,
|
|
"api_calls": 9,
|
|
"prompt_tokens_total": 288757,
|
|
"completion_tokens_total": 1996,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script",
|
|
"EditorToolset_EditorAppToolset_SearchCVars",
|
|
"EditorToolset_EditorAppToolset_GetCameraTransform",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment",
|
|
"editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script",
|
|
"EditorToolset_EditorAppToolset_SearchCVars",
|
|
"EditorToolset_EditorAppToolset_GetCameraTransform"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U1_spawn_named",
|
|
"mode": "eager",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 15.89,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 648323,
|
|
"completion_tokens_total": 215,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U2_viewport_shot",
|
|
"mode": "eager",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 14.7,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 648161,
|
|
"completion_tokens_total": 88,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_CaptureViewport"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U3_play_mode",
|
|
"mode": "eager",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 20.53,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 972457,
|
|
"completion_tokens_total": 151,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_StartPIE",
|
|
"EditorToolset_EditorAppToolset_StopPIE"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U4_selection_para",
|
|
"mode": "eager",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 17.38,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 648169,
|
|
"completion_tokens_total": 104,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_GetSelectedActors"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U5_shape_chain",
|
|
"mode": "eager",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 21.85,
|
|
"api_calls": 3,
|
|
"prompt_tokens_total": 972831,
|
|
"completion_tokens_total": 373,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class",
|
|
"editor_toolset_toolsets_primitive_PrimitiveTools_add_cube"
|
|
],
|
|
"success": false,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U6_impossible",
|
|
"mode": "eager",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 7.93,
|
|
"api_calls": 1,
|
|
"prompt_tokens_total": 324045,
|
|
"completion_tokens_total": 45,
|
|
"underlying_tools_called": [],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U7_deep_cut",
|
|
"mode": "eager",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 33.64,
|
|
"api_calls": 5,
|
|
"prompt_tokens_total": 1622364,
|
|
"completion_tokens_total": 767,
|
|
"underlying_tools_called": [
|
|
"editor_toolset_toolsets_asset_AssetTools_load_asset",
|
|
"NiagaraToolsets_NiagaraToolset_System_AddUserVariables",
|
|
"NiagaraToolsets_NiagaraToolset_System_GetUserVariables",
|
|
"editor_toolset_toolsets_asset_AssetTools_save_assets"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
},
|
|
{
|
|
"scenario_id": "U8_console_trap",
|
|
"mode": "eager",
|
|
"scale": "full",
|
|
"rep": 1,
|
|
"n_tools_registered": 830,
|
|
"elapsed_seconds": 19.59,
|
|
"api_calls": 2,
|
|
"prompt_tokens_total": 648270,
|
|
"completion_tokens_total": 437,
|
|
"underlying_tools_called": [
|
|
"EditorToolset_EditorAppToolset_SearchCVars"
|
|
],
|
|
"success": true,
|
|
"error": null,
|
|
"success_rescored": true
|
|
}
|
|
] |