fix: limit concurrent Modal sandbox creations to avoid deadlocks

- Add max_concurrent_tasks config (default 8) with semaphore in TB2 eval
- Pass cwd: /app via register_task_env_overrides for TB2 tasks
- Add /home/ to host path prefixes as safety net for container backends

When all 86 TerminalBench2 tasks fire simultaneously, each creates a Modal sandbox
via asyncio.run() inside a thread pool worker. Modal's blocking calls deadlock
when too many are created at once. The semaphore ensures max 8 concurrent creations.

Co-Authored-By: hermes-agent[bot] <hermes-agent[bot]@users.noreply.github.com>

environments/benchmarks/terminalbench_2/terminalbench2_env.py

@@ -118,6 +118,15 @@ class TerminalBench2EvalConfig(HermesAgentEnvConfig):
         "Tasks exceeding this are scored as FAIL. Default 30 minutes.",
     )
 
+    # --- Concurrency control ---
+    max_concurrent_tasks: int = Field(
+        default=8,
+        description="Maximum number of tasks to run concurrently. "
+        "Limits concurrent Modal sandbox creations to avoid async/threading deadlocks. "
+        "Modal has internal limits and creating too many sandboxes simultaneously "
+        "causes blocking calls to deadlock inside the thread pool.",
+    )
+
 
 # Tasks that cannot run properly on Modal and are excluded from scoring.
 MODAL_INCOMPATIBLE_TASKS = {
@@ -430,7 +439,7 @@ class TerminalBench2EvalEnv(HermesAgentBaseEnv):
                 }
 
             # --- 2. Register per-task Modal image override ---
-            register_task_env_overrides(task_id, {"modal_image": modal_image})
+            register_task_env_overrides(task_id, {"modal_image": modal_image, "cwd": "/app"})
             logger.info(
                 "Task %s: registered image override for task_id %s",
                 task_name, task_id[:8],
@@ -733,12 +742,23 @@ class TerminalBench2EvalEnv(HermesAgentBaseEnv):
         print(f"  Tool thread pool: {self.config.tool_pool_size}")
         print(f"  Terminal timeout: {self.config.terminal_timeout}s/cmd")
         print(f"  Terminal lifetime: {self.config.terminal_lifetime}s (auto: task_timeout + 120)")
+        print(f"  Max concurrent tasks: {self.config.max_concurrent_tasks}")
         print(f"{'='*60}\n")
 
+        # Semaphore to limit concurrent Modal sandbox creations.
+        # Without this, all 86 tasks fire simultaneously, each creating a Modal
+        # sandbox via asyncio.run() inside a thread pool worker. Modal's blocking
+        # calls (App.lookup, etc.) deadlock when too many are created at once.
+        semaphore = asyncio.Semaphore(self.config.max_concurrent_tasks)
+
+        async def _eval_with_semaphore(item):
+            async with semaphore:
+                return await self._eval_with_timeout(item)
+
         # Fire all tasks with wall-clock timeout, track live accuracy on the bar
         total_tasks = len(self.all_eval_items)
         eval_tasks = [
-            asyncio.ensure_future(self._eval_with_timeout(item))
+            asyncio.ensure_future(_eval_with_semaphore(item))
             for item in self.all_eval_items
         ]