refactor: remove dummy eval item from debug simulation

Now that mocking is decoupled from evals (PR #1148), update the debug
command to use MockingContext directly instead of creating a dummy
EvaluationItem. This simplifies the code and better reflects the
separation of concerns.

Changes:
- Update load_simulation_config to return MockingContext instead of EvaluationItem
- Replace EvaluationSimulationTool with ToolSimulation
- Update all tests to work with MockingContext
- Remove unnecessary evaluation_criterias field

Co-Authored-By: Claude Sonnet 4.5 <[email protected]>

Author: mjnovice

src/uipath/_cli/cli_debug.py

@@ -17,17 +17,17 @@
 
 from uipath._cli._chat._bridge import get_chat_bridge
 from uipath._cli._debug._bridge import get_debug_bridge
-from uipath._cli._evals._models._evaluation_set import (
-    EvaluationItem,
-    EvaluationSimulationTool,
-    LLMMockingStrategy,
-    MockingStrategyType,
-)
 from uipath._cli._evals._span_collection import ExecutionSpanCollector
 from uipath._cli._evals.mocks.mocks import (
     clear_execution_context,
     set_execution_context,
 )
+from uipath._cli._evals.mocks.types import (
+    LLMMockingStrategy,
+    MockingContext,
+    MockingStrategyType,
+    ToolSimulation,
+)
 from uipath._cli._utils._debug import setup_debugging
 from uipath._cli._utils._studio_project import StudioClient
 from uipath._utils._bindings import ResourceOverwritesContext
@@ -40,11 +40,11 @@
 console = ConsoleLogger()
 
 
-def load_simulation_config() -> EvaluationItem | None:
-    """Load simulation.json from current directory and convert to EvaluationItem.
+def load_simulation_config() -> MockingContext | None:
+    """Load simulation.json from current directory and convert to MockingContext.
 
     Returns:
-        EvaluationItem with LLM mocking strategy if simulation.json exists and is valid,
+        MockingContext with LLM mocking strategy if simulation.json exists and is valid,
         None otherwise.
     """
     simulation_path = Path.cwd() / "simulation.json"
@@ -62,7 +62,7 @@ def load_simulation_config() -> EvaluationItem | None:
 
         # Extract tools to simulate
         tools_to_simulate = [
-            EvaluationSimulationTool(name=tool["name"])
+            ToolSimulation(name=tool["name"])
             for tool in simulation_data.get("toolsToSimulate", [])
         ]
 
@@ -76,18 +76,15 @@ def load_simulation_config() -> EvaluationItem | None:
             tools_to_simulate=tools_to_simulate,
         )
 
-        # Create a minimal EvaluationItem for debugging
-        # We use empty inputs since the actual input comes from the debug command
-        eval_item = EvaluationItem(
-            id=str(uuid.uuid4()),
+        # Create MockingContext for debugging
+        mocking_context = MockingContext(
+            strategy=mocking_strategy,
             name="debug-simulation",
             inputs={},
-            evaluation_criterias={},
-            mocking_strategy=mocking_strategy,
         )
 
         console.info(f"Loaded simulation config for {len(tools_to_simulate)} tool(s)")
-        return eval_item
+        return mocking_context
 
     except Exception as e:
         console.warning(f"Failed to load simulation.json: {e}")
@@ -183,15 +180,15 @@ async def execute_debug_runtime():
                     factory: UiPathRuntimeFactoryProtocol | None = None
 
                     # Load simulation config and set up execution context for tool mocking
-                    eval_item = load_simulation_config()
+                    mocking_ctx = load_simulation_config()
                     span_collector: ExecutionSpanCollector | None = None
                     execution_id = str(uuid.uuid4())
 
-                    if eval_item:
+                    if mocking_ctx:
                         # Create span collector for trace access during mocking
                         span_collector = ExecutionSpanCollector()
                         # Set execution context to enable tool simulation
-                        set_execution_context(eval_item, span_collector, execution_id)
+                        set_execution_context(mocking_ctx, span_collector, execution_id)
 
                     try:
                         trigger_poll_interval: float = 5.0
@@ -243,7 +240,7 @@ async def execute_debug_runtime():
 
                     finally:
                         # Clear execution context after debugging completes
-                        if eval_item:
+                        if mocking_ctx:
                             clear_execution_context()
 
                         if debug_runtime:

tests/cli/test_debug_simulation.py

@@ -9,14 +9,14 @@
 from click.testing import CliRunner
 
 from uipath._cli import cli
-from uipath._cli._evals._models._evaluation_set import (
-    EvaluationItem,
-    LLMMockingStrategy,
-)
 from uipath._cli._evals.mocks.mocks import (
     clear_execution_context,
     is_tool_simulated,
 )
+from uipath._cli._evals.mocks.types import (
+    LLMMockingStrategy,
+    MockingContext,
+)
 from uipath._cli.cli_debug import load_simulation_config
 from uipath._cli.middlewares import MiddlewareResult
 
@@ -78,16 +78,13 @@ def test_loads_valid_simulation_config(
             result = load_simulation_config()
 
             assert result is not None
-            assert isinstance(result, EvaluationItem)
+            assert isinstance(result, MockingContext)
             assert result.name == "debug-simulation"
-            assert result.mocking_strategy is not None
-            assert isinstance(result.mocking_strategy, LLMMockingStrategy)
-            assert (
-                result.mocking_strategy.prompt
-                == valid_simulation_config["instructions"]
-            )
-            assert len(result.mocking_strategy.tools_to_simulate) == 3
-            assert result.mocking_strategy.tools_to_simulate[0].name == "Web Reader"
+            assert result.strategy is not None
+            assert isinstance(result.strategy, LLMMockingStrategy)
+            assert result.strategy.prompt == valid_simulation_config["instructions"]
+            assert len(result.strategy.tools_to_simulate) == 3
+            assert result.strategy.tools_to_simulate[0].name == "Web Reader"
 
     def test_returns_none_when_disabled(
         self, temp_dir: str, disabled_simulation_config: dict[str, Any]
@@ -134,10 +131,10 @@ def test_handles_missing_required_fields(self, temp_dir: str):
             # Should return None because toolsToSimulate is missing/empty
             assert result is None
 
-    def test_creates_evaluation_item_with_empty_inputs(
+    def test_creates_mocking_context_with_empty_inputs(
         self, temp_dir: str, valid_simulation_config: dict[str, Any]
     ):
-        """Test that EvaluationItem is created with empty inputs."""
+        """Test that MockingContext is created with empty inputs."""
         simulation_path = Path(temp_dir) / "simulation.json"
         with open(simulation_path, "w", encoding="utf-8") as f:
             json.dump(valid_simulation_config, f)
@@ -147,7 +144,6 @@ def test_creates_evaluation_item_with_empty_inputs(
 
             assert result is not None
             assert result.inputs == {}
-            assert result.evaluation_criterias == {}
 
     def test_uses_default_empty_instructions_when_missing(self, temp_dir: str):
         """Test that empty string is used when instructions field is missing."""
@@ -163,7 +159,7 @@ def test_uses_default_empty_instructions_when_missing(self, temp_dir: str):
             result = load_simulation_config()
 
             assert result is not None
-            assert result.mocking_strategy.prompt == ""
+            assert result.strategy.prompt == ""
 
 
 class TestDebugCommandSimulationIntegration:
@@ -287,16 +283,13 @@ def main(input):
 
                                     # Verify set_execution_context was called
                                     assert mock_set_context.called
-                                    # Verify the EvaluationItem passed has the right structure
+                                    # Verify the MockingContext passed has the right structure
                                     call_args = mock_set_context.call_args
-                                    eval_item = call_args[0][0]
-                                    assert isinstance(eval_item, EvaluationItem)
-                                    assert eval_item.mocking_strategy is not None
+                                    mocking_ctx = call_args[0][0]
+                                    assert isinstance(mocking_ctx, MockingContext)
+                                    assert mocking_ctx.strategy is not None
                                     assert (
-                                        len(
-                                            eval_item.mocking_strategy.tools_to_simulate
-                                        )
-                                        == 3
+                                        len(mocking_ctx.strategy.tools_to_simulate) == 3
                                     )
 
                                     # Verify clear_execution_context was called in finally block
@@ -348,16 +341,16 @@ def test_simulation_config_enables_tool_mocking(
             json.dump(valid_simulation_config, f)
 
         with patch("pathlib.Path.cwd", return_value=Path(temp_dir)):
-            eval_item = load_simulation_config()
-            assert eval_item is not None
+            mocking_ctx = load_simulation_config()
+            assert mocking_ctx is not None
 
             # Manually set context (simulating what debug command does)
             from uipath._cli._evals._span_collection import ExecutionSpanCollector
 
             span_collector = ExecutionSpanCollector()
             from uipath._cli._evals.mocks.mocks import set_execution_context
 
-            set_execution_context(eval_item, span_collector, "test-execution-id")
+            set_execution_context(mocking_ctx, span_collector, "test-execution-id")
 
             # Verify tools are detected as simulated
             assert is_tool_simulated("Web Reader") is True
@@ -438,15 +431,15 @@ def test_handles_tool_name_normalization(self, temp_dir: str):
             json.dump(config, f)
 
         with patch("pathlib.Path.cwd", return_value=Path(temp_dir)):
-            eval_item = load_simulation_config()
-            assert eval_item is not None
+            mocking_ctx = load_simulation_config()
+            assert mocking_ctx is not None
 
             # Set context to test name normalization
             from uipath._cli._evals._span_collection import ExecutionSpanCollector
             from uipath._cli._evals.mocks.mocks import set_execution_context
 
             span_collector = ExecutionSpanCollector()
-            set_execution_context(eval_item, span_collector, "test-id")
+            set_execution_context(mocking_ctx, span_collector, "test-id")
 
             # Both underscore and space versions should be detected
             assert is_tool_simulated("Web_Reader") is True