fix hte

Author: AAgnihotry

src/uipath/_cli/_evals/_evaluate.py

@@ -16,7 +16,7 @@ async def evaluate(
     trace_manager: UiPathTraceManager,
     eval_context: UiPathEvalContext,
     event_bus: EventBus,
-    live_tracking_exporter: LlmOpsHttpExporter | None = None,
+    live_tracking_exporter: LlmOpsHttpExporter,
 ) -> UiPathRuntimeResult:
     async with UiPathEvalRuntime(
         factory=runtime_factory,

src/uipath/_cli/_evals/_progress_reporter.py

@@ -76,7 +76,7 @@ async def wrapper(self, *args, **kwargs):
 class StudioWebProgressReporter:
     """Handles reporting evaluation progress to StudioWeb."""
 
-    def __init__(self, live_tracking_exporter: LlmOpsHttpExporter | None = None):
+    def __init__(self, live_tracking_exporter: LlmOpsHttpExporter):
         self.live_tracking_exporter = live_tracking_exporter
         logging.getLogger("uipath._cli.middlewares").setLevel(logging.CRITICAL)
         console_logger = ConsoleLogger.get_instance()
@@ -240,6 +240,9 @@ async def create_eval_set_run_sw(
             scoped="org" if self._is_localhost() else "tenant",
         )
         eval_set_run_id = json.loads(response.content)["id"]
+        logger.info(
+            f"[TraceID] StudioWeb backend generated eval_set_run_id = {eval_set_run_id}"
+        )
         return eval_set_run_id
 
     @gracefully_handle_errors
@@ -376,15 +379,28 @@ async def handle_create_eval_set_run(self, payload: EvalSetRunCreatedEvent) -> N
                     evaluators=payload.evaluators,
                     is_coded=is_coded,
                 )
+                logger.info(
+                    f"[TraceID] Generated new eval_set_run_id = {eval_set_run_id} (not provided by user)"
+                )
+            else:
+                logger.info(
+                    f"[TraceID] Received eval_set_run_id from event = {eval_set_run_id} (from payload)"
+                )
+
+            # Always set trace_id to ensure it matches eval_set_run_id
+            # This ensures consistency whether the ID was provided by user or generated
+            if self.live_tracking_exporter and eval_set_run_id:
+                logger.info(
+                    f"[TraceID] Setting live_tracking_exporter.trace_id = {eval_set_run_id} "
+                    f"(current value: {getattr(self.live_tracking_exporter, 'trace_id', None)})"
+                )
+                self.live_tracking_exporter.trace_id = eval_set_run_id
+
             self.eval_set_run_ids[payload.execution_id] = eval_set_run_id
             current_span = trace.get_current_span()
             if current_span.is_recording():
                 current_span.set_attribute("eval_set_run_id", eval_set_run_id)
 
-            # Set trace_id for live tracking to group all spans under eval_set_run_id
-            if self.live_tracking_exporter and eval_set_run_id:
-                self.live_tracking_exporter.trace_id = eval_set_run_id
-
             logger.debug(
                 f"Created eval set run with ID: {eval_set_run_id} (coded={is_coded})"
             )

src/uipath/_cli/_evals/_runtime.py

@@ -8,12 +8,9 @@
 from time import time
 from typing import (
     Any,
-    Awaitable,
-    Iterable,
     Iterator,
     Protocol,
     Sequence,
-    Tuple,
     runtime_checkable,
 )
 
@@ -203,6 +200,15 @@ def _upsert_span_async(
 
         def _upsert():
             try:
+                span_type = (
+                    span.attributes.get("span_type") if span.attributes else "unknown"
+                )
+                span_name = span.name if hasattr(span, "name") else "unknown"
+                exporter_trace_id = getattr(self.exporter, "trace_id", None)
+                logger.debug(
+                    f"[TraceID] Upserting span '{span_name}' (type={span_type}) "
+                    f"with exporter.trace_id = {exporter_trace_id}"
+                )
                 if status_override:
                     self.exporter.upsert_span(span, status_override=status_override)
                 else:
@@ -337,7 +343,7 @@ def __init__(
         factory: UiPathRuntimeFactoryProtocol,
         trace_manager: UiPathTraceManager,
         event_bus: EventBus,
-        live_tracking_exporter: "LlmOpsHttpExporter | None" = None,
+        live_tracking_exporter: LlmOpsHttpExporter,
     ):
         self.context: UiPathEvalContext = context
         # Wrap the factory to support model settings overrides
@@ -353,8 +359,11 @@ def __init__(
         self.trace_manager.tracer_provider.add_span_processor(span_processor)
 
         # Live tracking processor for real-time span updates
-        # Use provided exporter or create a new one
-        self.live_tracking_exporter = live_tracking_exporter or LlmOpsHttpExporter()
+        self.live_tracking_exporter = live_tracking_exporter
+        logger.info(
+            f"[TraceID] UiPathEvalRuntime initialized with live_tracking_exporter.trace_id = "
+            f"{getattr(live_tracking_exporter, 'trace_id', None)}"
+        )
         live_tracking_processor = LiveTrackingSpanProcessor(self.live_tracking_exporter)
         self.trace_manager.tracer_span_processors.append(live_tracking_processor)
         self.trace_manager.tracer_provider.add_span_processor(live_tracking_processor)
@@ -400,23 +409,38 @@ def _mocker_cache(self) -> Iterator[None]:
                     cache_manager.flush()
                 cache_manager_context.set(None)
 
-    async def initiate_evaluation(
-        self,
-        runtime: UiPathRuntimeProtocol,
-    ) -> Tuple[
-        EvaluationSet,
-        list[BaseEvaluator[Any, Any, Any]],
-        Iterable[Awaitable[EvaluationRunResult]],
-    ]:
+    async def execute(self) -> UiPathRuntimeResult:
+        logger.info("=" * 80)
+        logger.info("EVAL RUNTIME: Starting evaluation execution")
+        logger.info(f"EVAL RUNTIME: Execution ID: {self.execution_id}")
+        logger.info(f"EVAL RUNTIME: Job ID: {self.context.job_id}")
+        logger.info(f"EVAL RUNTIME: Resume mode: {self.context.resume}")
+        if self.context.resume:
+            logger.info(
+                "🟢 EVAL RUNTIME: RESUME MODE ENABLED - Will resume from suspended state"
+            )
+        logger.info("=" * 80)
+
+        # Configure model settings override before creating runtime
+        await self._configure_model_settings_override()
+
+        runtime = await self.factory.new_runtime(
+            entrypoint=self.context.entrypoint or "",
+            runtime_id=self.execution_id,
+        )
+
+        # CRITICAL: Load eval set and publish CREATE_EVAL_SET_RUN event BEFORE creating any spans
+        # This ensures eval_set_run_id is created and trace_id is set on the exporter early
         if self.context.eval_set is None:
             raise ValueError("eval_set must be provided for evaluation runs")
 
-        # Load eval set (path is already resolved in cli_eval.py)
         evaluation_set, _ = EvalHelpers.load_eval_set(
             self.context.eval_set, self.context.eval_ids
         )
         evaluators = await self._load_evaluators(evaluation_set, runtime)
 
+        # Publish CREATE_EVAL_SET_RUN event and WAIT for it to complete
+        # This creates the eval_set_run_id in progress reporter and sets trace_id on exporter
         await self.event_bus.publish(
             EvaluationEvents.CREATE_EVAL_SET_RUN,
             EvalSetRunCreatedEvent(
@@ -429,33 +453,9 @@ async def initiate_evaluation(
             ),
         )
 
-        return (
-            evaluation_set,
-            evaluators,
-            (
-                self._execute_eval(eval_item, evaluators, runtime)
-                for eval_item in evaluation_set.evaluations
-            ),
-        )
-
-    async def execute(self) -> UiPathRuntimeResult:
-        logger.info("=" * 80)
-        logger.info("EVAL RUNTIME: Starting evaluation execution")
-        logger.info(f"EVAL RUNTIME: Execution ID: {self.execution_id}")
-        logger.info(f"EVAL RUNTIME: Job ID: {self.context.job_id}")
-        logger.info(f"EVAL RUNTIME: Resume mode: {self.context.resume}")
-        if self.context.resume:
-            logger.info(
-                "🟢 EVAL RUNTIME: RESUME MODE ENABLED - Will resume from suspended state"
-            )
-        logger.info("=" * 80)
-
-        # Configure model settings override before creating runtime
-        await self._configure_model_settings_override()
-
-        runtime = await self.factory.new_runtime(
-            entrypoint=self.context.entrypoint or "",
-            runtime_id=self.execution_id,
+        logger.info(
+            f"[TraceID] After CREATE_EVAL_SET_RUN event, exporter.trace_id = "
+            f"{getattr(self.live_tracking_exporter, 'trace_id', None)}"
         )
         try:
             with self._mocker_cache():
@@ -475,11 +475,11 @@ async def execute(self) -> UiPathRuntimeResult:
                     "Evaluation Set Run", attributes=span_attributes
                 ) as span:
                     try:
-                        (
-                            evaluation_set,
-                            evaluators,
-                            evaluation_iterable,
-                        ) = await self.initiate_evaluation(runtime)
+                        # Evaluation set and evaluators already loaded and event published above
+                        evaluation_iterable = (
+                            self._execute_eval(eval_item, evaluators, runtime)
+                            for eval_item in evaluation_set.evaluations
+                        )
                         workers = self.context.workers or 1
                         assert workers >= 1
                         eval_run_result_list = await execute_parallel(

src/uipath/_cli/cli_eval.py

@@ -1,12 +1,15 @@
 import ast
 import asyncio
+import logging
 import os
 from typing import Any
 
 import click
 from uipath.core.tracing import UiPathTraceManager
 from uipath.runtime import UiPathRuntimeContext, UiPathRuntimeFactoryRegistry
 
+logger = logging.getLogger(__name__)
+
 from uipath._cli._evals._console_progress_reporter import ConsoleProgressReporter
 from uipath._cli._evals._evaluate import evaluate
 from uipath._cli._evals._progress_reporter import StudioWebProgressReporter
@@ -209,6 +212,9 @@ async def execute_eval():
                 # Set trace_id early if eval_set_run_id is already known
                 if eval_context.eval_set_run_id:
                     live_tracking_exporter.trace_id = eval_context.eval_set_run_id
+                    logger.info(
+                        f"[TraceID] Set live_tracking_exporter.trace_id = {eval_context.eval_set_run_id} (user-provided via --eval-set-run-id)"
+                    )
 
                 if should_register_progress_reporter:
                     progress_reporter = StudioWebProgressReporter(
@@ -233,11 +239,7 @@ async def execute_eval():
                     eval_context.job_id = ctx.job_id
 
                     if ctx.job_id:
-                        # Create exporter with trace_id set to eval_set_run_id
-                        job_exporter = LlmOpsHttpExporter()
-                        if eval_context.eval_set_run_id:
-                            job_exporter.trace_id = eval_context.eval_set_run_id
-                        trace_manager.add_span_exporter(job_exporter)
+                        trace_manager.add_span_exporter(live_tracking_exporter)
 
                     if trace_file:
                         trace_manager.add_span_exporter(

tests/cli/eval/test_eval_runtime_metadata.py

@@ -29,6 +29,13 @@
     UiPathEvalRuntime,
 )
 from uipath._events._event_bus import EventBus
+from uipath.tracing import LlmOpsHttpExporter
+
+
+@pytest.fixture
+def live_tracking_exporter():
+    """Create a live tracking exporter for tests."""
+    return LlmOpsHttpExporter()
 
 
 class MockRuntimeSchema(UiPathRuntimeSchema):
@@ -164,7 +171,9 @@ async def create_runtime():
             return BaseTestRuntime()
 
         factory = MockFactory(create_runtime)
-        return UiPathEvalRuntime(context, factory, trace_manager, event_bus)
+        return UiPathEvalRuntime(
+            context, factory, trace_manager, event_bus, LlmOpsHttpExporter()
+        )
 
     def test_finds_model_in_direct_runtime(self, eval_runtime):
         """Test finding agent model directly on runtime."""
@@ -228,7 +237,9 @@ async def create_runtime():
         factory = MockFactory(create_runtime)
         event_bus = EventBus()
         trace_manager = UiPathTraceManager()
-        eval_runtime = UiPathEvalRuntime(context, factory, trace_manager, event_bus)
+        eval_runtime = UiPathEvalRuntime(
+            context, factory, trace_manager, event_bus, LlmOpsHttpExporter()
+        )
 
         runtime = await create_runtime()
         model = await eval_runtime._get_agent_model(runtime)
@@ -243,7 +254,9 @@ async def create_runtime():
         factory = MockFactory(create_runtime)
         event_bus = EventBus()
         trace_manager = UiPathTraceManager()
-        eval_runtime = UiPathEvalRuntime(context, factory, trace_manager, event_bus)
+        eval_runtime = UiPathEvalRuntime(
+            context, factory, trace_manager, event_bus, LlmOpsHttpExporter()
+        )
 
         runtime = await create_runtime()
         model = await eval_runtime._get_agent_model(runtime)
@@ -258,7 +271,9 @@ async def create_runtime():
         factory = MockFactory(create_runtime)
         event_bus = EventBus()
         trace_manager = UiPathTraceManager()
-        eval_runtime = UiPathEvalRuntime(context, factory, trace_manager, event_bus)
+        eval_runtime = UiPathEvalRuntime(
+            context, factory, trace_manager, event_bus, LlmOpsHttpExporter()
+        )
 
         runtime = await create_runtime()
 
@@ -277,7 +292,9 @@ async def create_good_runtime():
         factory = MockFactory(create_good_runtime)
         event_bus = EventBus()
         trace_manager = UiPathTraceManager()
-        eval_runtime = UiPathEvalRuntime(context, factory, trace_manager, event_bus)
+        eval_runtime = UiPathEvalRuntime(
+            context, factory, trace_manager, event_bus, LlmOpsHttpExporter()
+        )
 
         # Create a bad runtime that raises during get_agent_model
         class BadRuntime(BaseTestRuntime):
@@ -310,7 +327,9 @@ async def create_runtime():
         factory = MockFactory(create_runtime)
         event_bus = EventBus()
         trace_manager = UiPathTraceManager()
-        eval_runtime = UiPathEvalRuntime(context, factory, trace_manager, event_bus)
+        eval_runtime = UiPathEvalRuntime(
+            context, factory, trace_manager, event_bus, LlmOpsHttpExporter()
+        )
 
         runtime = await create_runtime()
         schema = await eval_runtime.get_schema(runtime)
@@ -326,7 +345,9 @@ async def create_runtime():
         factory = MockFactory(create_runtime)
         event_bus = EventBus()
         trace_manager = UiPathTraceManager()
-        eval_runtime = UiPathEvalRuntime(context, factory, trace_manager, event_bus)
+        eval_runtime = UiPathEvalRuntime(
+            context, factory, trace_manager, event_bus, LlmOpsHttpExporter()
+        )
 
         runtime = await create_runtime()
 
@@ -346,7 +367,9 @@ async def create_runtime():
         factory = MockFactory(create_runtime)
         event_bus = EventBus()
         trace_manager = UiPathTraceManager()
-        eval_runtime = UiPathEvalRuntime(context, factory, trace_manager, event_bus)
+        eval_runtime = UiPathEvalRuntime(
+            context, factory, trace_manager, event_bus, LlmOpsHttpExporter()
+        )
 
         runtime = await create_runtime()
 
@@ -393,7 +416,9 @@ async def create_runtime():
         factory = MockFactory(create_runtime)
         event_bus = EventBus()
         trace_manager = UiPathTraceManager()
-        eval_runtime = UiPathEvalRuntime(context, factory, trace_manager, event_bus)
+        eval_runtime = UiPathEvalRuntime(
+            context, factory, trace_manager, event_bus, LlmOpsHttpExporter()
+        )
 
         model = await eval_runtime._get_agent_model(resumable_runtime)
         assert model == "gpt-4o-from-agent-json"