diff --git a/samples/csv-processor/entry-points.json b/samples/csv-processor/entry-points.json index 2e5a9b9a5..3748de53a 100644 --- a/samples/csv-processor/entry-points.json +++ b/samples/csv-processor/entry-points.json @@ -1,71 +1,5 @@ { "$schema": "https://cloud.uipath.com/draft/2024-12/entry-point", "$id": "entry-points.json", - "entryPoints": [ - { - "filePath": "main", - "uniqueId": "99d18636-8167-4c38-b3de-448a2fad8032", - "type": "agent", - "input": { - "type": "object", - "properties": { - "attachment": { - "$ref": "#/definitions/job-attachment" - } - }, - "required": [ - "attachment" - ], - "title": "Input", - "definitions": { - "job-attachment": { - "type": "object", - "required": [ - "ID" - ], - "x-uipath-resource-kind": "JobAttachment", - "properties": { - "ID": { - "type": "string" - }, - "FullName": { - "type": "string" - }, - "MimeType": { - "type": "string" - }, - "Metadata": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - } - } - } - }, - "output": { - "type": "object", - "properties": { - "ID": { - "format": "uuid", - "type": "string" - }, - "FullName": { - "title": "Fullname", - "type": "string" - }, - "MimeType": { - "title": "Mimetype", - "type": "string" - } - }, - "required": [ - "FullName", - "MimeType" - ], - "title": "UiPathAttachment" - } - } - ] + "entryPoints": [] } \ No newline at end of file diff --git a/samples/csv-processor/evaluations/eval-sets/file-input-tests-local.json b/samples/csv-processor/evaluations/eval-sets/file-input-tests-local.json new file mode 100644 index 000000000..70ebd24ff --- /dev/null +++ b/samples/csv-processor/evaluations/eval-sets/file-input-tests-local.json @@ -0,0 +1,74 @@ +{ + "version": "1.0", + "id": "FileInputTestsLocalEval", + "name": "File Input Scenario Tests (Local Mode)", + "evaluatorRefs": [ + "CSVShapeEvaluator", + "CSVColumnsEvaluator" + ], + "evaluations": [ + { + "id": "test-sales-data-csv", + "name": "Test Sales Data CSV Processing", + "description": "Test processing of sales_data.csv with 5 rows and 5 columns", + "inputs": { + "attachment": { + "ID": "11111111-1111-1111-1111-111111111111", + "FullName": "../../samples/csv-processor/test-data/sales_data.csv", + "MimeType": "text/csv" + } + }, + "evaluationCriterias": { + "CSVShapeEvaluator": { + "expected_rows": 5, + "expected_columns": 5 + }, + "CSVColumnsEvaluator": { + "expected_columns": ["date", "product", "quantity", "price", "total"] + } + } + }, + { + "id": "test-large-dataset-csv", + "name": "Test Large Dataset CSV Processing", + "description": "Test processing of large_dataset.csv with 5 rows and 13 columns", + "inputs": { + "attachment": { + "ID": "22222222-2222-2222-2222-222222222222", + "FullName": "../../samples/csv-processor/test-data/large_dataset.csv", + "MimeType": "text/csv" + } + }, + "evaluationCriterias": { + "CSVShapeEvaluator": { + "expected_rows": 5, + "expected_columns": 13 + }, + "CSVColumnsEvaluator": { + "expected_columns": ["id", "name", "email", "department", "salary", "status"] + } + } + }, + { + "id": "test-minimal-csv", + "name": "Test Minimal CSV Processing", + "description": "Test processing of minimal.csv with 1 row and 2 columns", + "inputs": { + "attachment": { + "ID": "33333333-3333-3333-3333-333333333333", + "FullName": "../../samples/csv-processor/test-data/minimal.csv", + "MimeType": "text/csv" + } + }, + "evaluationCriterias": { + "CSVShapeEvaluator": { + "expected_rows": 1, + "expected_columns": 2 + }, + "CSVColumnsEvaluator": { + "expected_columns": ["id", "value"] + } + } + } + ] +} diff --git a/samples/csv-processor/evaluations/evaluators/attachment_created_evaluator.json b/samples/csv-processor/evaluations/evaluators/attachment_created_evaluator.json new file mode 100644 index 000000000..f4b352c20 --- /dev/null +++ b/samples/csv-processor/evaluations/evaluators/attachment_created_evaluator.json @@ -0,0 +1,12 @@ +{ + "version": "1.0", + "id": "AttachmentCreatedEvaluator", + "description": "Checks if the agent successfully created an output attachment with processed CSV information", + "evaluatorSchema": "file://attachment_created_evaluator.py:AttachmentCreatedEvaluator", + "evaluatorConfig": { + "name": "AttachmentCreatedEvaluator", + "defaultEvaluationCriteria": { + "attachment_name": "processed_output.txt" + } + } +} diff --git a/samples/csv-processor/evaluations/evaluators/attachment_created_evaluator.py b/samples/csv-processor/evaluations/evaluators/attachment_created_evaluator.py new file mode 100644 index 000000000..5fd2d6376 --- /dev/null +++ b/samples/csv-processor/evaluations/evaluators/attachment_created_evaluator.py @@ -0,0 +1,80 @@ +from uipath.eval.evaluators import ( + BaseEvaluationCriteria, + BaseEvaluator, + BaseEvaluatorConfig, +) +from uipath.eval.models import AgentExecution, EvaluationResult, NumericEvaluationResult + + +class AttachmentCreatedEvaluationCriteria(BaseEvaluationCriteria): + """Evaluation criteria for the attachment created evaluator.""" + + attachment_name: str = "processed_output.txt" + + +class AttachmentCreatedEvaluatorConfig( + BaseEvaluatorConfig[AttachmentCreatedEvaluationCriteria] +): + """Configuration for the attachment created evaluator.""" + + name: str = "AttachmentCreatedEvaluator" + default_evaluation_criteria: AttachmentCreatedEvaluationCriteria = ( + AttachmentCreatedEvaluationCriteria() + ) + + +class AttachmentCreatedEvaluator( + BaseEvaluator[ + AttachmentCreatedEvaluationCriteria, AttachmentCreatedEvaluatorConfig, None + ] +): + """A custom evaluator that checks if the agent successfully created an output attachment.""" + + @classmethod + def get_evaluator_id(cls) -> str: + return "AttachmentCreatedEvaluator" + + async def evaluate( + self, + agent_execution: AgentExecution, + evaluation_criteria: AttachmentCreatedEvaluationCriteria, + ) -> EvaluationResult: + # Check if the agent created an attachment by looking for: + # 1. Span with name containing "create_attachment" + # 2. Or output containing attachment ID/information + + attachment_created = False + + # Look for attachment creation in traces + for span in agent_execution.agent_trace: + # Check span name for attachment operations + if "attachment" in span.name.lower() or "create" in span.name.lower(): + attachment_created = True + break + + # Check span attributes for attachment information + if span.attributes: + for attr_key, attr_value in span.attributes.items(): + if isinstance(attr_value, str): + if ( + "attachment" in attr_key.lower() + or evaluation_criteria.attachment_name in attr_value + ): + attachment_created = True + break + + if attachment_created: + break + + # Also check if output contains attachment information + if not attachment_created and agent_execution.agent_output: + output_str = str(agent_execution.agent_output) + if ( + "attachment" in output_str.lower() + or evaluation_criteria.attachment_name in output_str + ): + attachment_created = True + + return NumericEvaluationResult( + score=float(attachment_created), + ) diff --git a/samples/csv-processor/evaluations/evaluators/csv_columns_evaluator.json b/samples/csv-processor/evaluations/evaluators/csv_columns_evaluator.json new file mode 100644 index 000000000..bddeea0ff --- /dev/null +++ b/samples/csv-processor/evaluations/evaluators/csv_columns_evaluator.json @@ -0,0 +1,12 @@ +{ + "version": "1.0", + "id": "CSVColumnsEvaluator", + "description": "Checks if the expected CSV column names are correctly identified in the output", + "evaluatorSchema": "file://csv_columns_evaluator.py:CSVColumnsEvaluator", + "evaluatorConfig": { + "name": "CSVColumnsEvaluator", + "defaultEvaluationCriteria": { + "expected_columns": [] + } + } +} diff --git a/samples/csv-processor/evaluations/evaluators/csv_columns_evaluator.py b/samples/csv-processor/evaluations/evaluators/csv_columns_evaluator.py new file mode 100644 index 000000000..500620eb7 --- /dev/null +++ b/samples/csv-processor/evaluations/evaluators/csv_columns_evaluator.py @@ -0,0 +1,81 @@ +from typing import List + +from uipath.eval.evaluators import ( + BaseEvaluationCriteria, + BaseEvaluator, + BaseEvaluatorConfig, +) +from uipath.eval.models import AgentExecution, EvaluationResult, NumericEvaluationResult + + +class CSVColumnsEvaluationCriteria(BaseEvaluationCriteria): + """Evaluation criteria for the CSV columns evaluator.""" + + expected_columns: List[str] + + +class CSVColumnsEvaluatorConfig(BaseEvaluatorConfig[CSVColumnsEvaluationCriteria]): + """Configuration for the CSV columns evaluator.""" + + name: str = "CSVColumnsEvaluator" + default_evaluation_criteria: CSVColumnsEvaluationCriteria = ( + CSVColumnsEvaluationCriteria(expected_columns=[]) + ) + + +class CSVColumnsEvaluator( + BaseEvaluator[CSVColumnsEvaluationCriteria, CSVColumnsEvaluatorConfig, None] +): + """A custom evaluator that checks if the CSV column names are correctly identified.""" + + @classmethod + def get_evaluator_id(cls) -> str: + return "CSVColumnsEvaluator" + + async def evaluate( + self, + agent_execution: AgentExecution, + evaluation_criteria: CSVColumnsEvaluationCriteria, + ) -> EvaluationResult: + # Check if all expected columns are mentioned in the output + # The agent writes: f"CSV shape {df.shape}\n\nCSV columns {df.columns}" + + columns_found = set() + total_columns = len(evaluation_criteria.expected_columns) + + if total_columns == 0: + return NumericEvaluationResult(score=1.0) + + # Look for column names in agent traces (where print output is captured) + for span in agent_execution.agent_trace: + # Check span attributes + if span.attributes: + for attr_value in span.attributes.values(): + if isinstance(attr_value, str): + for column in evaluation_criteria.expected_columns: + if column in attr_value: + columns_found.add(column) + + # Check span events (where stdout might be captured) + if hasattr(span, 'events') and span.events: + for event in span.events: + if hasattr(event, 'attributes') and event.attributes: + for attr_value in event.attributes.values(): + if isinstance(attr_value, str): + for column in evaluation_criteria.expected_columns: + if column in attr_value: + columns_found.add(column) + + # Also check in the output + if len(columns_found) < total_columns and agent_execution.agent_output: + output_str = str(agent_execution.agent_output) + for column in evaluation_criteria.expected_columns: + if column in output_str: + columns_found.add(column) + + # Calculate score as ratio of found columns + score = len(columns_found) / total_columns + + return NumericEvaluationResult( + score=score, + ) diff --git a/samples/csv-processor/evaluations/evaluators/csv_shape_evaluator.json b/samples/csv-processor/evaluations/evaluators/csv_shape_evaluator.json new file mode 100644 index 000000000..c829b9466 --- /dev/null +++ b/samples/csv-processor/evaluations/evaluators/csv_shape_evaluator.json @@ -0,0 +1,13 @@ +{ + "version": "1.0", + "id": "CSVShapeEvaluator", + "description": "Checks if the CSV shape information (rows, columns) is correct in the output", + "evaluatorSchema": "file://csv_shape_evaluator.py:CSVShapeEvaluator", + "evaluatorConfig": { + "name": "CSVShapeEvaluator", + "defaultEvaluationCriteria": { + "expected_rows": 1, + "expected_columns": 1 + } + } +} diff --git a/samples/csv-processor/evaluations/evaluators/csv_shape_evaluator.py b/samples/csv-processor/evaluations/evaluators/csv_shape_evaluator.py new file mode 100644 index 000000000..7c96e61d3 --- /dev/null +++ b/samples/csv-processor/evaluations/evaluators/csv_shape_evaluator.py @@ -0,0 +1,73 @@ +from uipath.eval.evaluators import ( + BaseEvaluationCriteria, + BaseEvaluator, + BaseEvaluatorConfig, +) +from uipath.eval.models import AgentExecution, EvaluationResult, NumericEvaluationResult + + +class CSVShapeEvaluationCriteria(BaseEvaluationCriteria): + """Evaluation criteria for the CSV shape evaluator.""" + + expected_rows: int + expected_columns: int + + +class CSVShapeEvaluatorConfig(BaseEvaluatorConfig[CSVShapeEvaluationCriteria]): + """Configuration for the CSV shape evaluator.""" + + name: str = "CSVShapeEvaluator" + default_evaluation_criteria: CSVShapeEvaluationCriteria = ( + CSVShapeEvaluationCriteria(expected_rows=1, expected_columns=1) + ) + + +class CSVShapeEvaluator( + BaseEvaluator[CSVShapeEvaluationCriteria, CSVShapeEvaluatorConfig, None] +): + """A custom evaluator that checks if the CSV shape information is correct in the output attachment.""" + + @classmethod + def get_evaluator_id(cls) -> str: + return "CSVShapeEvaluator" + + async def evaluate( + self, + agent_execution: AgentExecution, + evaluation_criteria: CSVShapeEvaluationCriteria, + ) -> EvaluationResult: + # The agent prints: "CSV shape (rows, columns)\nCSV columns [...]" + # We need to find this in the captured output + + expected_shape = f"({evaluation_criteria.expected_rows}, {evaluation_criteria.expected_columns})" + shape_found = False + + # Check agent traces (where print output is captured) + for span in agent_execution.agent_trace: + # Check span attributes + if span.attributes: + for attr_value in span.attributes.values(): + if isinstance(attr_value, str) and expected_shape in attr_value: + shape_found = True + break + + # Check span events (where stdout might be captured) + if hasattr(span, 'events') and span.events: + for event in span.events: + if hasattr(event, 'attributes') and event.attributes: + for attr_value in event.attributes.values(): + if isinstance(attr_value, str) and expected_shape in attr_value: + shape_found = True + break + + if shape_found: + break + + # Check agent output + if not shape_found and agent_execution.agent_output: + output_str = str(agent_execution.agent_output) + shape_found = expected_shape in output_str + + return NumericEvaluationResult( + score=float(shape_found), + ) diff --git a/samples/csv-processor/main.py b/samples/csv-processor/main.py index 761d7bb92..db2a8c8ec 100644 --- a/samples/csv-processor/main.py +++ b/samples/csv-processor/main.py @@ -1,26 +1,57 @@ -import logging import io +import logging +import os + import pandas as pd +from pydantic import BaseModel from uipath.platform import UiPath from uipath.platform.attachments import Attachment -from pydantic import BaseModel from uipath.platform.common import UiPathConfig + logger = logging.getLogger(__name__) + class Input(BaseModel): attachment: Attachment -async def main(input: Input) -> None: - uipath = UiPath() - async with uipath.attachments.open_async(attachment=input.attachment) as (attachment, response): - async for raw_bytes in response.aiter_raw(): - df = pd.read_csv(io.BytesIO(raw_bytes)) - - processing_output = f"CSV shape {df.shape}\n\nCSV columns {df.columns}" - await uipath.jobs.create_attachment_async( - name="processed_output.txt", - content=str(processing_output), - folder_key=UiPathConfig.folder_key, - job_key=UiPathConfig.job_key, + +class Output(BaseModel): + processing_output: str + + +async def main(input: Input) -> Output: + # Check if full_name points to a local file (for testing) + attachment_path = input.attachment.full_name + + if os.path.exists(attachment_path): + # Local file mode for testing + logger.info(f"Reading local file: {attachment_path}") + with open(attachment_path, "rb") as f: + df = pd.read_csv(io.BytesIO(f.read())) + processing_output = ( + f"CSV shape {df.shape}\n\nCSV columns {df.columns.tolist()}" ) + logger.info(f"Processed CSV: {processing_output}") + print(processing_output) + return Output(processing_output=processing_output) + else: + # Platform mode - use attachment API + uipath = UiPath() + async with uipath.attachments.open_async(attachment=input.attachment) as ( + attachment, + response, + ): + async for raw_bytes in response.aiter_raw(): + df = pd.read_csv(io.BytesIO(raw_bytes)) + + processing_output = ( + f"CSV shape {df.shape}\n\nCSV columns {df.columns.tolist()}" + ) + await uipath.jobs.create_attachment_async( + name="processed_output.txt", + content=str(processing_output), + folder_key=UiPathConfig.folder_key, + job_key=UiPathConfig.job_key, + ) + return Output(processing_output=processing_output) diff --git a/samples/csv-processor/pyproject.toml b/samples/csv-processor/pyproject.toml index a8185fe91..d1936f3e5 100644 --- a/samples/csv-processor/pyproject.toml +++ b/samples/csv-processor/pyproject.toml @@ -2,7 +2,6 @@ name = "csv-processor" version = "0.0.1" description = "csv-processor" -authors = [{ name = "John Doe", email = "john.doe@myemail.com" }] dependencies = [ "uipath>=2.4.15, <2.5.0", "pandas>=2.3.3", diff --git a/samples/csv-processor/test-data/large_dataset.csv b/samples/csv-processor/test-data/large_dataset.csv new file mode 100644 index 000000000..2596e085b --- /dev/null +++ b/samples/csv-processor/test-data/large_dataset.csv @@ -0,0 +1,6 @@ +id,name,email,age,department,salary,hire_date,address,city,state,zip,phone,status +1,John Doe,john@example.com,30,Engineering,75000,2020-01-15,123 Main St,New York,NY,10001,555-0101,Active +2,Jane Smith,jane@example.com,28,Marketing,65000,2021-03-20,456 Oak Ave,Los Angeles,CA,90001,555-0102,Active +3,Bob Johnson,bob@example.com,35,Sales,70000,2019-06-10,789 Pine Rd,Chicago,IL,60601,555-0103,Active +4,Alice Brown,alice@example.com,32,Engineering,80000,2018-11-05,321 Elm St,Houston,TX,77001,555-0104,On Leave +5,Charlie Wilson,charlie@example.com,29,HR,60000,2022-02-28,654 Maple Dr,Phoenix,AZ,85001,555-0105,Active diff --git a/samples/csv-processor/test-data/minimal.csv b/samples/csv-processor/test-data/minimal.csv new file mode 100644 index 000000000..a8d8d2c21 --- /dev/null +++ b/samples/csv-processor/test-data/minimal.csv @@ -0,0 +1,2 @@ +id,value +1,test diff --git a/samples/csv-processor/test-data/sales_data.csv b/samples/csv-processor/test-data/sales_data.csv new file mode 100644 index 000000000..155cdd47e --- /dev/null +++ b/samples/csv-processor/test-data/sales_data.csv @@ -0,0 +1,6 @@ +date,product,quantity,price,total +2024-01-01,Laptop,5,999.99,4999.95 +2024-01-02,Mouse,15,25.50,382.50 +2024-01-03,Keyboard,10,75.00,750.00 +2024-01-04,Monitor,3,299.99,899.97 +2024-01-05,Headphones,8,150.00,1200.00 diff --git a/samples/csv-processor/uipath.json b/samples/csv-processor/uipath.json index 646d7c818..2f1e23ce5 100644 --- a/samples/csv-processor/uipath.json +++ b/samples/csv-processor/uipath.json @@ -11,4 +11,4 @@ "includeUvLock": true }, "functions": {"main": "main.py:main"} -} +} \ No newline at end of file diff --git a/src/uipath/_cli/_evals/_evaluator_factory.py b/src/uipath/_cli/_evals/_evaluator_factory.py index 851c1aa0f..87c418b4a 100644 --- a/src/uipath/_cli/_evals/_evaluator_factory.py +++ b/src/uipath/_cli/_evals/_evaluator_factory.py @@ -208,9 +208,12 @@ def _create_coded_evaluator_internal( file_path = Path(file_path_str) if not file_path.is_absolute(): if not file_path.exists(): - # Try using the provided evaluators_dir first if evaluators_dir is not None: - file_path = evaluators_dir / "custom" / file_path_str + # Try the file directly in evaluators_dir first + file_path = evaluators_dir / file_path_str + if not file_path.exists(): + # Fall back to evaluators_dir/custom/ + file_path = evaluators_dir / "custom" / file_path_str else: # Fall back to the old behavior file_path = ( diff --git a/testcases/csv-processor-evals/pyproject.toml b/testcases/csv-processor-evals/pyproject.toml new file mode 100644 index 000000000..144e93cf2 --- /dev/null +++ b/testcases/csv-processor-evals/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "csv-processor-agent" +version = "0.0.1" +description = "CSV processor agent testcase with file input evaluations" +authors = [{ name = "John Doe", email = "john.doe@myemail.com" }] +dependencies = [ + "uipath", + "pandas>=2.3.3", +] +requires-python = ">=3.11" + +[tool.uv.sources] +uipath = { path = "../../", editable = true } diff --git a/testcases/csv-processor-evals/run.sh b/testcases/csv-processor-evals/run.sh new file mode 100755 index 000000000..8a3eec5f3 --- /dev/null +++ b/testcases/csv-processor-evals/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +echo "Syncing dependencies..." +uv sync + +echo "Running evaluations with custom evaluators..." +uv run uipath eval main ../../samples/csv-processor/evaluations/eval-sets/file-input-tests-local.json --no-report --output-file file-input-tests-local.json + +echo "Running assertions..." +uv run python src/assert.py + +echo "Test completed successfully!" diff --git a/testcases/csv-processor-evals/src/assert.py b/testcases/csv-processor-evals/src/assert.py new file mode 100644 index 000000000..999f89412 --- /dev/null +++ b/testcases/csv-processor-evals/src/assert.py @@ -0,0 +1,116 @@ +"""Assertions for csv-processor-evals testcase. + +This script validates that the CSV processor evaluations work correctly by: +1. Reading the evaluation output from file-input-tests-local.json +2. Validating that evaluations have scores equal to 1.0 +""" + +import json +import os + + +def main() -> None: + """Main assertion logic.""" + # Check if output file exists + output_file = "file-input-tests-local.json" + assert os.path.isfile(output_file), ( + f"Evaluation output file '{output_file}' not found" + ) + print(f"✓ Found evaluation output file: {output_file}") + + # Load evaluation results + with open(output_file, "r", encoding="utf-8") as f: + output_data = json.load(f) + + print("✓ Loaded evaluation output") + + # Extract output data + output = output_data + + # Validate structure + assert "evaluationSetResults" in output, "Missing 'evaluationSetResults' in output" + + evaluation_results = output["evaluationSetResults"] + assert len(evaluation_results) > 0, "No evaluation results found" + + print(f"✓ Found {len(evaluation_results)} evaluation result(s)") + + # Validate each evaluation result + passed_count = 0 + failed_count = 0 + skipped_count = 0 + has_perfect_scores = False + + for eval_result in evaluation_results: + eval_name = eval_result.get("evaluationName", "Unknown") + print(f"\n→ Validating: {eval_name}") + + try: + # Validate evaluation results are present + eval_run_results = eval_result.get("evaluationRunResults", []) + if len(eval_run_results) == 0: + print(f" ⊘ Skipping '{eval_name}' (no evaluation run results)") + skipped_count += 1 + continue + + # Check that evaluations have scores = 1.0 + all_passed = True + min_score = 100 + for eval_run in eval_run_results: + evaluator_name = eval_run.get("evaluatorName", "Unknown") + result = eval_run.get("result", {}) + score = result.get("score", 0) + min_score = min(min_score, score) + + # Check if score is 1.0 + if score == 1.0: + has_perfect_scores = True + print(f" ✓ {evaluator_name}: score={score:.1f}") + else: + print(f" ✗ {evaluator_name}: score={score:.1f} (expected 1.0)") + all_passed = False + + if all_passed and min_score == 1.0: + print(f" ✓ All evaluators passed for '{eval_name}' (all scores: 1.0)") + passed_count += 1 + else: + print(f" ✗ Some evaluators failed for '{eval_name}'") + failed_count += 1 + + except Exception as e: + print(f" ✗ Error validating '{eval_name}': {e}") + failed_count += 1 + + # Final summary + print(f"\n{'=' * 60}") + print("Summary:") + print(f" Total evaluations: {passed_count + failed_count + skipped_count}") + print(f" ✓ Passed: {passed_count}") + print(f" ✗ Failed: {failed_count}") + print(f" ⊘ Skipped: {skipped_count}") + print(f"{'=' * 60}") + + assert failed_count == 0, "Some assertions failed" + assert has_perfect_scores, "No evaluation scores equal to 1.0 were found" + + print("\n✅ All assertions passed!") + + # Check __uipath/output.json if it exists + output_file = "__uipath/output.json" + if os.path.isfile(output_file): + print(f"\n✓ Found evaluation output file: {output_file}") + + # Load evaluation results + with open(output_file, "r", encoding="utf-8") as f: + output_data = json.load(f) + + print("✓ Loaded evaluation output") + + # Check status + status = output_data.get("status") + assert status == "successful", f"Evaluation run failed with status: {status}" + print("✓ Evaluation run status: successful") + + +if __name__ == "__main__": + main() diff --git a/testcases/csv-processor-evals/uipath.json b/testcases/csv-processor-evals/uipath.json new file mode 100644 index 000000000..f9241ecb0 --- /dev/null +++ b/testcases/csv-processor-evals/uipath.json @@ -0,0 +1,5 @@ +{ + "functions": { + "main": "../../samples/csv-processor/main.py:main" + } +}