diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 448435f91..e328d6dad 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -67,6 +67,9 @@ jobs: APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ secrets.APPLICATIONINSIGHTS_CONNECTION_STRING }} APP_INSIGHTS_APP_ID: ${{ secrets.APP_INSIGHTS_APP_ID }} APP_INSIGHTS_API_KEY: ${{ secrets.APP_INSIGHTS_API_KEY }} + + # Azure Blob Storage for performance metrics + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} working-directory: testcases/${{ matrix.testcase }} run: | # If any errors occur execution will stop with exit code @@ -80,6 +83,14 @@ jobs: bash run.sh bash ../common/validate_output.sh + - name: Upload testcase artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: artifacts-${{ matrix.testcase }}-${{ matrix.environment }} + path: testcases/${{ matrix.testcase }}/artifacts/ + if-no-files-found: ignore + summarize-results: needs: [integration-tests] runs-on: ubuntu-latest diff --git a/testcases/performance-testcase/README.md b/testcases/performance-testcase/README.md new file mode 100644 index 000000000..1b5529093 --- /dev/null +++ b/testcases/performance-testcase/README.md @@ -0,0 +1,424 @@ +# Performance Testcase + +This testcase measures the **framework overhead** of `uipath run` by profiling a minimal function that does almost no work. This helps identify performance bottlenecks in the CLI, runtime, and platform infrastructure. + +## What It Does + +1. **Runs a minimal function** ([main.py](main.py)) that only performs basic string operations +2. **Profiles with py-spy** (speedscope format) to capture execution timing +3. **Profiles memory usage** with Python's tracemalloc +4. **Collects performance metrics** including: + - Total execution time + - Time spent in user function vs framework overhead + - Time spent in imports/module loading + - Memory usage (peak and current) + - File sizes of profiling artifacts +5. **Uploads metrics to Azure Blob Storage** for historical tracking and analysis + +## Test Function + +```python +def main(input: EchoIn) -> EchoOut: + result = [] + for _ in range(input.repeat): + line = input.message + if input.prefix: + line = f"{input.prefix}: {line}" + result.append(line) + return EchoOut(message="\n".join(result)) +``` + +This function is deliberately minimal to isolate framework overhead. + +## Metrics Collected + +The testcase generates a `metrics.json` file with: + +```json +{ + "timestamp": "2026-01-15T10:30:45.123456+00:00", + "framework": "uipath", + "testcase": "performance-testcase", + "function": "main (echo function - minimal work)", + "timing": { + "total_time_seconds": 2.456, + "total_time_ms": 2456.78, + "user_function": { + "time_ms": 12.34, + "time_seconds": 0.012, + "percentage": 0.50 + }, + "framework_overhead": { + "time_ms": 412.89, + "time_seconds": 0.413, + "percentage": 16.81 + }, + "import_time": { + "time_ms": 2031.55, + "time_seconds": 2.032, + "percentage": 82.69 + }, + "sample_count": 24567, + "unique_frames": 245 + }, + "memory": { + "current_bytes": 45678912, + "peak_bytes": 52341256, + "current_mb": 43.56, + "peak_mb": 49.91 + }, + "execution_time_seconds": 2.458, + "file_sizes": { + "profile_json": 123456, + "memory_profile_json": 5678 + }, + "environment": { + "python_version": "3.11.0", + "platform": "linux", + "ci": "true", + "runner": "Linux", + "github_run_id": "12345", + "github_sha": "abc123", + "branch": "main" + } +} +``` + +### Key Metrics Explained + +- **framework**: Framework discriminator (`uipath`, `uipath-langgraph`, or `uipath-llamaindex`) +- **timing.total_time_seconds**: Total execution time from start to finish +- **timing.user_function**: Time spent executing the user's `main()` function +- **timing.framework_overhead**: Time spent in framework code (excluding imports) +- **timing.import_time**: Time spent loading Python modules +- **memory.peak_mb**: Peak memory usage during execution +- **memory.current_mb**: Memory usage at the end of execution +- **execution_time_seconds**: Wall-clock time measured by tracemalloc + +## Artifacts Generated + +| File | Format | Purpose | +|------|--------|---------| +| `profile.json` | Speedscope JSON | CPU profiling data with timing information - view at [speedscope.app](https://speedscope.app) | +| `memory_profile.json` | JSON | Memory profiling data with peak/current usage and top allocations | +| `metrics.json` | JSON | Combined metrics (timing + memory) for Azure Data Explorer ingestion | + +## Azure Blob Storage Setup + +### 1. Create Storage Account + +```bash +# Using Azure CLI +az storage account create \ + --name uipathperfmetrics \ + --resource-group uipath-performance \ + --location eastus \ + --sku Standard_LRS + +# Create container +az storage container create \ + --name performance-metrics \ + --account-name uipathperfmetrics +``` + +### 2. Get Connection String + +```bash +az storage account show-connection-string \ + --name uipathperfmetrics \ + --resource-group uipath-performance \ + --output tsv +``` + +### 3. Configure GitHub Secret + +1. Go to repository Settings → Secrets and variables → Actions +2. Add new secret: `AZURE_STORAGE_CONNECTION_STRING` +3. Paste the connection string from step 2 + +### Blob Naming Convention + +Metrics are uploaded with hierarchical names including the framework discriminator: + +``` +{framework}/{branch}/{github_run_id}/{timestamp}_metrics.json +``` + +Example: +``` +uipath/main/12345678/20260115_103045_metrics.json +uipath-langgraph/main/12345679/20260115_110230_metrics.json +uipath-llamaindex/feature/optimize-imports/12345680/20260115_112015_metrics.json +``` + +This allows: +- **Comparing frameworks** (uipath vs uipath-langgraph vs uipath-llamaindex) +- Tracking metrics over time +- Comparing branches within the same framework +- Correlating with CI runs + +## Running Locally + +### Prerequisites + +```bash +# Install dependencies +uv add py-spy azure-storage-blob + +# Set environment variables +export CLIENT_ID="your-client-id" +export CLIENT_SECRET="your-client-secret" +export BASE_URL="https://cloud.uipath.com/your-org" +export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=https;..." + +# Optional: Set framework discriminator (defaults to "uipath") +# export FRAMEWORK="uipath" # or "uipath-langgraph" or "uipath-llamaindex" +``` + +### Run Test + +```bash +cd testcases/performance-testcase +bash run.sh +``` + +**Note**: The `FRAMEWORK` environment variable defaults to `uipath`. For testing other frameworks: +```bash +export FRAMEWORK="uipath-langgraph" +bash run.sh +``` + +### View Results + +```bash +# View combined metrics +cat artifacts/metrics.json | jq + +# View timing breakdown +cat artifacts/metrics.json | jq '.timing' + +# View memory usage +cat artifacts/memory_profile.json | jq '.memory' + +# Upload to speedscope.app for interactive timeline view +# Go to https://speedscope.app +# Load artifacts/profile.json +``` + +## CI/CD Integration + +The testcase runs automatically in GitHub Actions across three environments: +- Alpha +- Staging +- Cloud (production) + +Each run: +1. Profiles the agent execution +2. Collects metrics +3. Uploads to Azure Blob Storage +4. Uploads artifacts to GitHub Actions + +### Workflow File + +See [`.github/workflows/integration_tests.yml`](../../.github/workflows/integration_tests.yml) + +## Azure Data Explorer (ADX) Ingestion + +### Create ADX Table + +```kql +.create table PerformanceMetrics ( + Timestamp: datetime, + Testcase: string, + Function: string, + Framework: string, + TotalTimeSeconds: real, + UserFunctionTimeSeconds: real, + UserFunctionPercentage: real, + FrameworkOverheadSeconds: real, + FrameworkOverheadPercentage: real, + ImportTimeSeconds: real, + ImportPercentage: real, + SampleCount: int, + UniqueFrames: int, + PeakMemoryMB: real, + CurrentMemoryMB: real, + ExecutionTimeSeconds: real, + ProfileSizeBytes: long, + MemoryProfileSizeBytes: long, + PythonVersion: string, + Platform: string, + CI: bool, + RunnerOS: string, + GitHubRunId: string, + GitHubSHA: string, + Branch: string +) +``` + +### Create Data Connection + +```kql +.create table PerformanceMetrics ingestion json mapping 'PerformanceMetricsMapping' +``` +```json +[ + {"column": "Timestamp", "path": "$.timestamp", "datatype": "datetime"}, + {"column": "Testcase", "path": "$.testcase", "datatype": "string"}, + {"column": "Function", "path": "$.function", "datatype": "string"}, + {"column": "Framework", "path": "$.framework", "datatype": "string"}, + {"column": "TotalTimeSeconds", "path": "$.timing.total_time_seconds", "datatype": "real"}, + {"column": "UserFunctionTimeSeconds", "path": "$.timing.user_function.time_seconds", "datatype": "real"}, + {"column": "UserFunctionPercentage", "path": "$.timing.user_function.percentage", "datatype": "real"}, + {"column": "FrameworkOverheadSeconds", "path": "$.timing.framework_overhead.time_seconds", "datatype": "real"}, + {"column": "FrameworkOverheadPercentage", "path": "$.timing.framework_overhead.percentage", "datatype": "real"}, + {"column": "ImportTimeSeconds", "path": "$.timing.import_time.time_seconds", "datatype": "real"}, + {"column": "ImportPercentage", "path": "$.timing.import_time.percentage", "datatype": "real"}, + {"column": "SampleCount", "path": "$.timing.sample_count", "datatype": "int"}, + {"column": "UniqueFrames", "path": "$.timing.unique_frames", "datatype": "int"}, + {"column": "PeakMemoryMB", "path": "$.memory.peak_mb", "datatype": "real"}, + {"column": "CurrentMemoryMB", "path": "$.memory.current_mb", "datatype": "real"}, + {"column": "ExecutionTimeSeconds", "path": "$.execution_time_seconds", "datatype": "real"}, + {"column": "ProfileSizeBytes", "path": "$.file_sizes.profile_json", "datatype": "long"}, + {"column": "MemoryProfileSizeBytes", "path": "$.file_sizes.memory_profile_json", "datatype": "long"}, + {"column": "PythonVersion", "path": "$.environment.python_version", "datatype": "string"}, + {"column": "Platform", "path": "$.environment.platform", "datatype": "string"}, + {"column": "CI", "path": "$.environment.ci", "datatype": "bool"}, + {"column": "RunnerOS", "path": "$.environment.runner", "datatype": "string"}, + {"column": "GitHubRunId", "path": "$.environment.github_run_id", "datatype": "string"}, + {"column": "GitHubSHA", "path": "$.environment.github_sha", "datatype": "string"}, + {"column": "Branch", "path": "$.environment.branch", "datatype": "string"} +] +``` + +### Setup Event Grid Ingestion + +```bash +# Create data connection from Blob Storage to ADX +az kusto data-connection event-grid create \ + --cluster-name uipath-performance-cluster \ + --database-name PerformanceDB \ + --data-connection-name blob-ingestion \ + --resource-group uipath-performance \ + --storage-account-resource-id "/subscriptions/.../uipathperfmetrics" \ + --event-hub-resource-id "/subscriptions/.../eventhub" \ + --consumer-group '$Default' \ + --table-name PerformanceMetrics \ + --mapping-rule-name PerformanceMetricsMapping \ + --data-format json \ + --blob-storage-event-type Microsoft.Storage.BlobCreated +``` + +### Query Metrics in ADX + +```kql +// View recent metrics for a specific framework +PerformanceMetrics +| where Timestamp > ago(7d) and Framework == "uipath" +| project Timestamp, Framework, Branch, TotalTimeSeconds, ImportPercentage, PeakMemoryMB, UserFunctionPercentage +| order by Timestamp desc + +// Compare frameworks - execution time breakdown +PerformanceMetrics +| where Timestamp > ago(30d) and Branch == "main" +| summarize + AvgTotalTime = avg(TotalTimeSeconds), + AvgUserFunctionTime = avg(UserFunctionTimeSeconds), + AvgFrameworkOverhead = avg(FrameworkOverheadSeconds), + AvgImportTime = avg(ImportTimeSeconds), + AvgPeakMemoryMB = avg(PeakMemoryMB) + by Framework +| order by AvgTotalTime desc + +// Compare branches within a framework +PerformanceMetrics +| where Timestamp > ago(30d) and Framework == "uipath" +| summarize + AvgTotalTime = avg(TotalTimeSeconds), + AvgImportPct = avg(ImportPercentage), + AvgFrameworkPct = avg(FrameworkOverheadPercentage) + by Branch +| order by AvgTotalTime desc + +// Trend over time for a framework - import percentage +PerformanceMetrics +| where Branch == "main" and Framework == "uipath" +| summarize ImportPct = avg(ImportPercentage) by bin(Timestamp, 1d) +| render timechart + +// Compare all three frameworks side by side +PerformanceMetrics +| where Timestamp > ago(7d) and Branch == "main" +| summarize + AvgTotalTime = avg(TotalTimeSeconds), + AvgUserFunctionPct = avg(UserFunctionPercentage), + AvgFrameworkPct = avg(FrameworkOverheadPercentage), + AvgImportPct = avg(ImportPercentage), + AvgPeakMemoryMB = avg(PeakMemoryMB) + by Framework +| order by AvgTotalTime desc + +// Memory usage trend +PerformanceMetrics +| where Branch == "main" and Framework == "uipath" +| summarize AvgMemory = avg(PeakMemoryMB) by bin(Timestamp, 1d) +| render timechart +``` + +## Troubleshooting + +### Metrics not uploading + +1. Check `AZURE_STORAGE_CONNECTION_STRING` is set: + ```bash + echo $AZURE_STORAGE_CONNECTION_STRING + ``` + +2. Verify storage account exists and is accessible: + ```bash + az storage container list \ + --connection-string "$AZURE_STORAGE_CONNECTION_STRING" + ``` + +3. Check script output for error messages + +### Profile files empty or missing + +1. Ensure py-spy is installed: `pip show py-spy` +2. Check process permissions (py-spy needs ptrace access on Linux) +3. Verify `uv run uipath run` executes successfully + +### Timing metrics seem off + +Timing metrics are extracted from speedscope profile data: +- **User function time**: Samples where stack contains `main` from testcases directory +- **Import time**: Samples where stack contains `_find_and_load` or `_load_unlocked` +- **Framework overhead**: All other samples (excluding imports and user function) + +The percentages should add up to 100%. If they don't, check that the speedscope JSON is valid. + +### Memory profiling failed + +Memory profiling uses Python's `tracemalloc` which may fail if: +1. The subprocess can't be executed +2. Insufficient permissions +3. Python crashes during execution + +Check the error output from `profile_memory.py` for details. + +## Related Files + +- [collect_metrics.py](collect_metrics.py) - Metrics collection script (parses speedscope and memory data) +- [profile_memory.py](profile_memory.py) - Memory profiling script using tracemalloc +- [main.py](main.py) - Minimal test function +- [run.sh](run.sh) - Test runner with profiling commands +- [../../.github/workflows/integration_tests.yml](../../.github/workflows/integration_tests.yml) - CI workflow + +## Future Enhancements + +- [ ] Add more granular timing breakdowns (e.g., HTTP requests, database queries) +- [ ] Track process startup time separately +- [ ] Measure network latency to UiPath services +- [ ] Compare performance across Python versions +- [ ] Add alerting for performance regressions (e.g., >10% slowdown) +- [ ] Generate performance regression reports in PRs diff --git a/testcases/performance-testcase/collect_metrics.py b/testcases/performance-testcase/collect_metrics.py new file mode 100644 index 000000000..e3cd66278 --- /dev/null +++ b/testcases/performance-testcase/collect_metrics.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +"""Collect performance metrics and upload to Azure Blob Storage.""" + +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def load_timing_metrics(artifacts_dir: str = "artifacts") -> dict: + """Load timing metrics from instrumentation files. + + Returns timing breakdown: + - User code time (from main.py instrumentation) + - Total execution time (from py-spy speedscope profile) + - Framework overhead (calculated as total - user code) + """ + artifacts_path = Path(artifacts_dir) + + # Load user code timing (instrumented in main.py) + user_timing_path = artifacts_path / "user_code_timing.json" + user_code_time = 0 + + if user_timing_path.exists(): + try: + with open(user_timing_path, "r", encoding="utf-8") as f: + user_timing = json.load(f) + user_code_time = user_timing.get("user_code_time_seconds", 0) + except json.JSONDecodeError: + pass + + # Extract total execution time from py-spy speedscope profile + profile_path = artifacts_path / "profile.json" + total_time = 0 + if profile_path.exists(): + try: + with open(profile_path, "r", encoding="utf-8") as f: + speedscope_data = json.load(f) + # Get weights from first profile (main process) + profiles = speedscope_data.get("profiles", []) + if profiles: + weights = profiles[0].get("weights", []) + # Sum all weights (in microseconds) to get total time + total_time_us = sum(weights) + total_time = total_time_us / 1_000_000 # Convert to seconds + except (json.JSONDecodeError, KeyError, IndexError): + pass + + # Calculate framework overhead + framework_overhead = total_time - user_code_time + + # Calculate percentages + user_percentage = (user_code_time / total_time * 100) if total_time > 0 else 0 + framework_percentage = (framework_overhead / total_time * 100) if total_time > 0 else 0 + + return { + "total_time_seconds": round(total_time, 3), + "total_time_ms": round(total_time * 1000, 2), + "user_code_time": { + "time_ms": round(user_code_time * 1000, 2), + "time_seconds": round(user_code_time, 6), + "percentage": round(user_percentage, 2) + }, + "framework_overhead": { + "time_ms": round(framework_overhead * 1000, 2), + "time_seconds": round(framework_overhead, 3), + "percentage": round(framework_percentage, 2) + } + } + + +def load_memory_metrics(artifacts_dir: str = "artifacts") -> dict: + """Load memory metrics from memray stats output.""" + memory_stats_path = Path(artifacts_dir) / "memory_stats.json" + if not memory_stats_path.exists(): + return {} + + try: + with open(memory_stats_path, "r", encoding="utf-8") as f: + return json.load(f) + except json.JSONDecodeError: + return {} + + +def get_file_size(file_path: str) -> int: + """Get file size in bytes.""" + if not Path(file_path).exists(): + return 0 + return Path(file_path).stat().st_size + + +def collect_metrics( + artifacts_dir: str = "artifacts", + framework: str | None = None, + testcase: str | None = None, +) -> dict: + """Collect all performance metrics from artifacts directory. + + Args: + artifacts_dir: Directory containing profile artifacts + framework: Framework discriminator (uipath, uipath-langgraph, uipath-llamaindex) + If None, auto-detects from FRAMEWORK env var or defaults to 'uipath' + testcase: Testcase name (defaults to TESTCASE env var or current directory name) + """ + artifacts_path = Path(artifacts_dir) + + # Auto-detect framework if not specified + if framework is None: + framework = os.getenv("FRAMEWORK", "uipath") + + # Auto-detect testcase from current directory if not specified + if testcase is None: + testcase = os.getenv("TESTCASE", Path.cwd().name) + + # Load timing metrics (user code + total execution) + timing_metrics = load_timing_metrics(artifacts_dir) + + # Load memory metrics (from memray) + memory_metrics = load_memory_metrics(artifacts_dir) + + # Get artifact file sizes + file_sizes = { + "profile_json": get_file_size(str(artifacts_path / "profile.json")), + "user_code_timing_json": get_file_size(str(artifacts_path / "user_code_timing.json")), + "memory_bin": get_file_size(str(artifacts_path / "memory.bin")), + "memory_stats_json": get_file_size(str(artifacts_path / "memory_stats.json")), + } + + # Build complete metrics object + metrics = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "framework": framework, # Discriminator: uipath, uipath-langgraph, uipath-llamaindex + "testcase": testcase, + "function": "main (echo function - minimal work)", + "timing": timing_metrics, + "memory": memory_metrics, + "file_sizes": file_sizes, + "environment": { + "python_version": sys.version.split()[0], + "platform": sys.platform, + "ci": os.getenv("CI", "false"), + "runner": os.getenv("RUNNER_OS", "unknown"), + "github_run_id": os.getenv("GITHUB_RUN_ID", "local"), + "github_sha": os.getenv("GITHUB_SHA", "unknown"), + "branch": os.getenv("GITHUB_REF_NAME", "unknown"), + }, + } + + return metrics + + +def save_metrics_json(metrics: dict, output_path: str = "artifacts/metrics.json"): + """Save metrics to JSON file.""" + Path(output_path).parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(metrics, f, indent=2) + print(f"✓ Metrics saved to {output_path}") + + +def upload_to_blob_storage( + file_path: str, + connection_string: str | None = None, + container_name: str = "performance-metrics", + blob_name: str | None = None, +) -> bool: + """Upload file to Azure Blob Storage. + + Args: + file_path: Path to file to upload + connection_string: Azure Storage connection string (or uses AZURE_STORAGE_CONNECTION_STRING env var) + container_name: Blob container name + blob_name: Name for blob (defaults to filename with timestamp) + + Returns: + True if upload succeeded, False otherwise + """ + try: + from azure.storage.blob import BlobServiceClient + except ImportError: + print("⚠️ azure-storage-blob not installed. Run: pip install azure-storage-blob") + return False + + connection_string = connection_string or os.getenv("AZURE_STORAGE_CONNECTION_STRING") + if not connection_string: + print("⚠️ AZURE_STORAGE_CONNECTION_STRING not set") + return False + + if not Path(file_path).exists(): + print(f"⚠️ File not found: {file_path}") + return False + + # Generate blob name with timestamp if not provided + if blob_name is None: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + filename = Path(file_path).name + blob_name = f"{timestamp}_{filename}" + + try: + # Create BlobServiceClient + blob_service_client = BlobServiceClient.from_connection_string(connection_string) + + # Create container if it doesn't exist + try: + container_client = blob_service_client.get_container_client(container_name) + if not container_client.exists(): + blob_service_client.create_container(container_name) + print(f"✓ Created container: {container_name}") + except Exception as e: + print(f"⚠️ Container check/creation warning: {e}") + + # Upload file + blob_client = blob_service_client.get_blob_client( + container=container_name, blob=blob_name + ) + + with open(file_path, "rb") as data: + blob_client.upload_blob(data, overwrite=True) + + blob_url = blob_client.url + print(f"✓ Uploaded to Azure Blob Storage: {blob_url}") + return True + + except Exception as e: + print(f"⚠️ Upload failed: {e}") + return False + + +def main(): + """Main entry point.""" + print("=" * 60) + print("Performance Metrics Collection") + print("=" * 60) + + # Collect metrics + print("\n📊 Collecting metrics...") + metrics = collect_metrics() + + # Print summary + print("\n📈 Metrics Summary:") + print(f" Framework: {metrics['framework']}") + print(f" Testcase: {metrics['testcase']}") + + # Timing metrics + timing = metrics.get('timing', {}) + if timing: + print(f"\n⏱️ Timing Metrics:") + print(f" Total execution time: {timing.get('total_time_seconds', 0)}s ({timing.get('total_time_ms', 0)}ms)") + + user_code = timing.get('user_code_time', {}) + print(f" User code time: {user_code.get('time_seconds', 0)}s ({user_code.get('percentage', 0)}%)") + + framework = timing.get('framework_overhead', {}) + print(f" Framework overhead: {framework.get('time_seconds', 0)}s ({framework.get('percentage', 0)}%)") + + # Memory metrics + memory = metrics.get('memory', {}) + if memory: + print(f"\n💾 Memory Metrics:") + print(f" Peak memory: {memory.get('peak_mb', 0)} MB") + if 'total_allocations' in memory: + print(f" Total allocations: {memory.get('total_allocations', 0):,}") + + # Save metrics JSON + print("\n💾 Saving metrics...") + metrics_path = "artifacts/metrics.json" + save_metrics_json(metrics, metrics_path) + + # Upload to Azure Blob Storage if connection string is available + connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING") + if connection_string: + print("\n☁️ Uploading to Azure Blob Storage...") + + # Generate blob name with metadata including framework discriminator + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + framework = metrics["framework"] + branch = metrics["environment"]["branch"].replace("/", "_") + run_id = metrics["environment"]["github_run_id"] + blob_name = f"{framework}/{branch}/{run_id}/{timestamp}_metrics.json" + + upload_to_blob_storage( + metrics_path, + connection_string=connection_string, + container_name="performance-metrics", + blob_name=blob_name, + ) + else: + print("\n⚠️ AZURE_STORAGE_CONNECTION_STRING not set - skipping upload") + print(" To enable upload, set environment variable:") + print(" export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=...'") + + print("\n✅ Metrics collection complete!") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/testcases/performance-testcase/extract_memory_stats.py b/testcases/performance-testcase/extract_memory_stats.py new file mode 100644 index 000000000..b27011753 --- /dev/null +++ b/testcases/performance-testcase/extract_memory_stats.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Extract memory statistics from memray binary output.""" + +import json +import subprocess +import sys +from pathlib import Path + + +def extract_memory_stats(): + """Extract memory stats from memray binary file using memray stats command.""" + print("Extracting memory stats from memray output...") + + memray_file = Path("artifacts/memory.bin") + if not memray_file.exists(): + print(f"⚠️ Memray file not found: {memray_file}") + return False + + try: + # Run memray stats to get peak memory and total allocations + result = subprocess.run( + ["memray", "stats", str(memray_file)], + capture_output=True, + text=True, + check=True + ) + + # Parse the stats output + # Format: + # Total memory allocated: X.XX GB + # Total allocations: X + # Histogram of allocation size: ... + # ... + # High watermark: X.XX GB + output_lines = result.stdout.split("\n") + + peak_memory_mb = 0 + total_allocations = 0 + + for line in output_lines: + if "High watermark" in line or "peak memory" in line.lower(): + # Extract memory value (could be in KB, MB, or GB) + parts = line.split(":") + if len(parts) >= 2: + value_str = parts[1].strip().split()[0] + try: + value = float(value_str) + # Check units + if "GB" in line: + peak_memory_mb = value * 1024 + elif "MB" in line: + peak_memory_mb = value + elif "KB" in line: + peak_memory_mb = value / 1024 + except ValueError: + pass + + if "Total allocations" in line: + parts = line.split(":") + if len(parts) >= 2: + try: + total_allocations = int(parts[1].strip()) + except ValueError: + pass + + # Save memory metrics + memory_metrics = { + "peak_mb": round(peak_memory_mb, 2), + "total_allocations": total_allocations + } + + output_path = Path("artifacts/memory_stats.json") + with open(output_path, "w") as f: + json.dump(memory_metrics, f, indent=2) + + print(f"✓ Memory stats saved to {output_path}") + print(f" Peak memory: {memory_metrics['peak_mb']} MB") + print(f" Total allocations: {memory_metrics['total_allocations']}") + + return True + + except subprocess.CalledProcessError as e: + print(f"⚠️ Failed to extract memory stats: {e}") + print(f" stdout: {e.stdout}") + print(f" stderr: {e.stderr}") + return False + except Exception as e: + print(f"⚠️ Error extracting memory stats: {e}") + return False + + +if __name__ == "__main__": + success = extract_memory_stats() + sys.exit(0 if success else 1) diff --git a/testcases/performance-testcase/main.py b/testcases/performance-testcase/main.py new file mode 100644 index 000000000..2fc7c1343 --- /dev/null +++ b/testcases/performance-testcase/main.py @@ -0,0 +1,48 @@ +import json +import logging +import time +from dataclasses import dataclass +from pathlib import Path + + +logger = logging.getLogger(__name__) + + +@dataclass +class EchoIn: + message: str + repeat: int | None = 1 + prefix: str | None = None + + +@dataclass +class EchoOut: + message: str + + +def main(input: EchoIn) -> EchoOut: + # Record start time + start_time = time.perf_counter() + + result = [] + + for _ in range(input.repeat): + line = input.message + if input.prefix: + line = f"{input.prefix}: {line}" + result.append(line) + + # Record end time + end_time = time.perf_counter() + user_code_time = end_time - start_time + + # Write timing to file for later collection + timing_file = Path("artifacts/user_code_timing.json") + timing_file.parent.mkdir(parents=True, exist_ok=True) + timing_file.write_text(json.dumps({ + "user_code_time_seconds": user_code_time, + "start_time": start_time, + "end_time": end_time + })) + + return EchoOut(message="\n".join(result)) diff --git a/testcases/performance-testcase/profile_memory.py b/testcases/performance-testcase/profile_memory.py new file mode 100644 index 000000000..ca7a48226 --- /dev/null +++ b/testcases/performance-testcase/profile_memory.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Profile memory usage and total execution time during agent execution.""" + +import json +import subprocess +import sys +import time +from pathlib import Path + + +def profile_agent_execution(): + """Run the agent and measure total execution time. + + Note: Memory profiling of subprocesses requires psutil which adds overhead. + For now, we focus on accurate timing measurement. + """ + print("Starting performance profiling...") + + # Record total execution start time + total_start_time = time.perf_counter() + + # Run the agent and measure total time + try: + result = subprocess.run( + [ + "uv", "run", "uipath", "run", "main", + '{"message": "abc", "repeat": 2, "prefix": "xyz"}' + ], + capture_output=True, + text=True, + check=True + ) + success = True + error = None + except subprocess.CalledProcessError as e: + success = False + error = str(e) + result = e + + # Record total execution end time + total_end_time = time.perf_counter() + total_execution_time = total_end_time - total_start_time + + # Build metrics + metrics = { + "total_execution_time_seconds": round(total_execution_time, 3), + "success": success, + "error": error + } + + # Save metrics + output_path = Path("artifacts/total_execution.json") + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(metrics, f, indent=2) + + print(f"✓ Performance profile saved to {output_path}") + print(f" Total execution time: {metrics['total_execution_time_seconds']}s") + + return metrics + + +if __name__ == "__main__": + try: + profile_agent_execution() + except Exception as e: + print(f"⚠️ Performance profiling failed: {e}", file=sys.stderr) + sys.exit(1) diff --git a/testcases/performance-testcase/pyproject.toml b/testcases/performance-testcase/pyproject.toml new file mode 100644 index 000000000..cc8781624 --- /dev/null +++ b/testcases/performance-testcase/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "agent" +version = "0.0.1" +description = "agent" +authors = [{ name = "John Doe", email = "john.doe@myemail.com" }] +dependencies = [ + "uipath", +] +requires-python = ">=3.11" + +[tool.uv.sources] +uipath = { path = "../../", editable = true } \ No newline at end of file diff --git a/testcases/performance-testcase/run.sh b/testcases/performance-testcase/run.sh new file mode 100644 index 000000000..14eb82fa9 --- /dev/null +++ b/testcases/performance-testcase/run.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -e + +echo "Syncing dependencies..." +uv sync + +echo "Installing profiling tools and Azure SDK..." +uv add py-spy memray azure-storage-blob + +echo "Authenticating with UiPath..." +uv run uipath auth --client-id="$CLIENT_ID" --client-secret="$CLIENT_SECRET" --base-url="$BASE_URL" + +echo "Run init..." +uv run uipath init + +echo "Packing agent..." +uv run uipath pack + +echo "Creating artifacts directory..." +mkdir -p artifacts + +echo "Run agent with py-spy profiling (speedscope JSON with timing data)" +uv run py-spy record --subprocesses -f speedscope -o artifacts/profile.json -- uv run uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}' + +echo "Run agent with memray memory profiling" +uv run memray run --output artifacts/memory.bin -m uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}' 2>&1 | tee artifacts/memray_output.txt + +echo "Extract memory stats from memray output" +uv run python extract_memory_stats.py + +echo "Collecting performance metrics and uploading to Azure..." +uv run python collect_metrics.py diff --git a/testcases/performance-testcase/src/assert.py b/testcases/performance-testcase/src/assert.py new file mode 100644 index 000000000..025ddf212 --- /dev/null +++ b/testcases/performance-testcase/src/assert.py @@ -0,0 +1,32 @@ +import json +import os + +# Check NuGet package +uipath_dir = ".uipath" +assert os.path.exists(uipath_dir), "NuGet package directory (.uipath) not found" + +nupkg_files = [f for f in os.listdir(uipath_dir) if f.endswith(".nupkg")] +assert nupkg_files, "NuGet package file (.nupkg) not found in .uipath directory" + +print(f"NuGet package found: {nupkg_files[0]}") + +# Check agent output file +output_file = "__uipath/output.json" +assert os.path.isfile(output_file), "Agent output file not found" + +print("Agent output file found") + +# Check status and required fields +with open(output_file, "r", encoding="utf-8") as f: + output_data = json.load(f) + +# Check status +status = output_data.get("status") +assert status == "successful", f"Agent execution failed with status: {status}" + +print("Agent execution status: successful") + +# Check required fields for ticket classification agent +assert "output" in output_data, "Missing 'output' field in agent response" + +print("Required fields validation passed") diff --git a/testcases/performance-testcase/uipath.json b/testcases/performance-testcase/uipath.json new file mode 100644 index 000000000..ee698d9df --- /dev/null +++ b/testcases/performance-testcase/uipath.json @@ -0,0 +1,5 @@ +{ + "functions": { + "main": "main.py:main" + } +} \ No newline at end of file diff --git a/tmpclaude-46e7-cwd b/tmpclaude-46e7-cwd new file mode 100644 index 000000000..9c7a044e4 --- /dev/null +++ b/tmpclaude-46e7-cwd @@ -0,0 +1 @@ +/c/Work/uipath-python2 diff --git a/tmpclaude-6847-cwd b/tmpclaude-6847-cwd new file mode 100644 index 000000000..9c7a044e4 --- /dev/null +++ b/tmpclaude-6847-cwd @@ -0,0 +1 @@ +/c/Work/uipath-python2