add vcs benchmark capability

Author: breardon2011

sandbox-sidecar/src/runners/e2bRunner.ts

@@ -47,27 +47,32 @@ export class E2BSandboxRunner implements SandboxRunner {
         appendLog?.(chunk);
       };
 
-      // Run terraform init
+      // Run terraform init (with AWS creds if configured for benchmark)
+      const metadata = job.payload.metadata;
       await this.runTerraformCommand(
         sandbox,
         workDir,
         ["init", "-input=false", "-no-color"],
         logs,
         streamLog,
+        metadata,
       );
 
       // Run terraform plan
       const planArgs = ["plan", "-input=false", "-no-color", "-out=tfplan.binary"];
       if (job.payload.isDestroy) {
         planArgs.splice(1, 0, "-destroy");
       }
-      await this.runTerraformCommand(sandbox, workDir, planArgs, logs, streamLog);
+      await this.runTerraformCommand(sandbox, workDir, planArgs, logs, streamLog, metadata);
 
       // Get plan JSON
       const showResult = await this.runTerraformCommand(
         sandbox,
         workDir,
         ["show", "-json", "tfplan.binary"],
+        undefined,
+        undefined,
+        metadata,
       );
 
       const planJSON = showResult.stdout;
@@ -103,30 +108,61 @@ export class E2BSandboxRunner implements SandboxRunner {
           appendLog?.(chunk);
         };
 
-      // Run terraform init
+      // Run terraform init (with AWS creds if configured for benchmark)
+      const metadata = job.payload.metadata;
       await this.runTerraformCommand(
         sandbox,
         workDir,
         ["init", "-input=false", "-no-color"],
         logs,
         streamLog,
+        metadata,
       );
 
       // Run terraform apply/destroy
       const applyCommand = job.payload.isDestroy ? "destroy" : "apply";
-      await this.runTerraformCommand(
+      const applyResult = await this.runTerraformCommand(
         sandbox,
         workDir,
         [applyCommand, "-auto-approve", "-input=false", "-no-color"],
         logs,
         streamLog,
+        metadata,
       );
 
-      // Read the state file
-      const statePath = `${workDir}/terraform.tfstate`;
-      const stateContent = await sandbox.files.read(statePath);
+      // Log the apply output for debugging
+      logger.info({ 
+        stdout: applyResult.stdout.slice(-500),
+        stderr: applyResult.stderr.slice(-500),
+      }, "terraform apply output (last 500 chars)");
+
+      // Use terraform show to get state - works regardless of workspace configuration
+      // This handles both: terraform.tfstate and terraform.tfstate.d/<workspace>/terraform.tfstate
+      let stateBase64 = "";
+      
+      try {
+        const showResult = await this.runTerraformCommand(
+          sandbox,
+          workDir,
+          ["show", "-json"],
+          undefined,
+          undefined,
+          metadata,
+        );
+        
+        if (showResult.stdout && showResult.stdout.trim() !== "{}") {
+          stateBase64 = Buffer.from(showResult.stdout, "utf8").toString("base64");
+          logger.info({ stateSize: showResult.stdout.length }, "captured state via terraform show");
+        } else {
+          logger.info("terraform show returned empty state");
+        }
+      } catch (err) {
+        // State doesn't exist - this is OK for empty applies or destroys
+        logger.warn({ error: err }, "no state found after apply (may be empty apply)");
+      }
+
       const result: SandboxRunResult = {
-        state: Buffer.from(stateContent, "utf8").toString("base64"),
+        state: stateBase64,
       };
 
       return { logs: logs.join(""), result };
@@ -169,6 +205,27 @@ export class E2BSandboxRunner implements SandboxRunner {
     return { sandbox, needsInstall };
   }
 
+  /**
+   * Build environment variables for Terraform execution.
+   * Includes AWS credentials if provided in metadata for benchmark runs.
+   */
+  private buildTerraformEnvs(metadata?: Record<string, string>): Record<string, string> {
+    const envs: Record<string, string> = {
+      TF_IN_AUTOMATION: "1",
+    };
+
+    // Inject AWS credentials if provided (for benchmark runs with real resources)
+    if (metadata?.AWS_ACCESS_KEY_ID) {
+      envs.AWS_ACCESS_KEY_ID = metadata.AWS_ACCESS_KEY_ID;
+      envs.AWS_SECRET_ACCESS_KEY = metadata.AWS_SECRET_ACCESS_KEY || "";
+      envs.AWS_REGION = metadata.AWS_REGION || "us-east-1";
+      // Also set default region for AWS SDK
+      envs.AWS_DEFAULT_REGION = envs.AWS_REGION;
+    }
+
+    return envs;
+  }
+
 
   private async installIacTool(sandbox: Sandbox, engine: string, version: string): Promise<void> {
     logger.info({ engine, version }, "installing IaC tool at runtime");
@@ -231,6 +288,20 @@ export class E2BSandboxRunner implements SandboxRunner {
     // Use gunzip + tar separately for better compatibility across tar versions
     await sandbox.commands.run(`cd ${workDir} && gunzip -c bundle.tar.gz | tar -x --exclude='terraform.tfstate' --exclude='terraform.tfstate.backup'`);
 
+    // Debug: List extracted files to understand archive structure
+    const listResult = await sandbox.commands.run(`find ${workDir} -type f -name "*.tf" | head -20`);
+    logger.info({ 
+      tfFiles: listResult.stdout.trim().split('\n').filter(Boolean),
+      workDir,
+      workingDirectory: job.payload.workingDirectory || '(none)'
+    }, "extracted terraform files");
+
+    // Also list all files for debugging
+    const allFilesResult = await sandbox.commands.run(`ls -la ${workDir}`);
+    logger.info({ 
+      files: allFilesResult.stdout
+    }, "workspace directory listing");
+
     // Determine the execution directory
     const execDir = job.payload.workingDirectory
       ? `${workDir}/${job.payload.workingDirectory}`
@@ -273,6 +344,7 @@ export class E2BSandboxRunner implements SandboxRunner {
     args: string[],
     logBuffer?: string[],
     appendLog?: (chunk: string) => void,
+    metadata?: Record<string, string>,
   ): Promise<{ stdout: string; stderr: string }> {
     const engine = (sandbox as any)._requestedEngine || "terraform";
     const binaryName = engine === "tofu" ? "tofu" : "terraform";
@@ -291,9 +363,7 @@ export class E2BSandboxRunner implements SandboxRunner {
 
     const result = await sandbox.commands.run(cmdStr, {
       cwd,
-      envs: {
-        TF_IN_AUTOMATION: "1",
-      },
+      envs: this.buildTerraformEnvs(metadata),
       onStdout: pipeChunk,
       onStderr: pipeChunk,
     });

taco/cmd/statesman/main.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strings"
 	"time"
 
 	"github.com/diggerhq/digger/opentaco/internal/analytics"
@@ -21,6 +22,7 @@ import (
 	"github.com/diggerhq/digger/opentaco/internal/repositories"
 	"github.com/diggerhq/digger/opentaco/internal/sandbox"
 	"github.com/diggerhq/digger/opentaco/internal/storage"
+	"github.com/google/uuid"
 	"github.com/kelseyhightower/envconfig"
 	"github.com/labstack/echo/v4"
 	echomiddleware "github.com/labstack/echo/v4/middleware"
@@ -101,7 +103,16 @@ func main() {
 		if err != nil {
 			slog.Warn("Failed to list units from storage", "error", err)
 		} else {
+			syncedCount := 0
+			skippedCount := 0
 			for _, unit := range units {
+				// Skip non-unit paths (config-versions, plans, runs, etc.)
+				// Valid unit paths are: {org-uuid}/{unit-uuid}
+				if !isValidUnitPath(unit.ID) {
+					skippedCount++
+					continue
+				}
+				
 				if err := queryStore.SyncEnsureUnit(context.Background(), unit.ID); err != nil {
 					slog.Warn("Failed to sync unit", "unit_id", unit.ID, "error", err)
 					continue
@@ -110,8 +121,9 @@ func main() {
 				if err := queryStore.SyncUnitMetadata(context.Background(), unit.ID, unit.Size, unit.Updated); err != nil {
 					slog.Warn("Failed to sync metadata for unit", "unit_id", unit.ID, "error", err)
 				}
+				syncedCount++
 			}
-			slog.Info("Synced units from storage to database", "count", len(units))
+			slog.Info("Synced units from storage to database", "synced", syncedCount, "skipped_non_units", skippedCount)
 		}
 	} else {
 		slog.Info("Query backend already has units, skipping sync", "count", len(existingUnits))
@@ -275,3 +287,22 @@ func main() {
 	analytics.SendEssential("server_shutdown_complete")
 	slog.Info("Server shutdown complete")
 }
+
+// isValidUnitPath checks if a storage path matches the expected unit format: {org-uuid}/{unit-uuid}
+// This filters out TFE-related paths like config-versions/, plans/, runs/, etc.
+func isValidUnitPath(path string) bool {
+	parts := strings.SplitN(strings.Trim(path, "/"), "/", 2)
+	if len(parts) != 2 {
+		return false
+	}
+
+	// Both parts must be valid UUIDs
+	if _, err := uuid.Parse(parts[0]); err != nil {
+		return false
+	}
+	if _, err := uuid.Parse(parts[1]); err != nil {
+		return false
+	}
+
+	return true
+}

taco/cmd/statesman/statesman

@@ -5,9 +5,11 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"os"
 	"time"
 
 	"github.com/diggerhq/digger/opentaco/internal/analytics"
+	"github.com/diggerhq/digger/opentaco/internal/github"
 	"github.com/diggerhq/digger/opentaco/internal/tfe"
 
 	authpkg "github.com/diggerhq/digger/opentaco/internal/auth"
@@ -362,6 +364,48 @@ func RegisterRoutes(e *echo.Echo, deps Dependencies) {
 		})
 	})
 
+	// Register GitHub webhook for benchmarks (if OPENTACO_GITHUB_TOKEN is set)
+	RegisterGitHubWebhook(e, deps)
+
 	// Register webhook-authenticated internal routes (if OPENTACO_ENABLE_INTERNAL_ENDPOINTS is set)
 	RegisterInternalRoutes(e, deps)
 }
+
+// RegisterGitHubWebhook registers the GitHub webhook endpoint for benchmark operations.
+// This enables /opentaco plan, /opentaco apply, /opentaco destroy commands via PR comments.
+// Required env vars (BOTH must be set to enable):
+//   - OPENTACO_GITHUB_TOKEN: GitHub personal access token or app token
+//   - OPENTACO_GITHUB_WEBHOOK_SECRET: Secret for validating webhook signatures (required for security)
+func RegisterGitHubWebhook(e *echo.Echo, deps Dependencies) {
+	githubToken := os.Getenv("OPENTACO_GITHUB_TOKEN")
+	webhookSecret := os.Getenv("OPENTACO_GITHUB_WEBHOOK_SECRET")
+
+	// Require BOTH token and secret to enable - security by default
+	if githubToken == "" || webhookSecret == "" {
+		if githubToken != "" && webhookSecret == "" {
+			log.Println("WARNING: OPENTACO_GITHUB_TOKEN set but OPENTACO_GITHUB_WEBHOOK_SECRET missing - webhook disabled for security")
+		}
+		return
+	}
+
+	log.Println("Registering GitHub webhook endpoint at /webhooks/github")
+
+	// Create GitHub client
+	ghClient := github.NewClient(githubToken)
+
+	// Create command executor with sandbox and storage
+	executor := github.NewCommandExecutor(
+		ghClient,
+		deps.Sandbox,
+		deps.Repository,
+		deps.BlobStore,
+	)
+
+	// Create webhook handler
+	handler := github.NewWebhookHandler(ghClient, executor)
+
+	// Register the webhook endpoint (no auth required - uses webhook signature validation)
+	e.POST("/webhooks/github", handler.HandleWebhook)
+
+	log.Println("GitHub webhook registered successfully")
+}