fix(ci): Fix E2E test flakiness with stable checks instead of retries

Replace retry-based approach (PR #5830) with deterministic fixes:

### Simulator stability (Cirrus Labs Tart VMs)
- `wait_for_boot: true` / `erase_before_boot: false` on simulator-action
- `xcrun simctl bootstatus booted -b` to block until boot completes
- Settings.app warm-up for SpringBoard/system service initialization
- `MAESTRO_DRIVER_STARTUP_TIMEOUT` bumped to 180s

### e2e-v2 test runner (cli.mjs)
- Per-flow process isolation via individual `maestro test` calls
- Maestro driver warm-up flow before real tests (non-fatal)
- crash.yml runs first so the next flow verifies post-crash recovery
- `execSync` → `execFileSync` to avoid shell interpolation
- SENTRY_AUTH_TOKEN redaction in debug logs

### Sample application test fixes
- Search all envelopes for app start transaction (slow VM delivery)
- Sort envelopes by timestamp for deterministic ordering
- Allow-list for TTID/TTFD ops (`navigation`, `ui.load`)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: antonis

.github/workflows/e2e-v2.yml

@@ -508,12 +508,26 @@ jobs:
         with:
           model: ${{ env.IOS_DEVICE }}
           os_version: ${{ env.IOS_VERSION }}
+          wait_for_boot: true
+          erase_before_boot: false
+
+      - name: Wait for iOS simulator to be fully ready
+        if: ${{ steps.platform-check.outputs.skip != 'true' && matrix.platform == 'ios' }}
+        run: |
+          # Wait for boot to complete at the system level
+          xcrun simctl bootstatus booted -b
+          # Launch and dismiss Settings.app to ensure SpringBoard and system services
+          # are fully initialized — this avoids Maestro connecting to a half-booted
+          # simulator on Cirrus Labs Tart VMs.
+          xcrun simctl launch booted com.apple.Preferences
+          sleep 5
+          xcrun simctl terminate booted com.apple.Preferences
 
       - name: Run tests on iOS
         if: ${{ steps.platform-check.outputs.skip != 'true' && matrix.platform == 'ios' }}
         env:
           # Increase timeout for Maestro iOS driver startup (default is 60s, some CI runners need more time)
-          MAESTRO_DRIVER_STARTUP_TIMEOUT: 120000
+          MAESTRO_DRIVER_STARTUP_TIMEOUT: 180000
         run: ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test
 
       - name: Upload logs

.github/workflows/sample-application.yml

@@ -14,7 +14,7 @@ concurrency:
 env:
   SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
   MAESTRO_VERSION: '2.3.0'
-  MAESTRO_DRIVER_STARTUP_TIMEOUT: 90000 # Increase timeout from default 30s to 90s for CI stability
+  MAESTRO_DRIVER_STARTUP_TIMEOUT: 180000 # Increase timeout from default 30s to 180s for CI stability
   RN_SENTRY_POD_NAME: RNSentry
   IOS_APP_ARCHIVE_PATH: sentry-react-native-sample.app.zip
   ANDROID_APP_ARCHIVE_PATH: sentry-react-native-sample.apk.zip
@@ -332,6 +332,16 @@ jobs:
         with:
           model: ${{ env.IOS_DEVICE }}
           os_version: ${{ env.IOS_VERSION }}
+          wait_for_boot: true
+          erase_before_boot: false
+
+      - name: Wait for iOS simulator to be fully ready
+        if: ${{ steps.platform-check.outputs.skip != 'true' && matrix.platform == 'ios' }}
+        run: |
+          xcrun simctl bootstatus booted -b
+          xcrun simctl launch booted com.apple.Preferences
+          sleep 5
+          xcrun simctl terminate booted com.apple.Preferences
 
       - name: Run iOS Tests
         if: ${{ steps.platform-check.outputs.skip != 'true' && matrix.platform == 'ios' }}

dev-packages/e2e-tests/cli.mjs

@@ -290,35 +290,96 @@ if (actions.includes('test')) {
   if (!sentryAuthToken) {
     console.log('Skipping maestro test due to unavailable or empty SENTRY_AUTH_TOKEN');
   } else {
+    const maestroDir = path.join(e2eDir, 'maestro');
+    const flowFiles = fs.readdirSync(maestroDir)
+      .filter(f => f.endsWith('.yml') && !f.startsWith('utils'))
+      .sort((a, b) => {
+        // Run crash.yml first — it kills the app via nativeCrash(), and the
+        // NEXT flow's launchTestAppClear.yml verifies post-crash recovery
+        // (killApp + launchApp + assertTestReady). Per-flow process isolation
+        // ensures the crashed Maestro session doesn't affect other flows.
+        if (a === 'crash.yml') return -1;
+        if (b === 'crash.yml') return 1;
+        return a.localeCompare(b);
+      });
+
+    console.log(`Discovered ${flowFiles.length} Maestro flows: ${flowFiles.join(', ')}`);
+
+    // Warm up Maestro's driver connection before running test flows.
+    // The first Maestro launchApp after simulator boot can fail on Cirrus
+    // Labs Tart VMs because the IDB/XCUITest driver isn't fully connected.
+    // Running a lightweight warmup flow ensures the driver is ready.
+    const warmupFlow = path.join('maestro', 'utils', 'warmup.yml');
+    console.log('\n--- Warming up Maestro driver ---');
     try {
-      execSync(
-        `maestro test maestro \
-          --env=APP_ID="${appId}" \
-          --env=SENTRY_AUTH_TOKEN="${sentryAuthToken}" \
-          --debug-output maestro-logs \
-          --flatten-debug-output`,
-        {
-          stdio: 'inherit',
-          cwd: e2eDir,
-        },
-      );
-    } finally {
-      // Always redact sensitive data, even if the test fails
-      const redactScript = `
-        if [[ "$(uname)" == "Darwin" ]]; then
-          find ./maestro-logs -type f -exec sed -i '' "s/${sentryAuthToken}/[REDACTED]/g" {} +
-          echo 'Redacted sensitive data from logs on MacOS'
-        else
-          find ./maestro-logs -type f -exec sed -i "s/${sentryAuthToken}/[REDACTED]/g" {} +
-          echo 'Redacted sensitive data from logs on Ubuntu'
-        fi
-      `;
+      execFileSync('maestro', [
+        'test',
+        warmupFlow,
+        '--env', `APP_ID=${appId}`,
+        '--env', `SENTRY_AUTH_TOKEN=${sentryAuthToken}`,
+      ], {
+        stdio: 'inherit',
+        cwd: e2eDir,
+      });
+      console.log('--- Maestro driver warm-up: OK ---');
+    } catch (error) {
+      console.warn('--- Maestro driver warm-up failed (non-fatal, continuing with tests) ---');
+    }
+
+    const failedFlows = [];
 
+    // Run each flow in its own process to prevent crash cascade —
+    // when crash.yml kills the app, a shared Maestro session would fail
+    // all subsequent flows.
+    for (const flow of flowFiles) {
+      const flowPath = path.join('maestro', flow);
+      console.log(`\n--- Running flow: ${flow} ---`);
       try {
-        execSync(redactScript, { stdio: 'inherit', cwd: e2eDir, shell: '/bin/bash' });
+        execFileSync('maestro', [
+          'test',
+          flowPath,
+          '--env', `APP_ID=${appId}`,
+          '--env', `SENTRY_AUTH_TOKEN=${sentryAuthToken}`,
+          '--debug-output', 'maestro-logs',
+          '--flatten-debug-output',
+        ], {
+          stdio: 'inherit',
+          cwd: e2eDir,
+        });
+        console.log(`--- Flow ${flow}: PASSED ---`);
       } catch (error) {
-        console.warn('Failed to redact sensitive data from logs:', error.message);
+        console.error(`--- Flow ${flow}: FAILED ---`);
+        failedFlows.push(flow);
       }
     }
+
+    // Always redact sensitive data, even if some tests failed
+    try {
+      const logDir = path.join(e2eDir, 'maestro-logs');
+      if (fs.existsSync(logDir)) {
+        const redactFiles = (dir) => {
+          for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+            const fullPath = path.join(dir, entry.name);
+            if (entry.isDirectory()) {
+              redactFiles(fullPath);
+            } else {
+              const content = fs.readFileSync(fullPath, 'utf8');
+              if (content.includes(sentryAuthToken)) {
+                fs.writeFileSync(fullPath, content.replaceAll(sentryAuthToken, '[REDACTED]'));
+              }
+            }
+          }
+        };
+        redactFiles(logDir);
+        console.log('Redacted sensitive data from logs');
+      }
+    } catch (error) {
+      console.warn('Failed to redact sensitive data from logs:', error.message);
+    }
+
+    if (failedFlows.length > 0) {
+      console.error(`\nFailed flows: ${failedFlows.join(', ')}`);
+      process.exit(1);
+    }
   }
 }

dev-packages/e2e-tests/maestro/crash.yml

@@ -2,8 +2,4 @@ appId: ${APP_ID}
 jsEngine: graaljs
 ---
 - runFlow: utils/launchTestAppClear.yml
-- tapOn: "Crash"
-
-- launchApp
-
-- runFlow: utils/assertTestReady.yml
+- tapOn: 'Crash'

dev-packages/e2e-tests/maestro/utils/sentryApi.js

@@ -62,8 +62,22 @@ switch (fetch) {
     break;
   }
   case 'replay': {
-    const event = json(fetchFromSentry(`${baseUrl}/events/${eventId}/json/`));
-    const replayId = event._dsc.replay_id.replace(/\-/g, '');
+    // The replay_id may not be available immediately after the event is
+    // created — Sentry needs time to process and link the replay. Retry
+    // fetching the event until _dsc.replay_id is present.
+    let replayId;
+    for (let attempt = 0; attempt < RETRY_COUNT; attempt++) {
+      const event = json(fetchFromSentry(`${baseUrl}/events/${eventId}/json/`));
+      if (event._dsc && event._dsc.replay_id) {
+        replayId = event._dsc.replay_id.replace(/\-/g, '');
+        break;
+      }
+      console.log(`replay_id not yet available, retrying: ${attempt + 1}/${RETRY_COUNT}`);
+      sleep(RETRY_INTERVAL);
+    }
+    if (!replayId) {
+      throw new Error(`replay_id not available after ${RETRY_COUNT} retries`);
+    }
     const replay = json(fetchFromSentry(`${baseUrl}/replays/${replayId}/`));
     const segment = fetchFromSentry(`${baseUrl}/replays/${replayId}/videos/0/`);
 

dev-packages/e2e-tests/maestro/utils/warmup.yml

@@ -0,0 +1,13 @@
+appId: ${APP_ID}
+jsEngine: graaljs
+---
+# Warm up Maestro's IDB/XCUITest driver connection on the simulator.
+# The very first Maestro launchApp after simulator boot can fail on Cirrus
+# Labs Tart VMs — running a lightweight flow first ensures the driver is
+# fully connected before real test flows start.
+- launchApp:
+    clearState: true
+- extendedWaitUntil:
+    visible: "E2E Tests Ready"
+    timeout: 300_000 # 5 minutes
+- killApp

samples/react-native/e2e/tests/captureErrorScreenTransaction/captureErrorsScreenTransaction.test.ts

@@ -31,15 +31,23 @@ describe('Capture Errors Screen Transaction', () => {
   });
 
   it('envelope contains transaction context', async () => {
-    const envelope = getErrorsEnvelope();
-
-    const items = envelope[1];
-    const transactions = items.filter(([header]) => header.type === 'transaction');
-    const appStartTransaction = transactions.find(([_header, payload]) => {
-      const event = payload as any;
-      return event.transaction === 'ErrorsScreen' &&
-            event.contexts?.trace?.origin === 'auto.app.start';
-    });
+    // The app start transaction may arrive in a separate envelope on slow CI VMs,
+    // so search all matching envelopes instead of just the first one.
+    const allEnvelopes = sentryServer.getAllEnvelopes(
+      containingTransactionWithName('ErrorsScreen'),
+    );
+
+    let appStartTransaction: EventItem | undefined;
+    for (const envelope of allEnvelopes) {
+      const items = envelope[1];
+      const transactions = items.filter(([header]) => header.type === 'transaction') as EventItem[];
+      appStartTransaction = transactions.find(([_header, payload]) => {
+        const event = payload as any;
+        return event.transaction === 'ErrorsScreen' &&
+              event.contexts?.trace?.origin === 'auto.app.start';
+      });
+      if (appStartTransaction) break;
+    }
 
     expect(appStartTransaction).toBeDefined();
 

samples/react-native/e2e/tests/captureSpaceflightNewsScreenTransaction/captureSpaceflightNewsScreenTransaction.test.ts

@@ -42,6 +42,13 @@ describe('Capture Spaceflight News Screen Transaction', () => {
     await waitForSpaceflightNewsTx;
 
     newsEnvelopes = sentryServer.getAllEnvelopes(containingNewsScreen);
+    // Sort by transaction timestamp — envelope delivery order may vary on slow CI VMs,
+    // but test assertions depend on chronological order.
+    newsEnvelopes.sort((a, b) => {
+      const aItem = getItemOfTypeFrom<EventItem>(a, 'transaction');
+      const bItem = getItemOfTypeFrom<EventItem>(b, 'transaction');
+      return (aItem?.[1]?.timestamp ?? 0) - (bItem?.[1]?.timestamp ?? 0);
+    });
     allTransactionEnvelopes = sentryServer.getAllEnvelopes(
       containingTransaction,
     );
@@ -64,9 +71,11 @@ describe('Capture Spaceflight News Screen Transaction', () => {
     allTransactionEnvelopes
       .filter(envelope => {
         const item = getItemOfTypeFrom<EventItem>(envelope, 'transaction');
-        // Only check navigation transactions, not user interaction transactions
-        // User interaction transactions (ui.action.touch) don't have time-to-display measurements
-        return item?.[1]?.contexts?.trace?.op !== 'ui.action.touch';
+        // Only navigation and app start transactions have time-to-display measurements.
+        // Filter with an allow-list — other ops like 'ui.action.touch' or
+        // 'navigation.processing' do not include TTID/TTFD.
+        const op = item?.[1]?.contexts?.trace?.op;
+        return op === 'navigation' || op === 'ui.load';
       })
       .forEach(envelope => {
         expectToContainTimeToDisplayMeasurements(