Skip to content

Commit 4ae2ca9

Browse files
antonisclaude
andauthored
fix(ci): Fix failing E2E without retries (#5888)
* fix(ci): Fix E2E test flakiness with stable checks instead of retries --------- Co-authored-by: Claude Opus 4.6 (1M context) <[email protected]>
1 parent f3c7863 commit 4ae2ca9

File tree

11 files changed

+215
-56
lines changed

11 files changed

+215
-56
lines changed

.github/workflows/e2e-v2.yml

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -508,12 +508,26 @@ jobs:
508508
with:
509509
model: ${{ env.IOS_DEVICE }}
510510
os_version: ${{ env.IOS_VERSION }}
511+
wait_for_boot: true
512+
erase_before_boot: false
513+
514+
- name: Wait for iOS simulator to be fully ready
515+
if: ${{ steps.platform-check.outputs.skip != 'true' && matrix.platform == 'ios' }}
516+
run: |
517+
# Wait for boot to complete at the system level
518+
xcrun simctl bootstatus booted -b
519+
# Launch and dismiss Settings.app to ensure SpringBoard and system services
520+
# are fully initialized — this avoids Maestro connecting to a half-booted
521+
# simulator on Cirrus Labs Tart VMs.
522+
xcrun simctl launch booted com.apple.Preferences
523+
sleep 5
524+
xcrun simctl terminate booted com.apple.Preferences
511525
512526
- name: Run tests on iOS
513527
if: ${{ steps.platform-check.outputs.skip != 'true' && matrix.platform == 'ios' }}
514528
env:
515529
# Increase timeout for Maestro iOS driver startup (default is 60s, some CI runners need more time)
516-
MAESTRO_DRIVER_STARTUP_TIMEOUT: 120000
530+
MAESTRO_DRIVER_STARTUP_TIMEOUT: 180000
517531
run: ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test
518532

519533
- name: Upload logs

.github/workflows/sample-application.yml

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ concurrency:
1414
env:
1515
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
1616
MAESTRO_VERSION: '2.3.0'
17-
MAESTRO_DRIVER_STARTUP_TIMEOUT: 90000 # Increase timeout from default 30s to 90s for CI stability
17+
MAESTRO_DRIVER_STARTUP_TIMEOUT: 180000 # Increase timeout from default 30s to 180s for CI stability
1818
RN_SENTRY_POD_NAME: RNSentry
1919
IOS_APP_ARCHIVE_PATH: sentry-react-native-sample.app.zip
2020
ANDROID_APP_ARCHIVE_PATH: sentry-react-native-sample.apk.zip
@@ -337,6 +337,35 @@ jobs:
337337
with:
338338
model: ${{ env.IOS_DEVICE }}
339339
os_version: ${{ env.IOS_VERSION }}
340+
wait_for_boot: true
341+
erase_before_boot: false
342+
343+
- name: Wait for iOS simulator to be fully ready
344+
run: |
345+
xcrun simctl bootstatus booted -b
346+
xcrun simctl launch booted com.apple.Preferences
347+
sleep 5
348+
xcrun simctl terminate booted com.apple.Preferences
349+
350+
- name: Warm up Maestro driver on iOS
351+
continue-on-error: true
352+
working-directory: ${{ env.REACT_NATIVE_SAMPLE_PATH }}
353+
run: |
354+
xcrun simctl install booted sentryreactnativesample.app
355+
WARMUP=$(mktemp /tmp/maestro-warmup-XXXXXX.yml)
356+
cat > "$WARMUP" << 'YML'
357+
appId: io.sentry.reactnative.sample
358+
---
359+
- launchApp:
360+
clearState: true
361+
- extendedWaitUntil:
362+
visible: "Sentry React Native Sample"
363+
timeout: 120000
364+
- killApp
365+
YML
366+
sed -i '' 's/^ //' "$WARMUP"
367+
maestro test "$WARMUP" || true
368+
rm -f "$WARMUP"
340369
341370
- name: Run iOS Tests
342371
working-directory: ${{ env.REACT_NATIVE_SAMPLE_PATH }}

dev-packages/e2e-tests/cli.mjs

Lines changed: 97 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -290,35 +290,108 @@ if (actions.includes('test')) {
290290
if (!sentryAuthToken) {
291291
console.log('Skipping maestro test due to unavailable or empty SENTRY_AUTH_TOKEN');
292292
} else {
293+
const maestroDir = path.join(e2eDir, 'maestro');
294+
const flowFiles = fs.readdirSync(maestroDir)
295+
.filter(f => f.endsWith('.yml') && !f.startsWith('utils'))
296+
.sort((a, b) => {
297+
// Run crash.yml last — it kills the app via nativeCrash(), and
298+
// post-crash simulator state can be flaky on Cirrus Labs Tart VMs.
299+
if (a === 'crash.yml') return 1;
300+
if (b === 'crash.yml') return -1;
301+
return a.localeCompare(b);
302+
});
303+
304+
console.log(`Discovered ${flowFiles.length} Maestro flows: ${flowFiles.join(', ')}`);
305+
306+
// Warm up Maestro's driver connection before running test flows.
307+
// The first Maestro launchApp after simulator boot can fail on Cirrus
308+
// Labs Tart VMs because the IDB/XCUITest driver isn't fully connected.
309+
// Running a lightweight warmup flow ensures the driver is ready.
310+
const warmupFlow = path.join('maestro', 'utils', 'warmup.yml');
311+
console.log('Warming up Maestro driver...');
293312
try {
294-
execSync(
295-
`maestro test maestro \
296-
--env=APP_ID="${appId}" \
297-
--env=SENTRY_AUTH_TOKEN="${sentryAuthToken}" \
298-
--debug-output maestro-logs \
299-
--flatten-debug-output`,
300-
{
301-
stdio: 'inherit',
302-
cwd: e2eDir,
303-
},
304-
);
305-
} finally {
306-
// Always redact sensitive data, even if the test fails
307-
const redactScript = `
308-
if [[ "$(uname)" == "Darwin" ]]; then
309-
find ./maestro-logs -type f -exec sed -i '' "s/${sentryAuthToken}/[REDACTED]/g" {} +
310-
echo 'Redacted sensitive data from logs on MacOS'
311-
else
312-
find ./maestro-logs -type f -exec sed -i "s/${sentryAuthToken}/[REDACTED]/g" {} +
313-
echo 'Redacted sensitive data from logs on Ubuntu'
314-
fi
315-
`;
313+
execFileSync('maestro', [
314+
'test',
315+
warmupFlow,
316+
'--env', `APP_ID=${appId}`,
317+
'--env', `SENTRY_AUTH_TOKEN=${sentryAuthToken}`,
318+
], {
319+
stdio: 'pipe',
320+
cwd: e2eDir,
321+
});
322+
} catch (error) {
323+
console.warn('Maestro driver warm-up failed (non-fatal, continuing with tests)');
324+
}
316325

326+
const results = [];
327+
328+
// Run each flow in its own process to prevent crash cascade —
329+
// when crash.yml kills the app, a shared Maestro session would fail
330+
// all subsequent flows.
331+
console.log('Waiting for flows to complete...');
332+
for (const flow of flowFiles) {
333+
const flowPath = path.join('maestro', flow);
334+
const startTime = Date.now();
317335
try {
318-
execSync(redactScript, { stdio: 'inherit', cwd: e2eDir, shell: '/bin/bash' });
336+
execFileSync('maestro', [
337+
'test',
338+
flowPath,
339+
'--env', `APP_ID=${appId}`,
340+
'--env', `SENTRY_AUTH_TOKEN=${sentryAuthToken}`,
341+
'--debug-output', 'maestro-logs',
342+
'--flatten-debug-output',
343+
], {
344+
stdio: 'pipe',
345+
cwd: e2eDir,
346+
});
347+
const elapsed = Math.round((Date.now() - startTime) / 1000);
348+
const name = flow.replace('.yml', '');
349+
results.push({ name, passed: true, elapsed });
350+
console.log(`[Passed] ${name} (${elapsed}s)`);
319351
} catch (error) {
320-
console.warn('Failed to redact sensitive data from logs:', error.message);
352+
const elapsed = Math.round((Date.now() - startTime) / 1000);
353+
const name = flow.replace('.yml', '');
354+
const output = (error.stdout?.toString() || '') + (error.stderr?.toString() || '');
355+
const detail = output.split('\n').find(l =>
356+
l.includes('App crashed') || l.includes('Element not found') || l.includes('FAILED')) || '';
357+
results.push({ name, passed: false, elapsed, detail });
358+
console.log(`[Failed] ${name} (${elapsed}s)${detail ? ` (${detail.trim()})` : ''}`);
359+
// Dump Maestro output for failed flows to aid debugging
360+
if (output) {
361+
console.log(`\n--- ${name} output ---\n${output.trim()}\n--- end ${name} output ---\n`);
362+
}
321363
}
322364
}
365+
366+
const failedFlows = results.filter(r => !r.passed).map(r => r.name);
367+
368+
// Always redact sensitive data, even if some tests failed
369+
try {
370+
const logDir = path.join(e2eDir, 'maestro-logs');
371+
if (fs.existsSync(logDir)) {
372+
const redactFiles = (dir) => {
373+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
374+
const fullPath = path.join(dir, entry.name);
375+
if (entry.isDirectory()) {
376+
redactFiles(fullPath);
377+
} else {
378+
const content = fs.readFileSync(fullPath, 'utf8');
379+
if (content.includes(sentryAuthToken)) {
380+
fs.writeFileSync(fullPath, content.replaceAll(sentryAuthToken, '[REDACTED]'));
381+
}
382+
}
383+
}
384+
};
385+
redactFiles(logDir);
386+
console.log('Redacted sensitive data from logs');
387+
}
388+
} catch (error) {
389+
console.warn('Failed to redact sensitive data from logs:', error.message);
390+
}
391+
392+
if (failedFlows.length > 0) {
393+
console.error(`\nFailed flows: ${failedFlows.join(', ')}`);
394+
process.exit(1);
395+
}
323396
}
324397
}

dev-packages/e2e-tests/maestro/crash.yml

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,4 @@ appId: ${APP_ID}
22
jsEngine: graaljs
33
---
44
- runFlow: utils/launchTestAppClear.yml
5-
- tapOn: "Crash"
6-
7-
- launchApp
8-
9-
- runFlow: utils/assertTestReady.yml
5+
- tapOn: 'Crash'

dev-packages/e2e-tests/maestro/feedback/captureFlow-android.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@ jsEngine: graaljs
77
# Show feedback button
88
- tapOn: 'Feedback'
99

10-
# Open feedback widget
10+
# Wait for feedback widget button to appear, then open it
11+
- extendedWaitUntil:
12+
visible: 'Report a Bug'
13+
timeout: 10_000
1114
- tapOn: 'Report a Bug'
1215

1316
# Assert that the feedback form is visible

dev-packages/e2e-tests/maestro/feedback/happyFlow-android.yml

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,11 @@ jsEngine: graaljs
77
# Show feedback button
88
- tapOn: 'Feedback'
99

10-
# Open feedback widget
11-
- tapOn: 'Report a Bug'
12-
13-
# Assert that the feedback form is visible
10+
# Wait for feedback widget button to appear, then open it
1411
- extendedWaitUntil:
1512
visible: 'Report a Bug'
16-
timeout: 5_000
13+
timeout: 10_000
14+
- tapOn: 'Report a Bug'
1715

1816
# Fill out name field
1917
- tapOn: 'Your Name'

dev-packages/e2e-tests/maestro/utils/sentryApi.js

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,16 @@ switch (fetch) {
6262
break;
6363
}
6464
case 'replay': {
65+
// The replay_id is set by the SDK on the event before sending (in
66+
// contexts.replay.replay_id or _dsc.replay_id). It should be present
67+
// when the event is fetched from the API.
6568
const event = json(fetchFromSentry(`${baseUrl}/events/${eventId}/json/`));
66-
const replayId = event._dsc.replay_id.replace(/\-/g, '');
69+
const rawReplayId = (event.contexts && event.contexts.replay && event.contexts.replay.replay_id)
70+
|| (event._dsc && event._dsc.replay_id);
71+
if (!rawReplayId) {
72+
throw new Error('replay_id not available on the event');
73+
}
74+
const replayId = rawReplayId.replace(/\-/g, '');
6775
const replay = json(fetchFromSentry(`${baseUrl}/replays/${replayId}/`));
6876
const segment = fetchFromSentry(`${baseUrl}/replays/${replayId}/videos/0/`);
6977

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
appId: ${APP_ID}
2+
jsEngine: graaljs
3+
---
4+
# Warm up Maestro's IDB/XCUITest driver connection on the simulator.
5+
# The very first Maestro launchApp after simulator boot can fail on Cirrus
6+
# Labs Tart VMs — running a lightweight flow first ensures the driver is
7+
# fully connected before real test flows start.
8+
- launchApp:
9+
clearState: true
10+
- extendedWaitUntil:
11+
visible: "E2E Tests Ready"
12+
timeout: 300_000 # 5 minutes
13+
- killApp

dev-packages/e2e-tests/src/EndToEndTests.tsx

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,15 @@ const EndToEndTestsScreen = (): JSX.Element => {
1919

2020
// WARNING: This is only for testing purposes.
2121
// We only do this to render the eventId onto the UI for end to end tests.
22-
client.getOptions().beforeSend = (e) => {
23-
setEventId(e.event_id);
24-
return e;
22+
// Chain with existing beforeSend (set by mobileReplayIntegration) to
23+
// preserve replay_id processing.
24+
const existingBeforeSend = client.getOptions().beforeSend;
25+
client.getOptions().beforeSend = async (e, hint) => {
26+
const result = existingBeforeSend ? await existingBeforeSend(e, hint) : e;
27+
if (result) {
28+
setEventId(result.event_id);
29+
}
30+
return result;
2531
};
2632

2733
setIsReady(true);

samples/react-native/e2e/tests/captureErrorScreenTransaction/captureErrorsScreenTransaction.test.ts

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,23 @@ describe('Capture Errors Screen Transaction', () => {
3131
});
3232

3333
it('envelope contains transaction context', async () => {
34-
const envelope = getErrorsEnvelope();
35-
36-
const items = envelope[1];
37-
const transactions = items.filter(([header]) => header.type === 'transaction');
38-
const appStartTransaction = transactions.find(([_header, payload]) => {
39-
const event = payload as any;
40-
return event.transaction === 'ErrorsScreen' &&
41-
event.contexts?.trace?.origin === 'auto.app.start';
42-
});
34+
// The app start transaction may arrive in a separate envelope on slow CI VMs,
35+
// so search all matching envelopes instead of just the first one.
36+
const allEnvelopes = sentryServer.getAllEnvelopes(
37+
containingTransactionWithName('ErrorsScreen'),
38+
);
39+
40+
let appStartTransaction: EventItem | undefined;
41+
for (const envelope of allEnvelopes) {
42+
const items = envelope[1];
43+
const transactions = items.filter(([header]) => header.type === 'transaction') as EventItem[];
44+
appStartTransaction = transactions.find(([_header, payload]) => {
45+
const event = payload as any;
46+
return event.transaction === 'ErrorsScreen' &&
47+
event.contexts?.trace?.origin === 'auto.app.start';
48+
});
49+
if (appStartTransaction) break;
50+
}
4351

4452
expect(appStartTransaction).toBeDefined();
4553

0 commit comments

Comments
 (0)