of `${suite.name}::${test.name}` keys, runs just those.
+ * The progress bar denominator reflects the filtered count so the
+ * fill bar makes sense for both full runs and retries.
+ */
+async function runAll(only = null) {
+ const allTests = [];
+ for (const suite of suites) {
+ for (const test of suite.tests) {
+ if (!only || only.has(`${suite.name}::${test.name}`)) {
+ allTests.push({ suite, test });
+ }
+ }
+ }
+ const total = allTests.length;
+ updateProgress(0, total, only ? 'retrying…' : 'starting…', false);
let done = 0;
let anyFailed = false;
- for (const suite of suites) {
- const suiteResult = { name: suite.name, tests: [] };
- for (const test of suite.tests) {
- done++;
- updateProgress(done, total, `${suite.name} — ${test.name}`, anyFailed);
- const entry = { name: test.name, status: 'pass', error: null, time: 0 };
- const t0 = performance.now();
- try {
- if (suite.beforeEach) await suite.beforeEach();
- await test.fn();
- if (suite.afterEach) await suite.afterEach();
- } catch (err) {
- entry.status = 'fail';
- entry.error = err.message;
- anyFailed = true;
- // Always run afterEach on failure too
- try { if (suite.afterEach) await suite.afterEach(); } catch {}
- }
- entry.time = Math.round(performance.now() - t0);
- suiteResult.tests.push(entry);
+ const suiteMap = new Map();
+ for (const { suite, test } of allTests) {
+ done++;
+ updateProgress(done, total, `${suite.name} — ${test.name}`, anyFailed);
+ const entry = { name: test.name, status: 'pass', error: null, time: 0 };
+ const t0 = performance.now();
+ try {
+ if (suite.beforeEach) await suite.beforeEach();
+ await test.fn();
+ if (suite.afterEach) await suite.afterEach();
+ } catch (err) {
+ entry.status = 'fail';
+ entry.error = err.message;
+ anyFailed = true;
+ // Always run afterEach on failure too
+ try { if (suite.afterEach) await suite.afterEach(); } catch {}
}
- results.push(suiteResult);
+ entry.time = Math.round(performance.now() - t0);
+ if (!suiteMap.has(suite.name)) suiteMap.set(suite.name, { name: suite.name, tests: [] });
+ suiteMap.get(suite.name).tests.push(entry);
}
hideProgress();
- return results;
+ return [...suiteMap.values()];
}
// ── UI Panel ──
@@ -313,7 +323,7 @@ function createPanel() {
function renderResults(results) {
const output = panel.querySelector('#test-output');
const summary = panel.querySelector('#test-summary');
- let total = 0, passed = 0, failed = 0, totalTime = 0;
+ let total = 0, passed = 0, failed = 0, flaky = 0, totalTime = 0;
let html = '';
for (const suite of results) {
@@ -322,20 +332,33 @@ function renderResults(results) {
for (const t of suite.tests) {
total++;
totalTime += t.time;
- if (t.status === 'pass') {
+ if (t.status === 'pass' && t.retried) {
+ // Failed first time, passed on auto-retry → flaky.
+ // Counts as passed in the summary but coloured amber +
+ // shows the original error so the user can decide whether
+ // to investigate or move on.
+ passed++;
+ flaky++;
+ html += `⚠ ${esc(t.name)} ${t.time}ms (flaky — passed on retry)
`;
+ if (t.originalError) {
+ html += `first run: ${esc(t.originalError)}
`;
+ }
+ } else if (t.status === 'pass') {
passed++;
html += `✓ ${esc(t.name)} ${t.time}ms
`;
} else {
failed++;
- html += `✗ ${esc(t.name)} ${t.time}ms
`;
+ const retryNote = t.retried ? ' (failed twice)' : '';
+ html += `✗ ${esc(t.name)} ${t.time}ms${retryNote}
`;
html += `${esc(t.error)}
`;
}
}
html += ``;
}
output.innerHTML = html;
- summary.textContent = `${passed}/${total} passed, ${failed} failed — ${totalTime}ms`;
- summary.style.color = failed ? '#fb7185' : '#4ade80';
+ const flakyNote = flaky ? `, ${flaky} flaky` : '';
+ summary.textContent = `${passed}/${total} passed, ${failed} failed${flakyNote} — ${totalTime}ms`;
+ summary.style.color = failed ? '#fb7185' : (flaky ? '#fbbf24' : '#4ade80');
}
function esc(s) { return String(s).replace(//g, '>'); }
@@ -366,6 +389,42 @@ async function run() {
// 4. Run tests
const results = await runAll();
+ // 4b. Auto-retry: collect every failed (suite, test) pair, restore the
+ // backup so the retry runs against the same baseline state the first
+ // pass started from, then re-run *only* those tests. Pass-on-retry is
+ // marked so the user can tell flaky from genuinely broken at a glance.
+ const failedKeys = new Set();
+ for (const sr of results) {
+ for (const t of sr.tests) {
+ if (t.status === 'fail') failedKeys.add(`${sr.name}::${t.name}`);
+ }
+ }
+ if (failedKeys.size) {
+ await closeAll();
+ try {
+ await restoreBackup();
+ await window._reloadAll?.();
+ await sleep(200);
+ } catch {}
+ const retryResults = await runAll(failedKeys);
+ const retryMap = new Map();
+ for (const sr of retryResults) {
+ for (const t of sr.tests) {
+ retryMap.set(`${sr.name}::${t.name}`, t);
+ }
+ }
+ for (const sr of results) {
+ for (let i = 0; i < sr.tests.length; i++) {
+ const key = `${sr.name}::${sr.tests[i].name}`;
+ const retried = retryMap.get(key);
+ if (retried) {
+ const originalError = sr.tests[i].error;
+ sr.tests[i] = { ...retried, retried: true, originalError };
+ }
+ }
+ }
+ }
+
// 5. Clean up any leftover UI state
await closeAll();