Flake Hunt Toolkit

The Challenge

Technical Solution

Built a comprehensive toolkit for detecting, analyzing, and eliminating flaky tests with trace-on-failure debugging.

import { defineConfig } from '@playwright/test';

export default defineConfig({
  // Retry flaky tests to confirm stability
  retries: process.env.CI ? 2 : 0,

  // Capture trace on first retry for debugging
  use: {
    trace: 'on-first-retry',
    screenshot: 'only-on-failure',
    video: 'retain-on-failure',
  },

  // Reporter for flake detection
  reporter: [
    ['html'],
    ['json', { outputFile: 'test-results/results.json' }],
    ['./reporters/flake-detector.ts']
  ],
});

import type { Reporter, TestCase, TestResult } from '@playwright/test/reporter';

class FlakeDetector implements Reporter {
  private flakyTests: Map<string, number> = new Map();

  onTestEnd(test: TestCase, result: TestResult) {
    // Track tests that passed on retry (flaky)
    if (result.retry > 0 && result.status === 'passed') {
      const count = this.flakyTests.get(test.title) || 0;
      this.flakyTests.set(test.title, count + 1);
    }
  }

  onEnd() {
    if (this.flakyTests.size > 0) {
      console.log('⚠️ Flaky tests detected:');
      this.flakyTests.forEach((count, name) => {
        console.log(`  - ${name} (${count} retries)`);
      });
    }
  }
}

Key Implementation Details

• Trace-on-failure workflow capturing screenshots, videos, and network logs
• Custom reporter for tracking retry patterns and identifying chronic flakes
• Historical flakiness dashboard tracking test reliability over time
• Automatic quarantine system for tests exceeding flakiness threshold
• Root cause analysis templates for common flake patterns

Tech Stack

Impact & Results

• Reduced test flakiness by 92% through systematic identification and fixes
• Improved CI pipeline reliability to 99.2% from a 70% baseline
• Saved 10+ hours per week previously spent on false failure investigations
• Restored team confidence in test results with reliable feedback
• Faster debugging with comprehensive trace artifacts on failures