Add bayesian git bisection to identify problematic commits (#9788)

## What changed?
Added a Bayesian commit bisect analysis to the `flakereport` tool that
automatically identifies which commits most likely introduced test
flakiness. Inspired by this Python tool:
https://github.com/hauntsaninja/git_bayesect

Sample run of the report can be found
[here](https://github.com/temporalio/temporal/actions/runs/23914956307).

This change uses the set of test runs which are mapped to commit history
to infer with Bayesian statistics the probability that a commit
introduced a flaky test or tests. There is a fuller explanation in
`doc.go`. A tl;dr: The algorithm ranks each commit (as a transition) in
the history by assigning equal p_before and p_after and using a
Beta-Binomial marginal likelihood to determine the posterior
probability. There is a heuristic to lower the prior probability for
commits that only touch documentation or ci configuration.

## Why?
This can help us identify the actual commit that introduced a problem
and also help us get commit authors to validate their work instead of
engineers without all of the context for a change.

## How did you test it?
- [ ] built
- [X] run locally and tested manually
- [ ] covered by existing tests
- [ ] added new unit test(s)
- [ ] added new functional test(s)

## Potential risks
Minimal, reporting only

.github/CODEOWNERS

@@ -92,3 +92,5 @@
 /service/worker/replicator/ @temporalio/cgs
 
 /service/worker/parentclosepolicy/ @temporalio/oss-foundations @temporalio/act
+
+/tools/ @temporalio/server

.github/workflows/flaky-tests-report.yml

@@ -16,6 +16,11 @@ on:
         required: false
         default: "3"
         type: string
+      notify_slack:
+        description: "Send Slack notification"
+        required: false
+        default: true
+        type: boolean
 
 permissions:
   contents: read
@@ -49,7 +54,7 @@ jobs:
         id: process-flaky-tests
         env:
           GH_TOKEN: ${{ steps.generate_token.outputs.token }}
-          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          SLACK_WEBHOOK: ${{ (github.event_name == 'schedule' || github.event.inputs.notify_slack == 'true') && secrets.SLACK_WEBHOOK || '' }}
           DAYS_PARAM: ${{ github.event.inputs.days || '7' }}
           MAX_LINKS_PARAM: ${{ github.event.inputs.max_links || '3' }}
           RUN_ID: ${{ github.run_id }}
@@ -65,7 +70,9 @@ jobs:
             --slack-webhook "$SLACK_WEBHOOK" \
             --run-id "$RUN_ID" \
             --ref-name "$REF_NAME" \
-            --sha "$SHA"
+            --sha "$SHA" \
+            --bisect \
+            --bisect-days 28
 
       - name: Upload generated reports
         uses: actions/upload-artifact@v6

tools/flakereport/bisect.go

@@ -0,0 +1,483 @@
+package flakereport
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	minBisectFailures = 5  // skip test if fewer than this many failures in window
+	minBisectRuns     = 30 // skip test if ran fewer than this many times total
+)
+
+// logBeta returns log(Beta(a, b)) = lgamma(a) + lgamma(b) - lgamma(a+b).
+func logBeta(a, b float64) float64 {
+	la, _ := math.Lgamma(a)
+	lb, _ := math.Lgamma(b)
+	lab, _ := math.Lgamma(a + b)
+	return la + lb - lab
+}
+
+// logBetaBinomial returns the log marginal likelihood of observing k failures
+// in n trials with a Beta(alpha, beta) prior on the failure probability.
+// With Beta(1,1) prior this is the log Beta-Binomial marginal likelihood.
+func logBetaBinomial(n, k int, alpha, beta float64) float64 {
+	return logBeta(float64(k)+alpha, float64(n-k)+beta) - logBeta(alpha, beta)
+}
+
+// logSumExpNormalize converts log-weights to probabilities using the log-sum-exp trick
+// to avoid numerical underflow. Returns a slice of probabilities summing to 1.0.
+func logSumExpNormalize(logWeights []float64) []float64 {
+	if len(logWeights) == 0 {
+		return nil
+	}
+	// Find max for numerical stability
+	maxLog := logWeights[0]
+	for _, lw := range logWeights[1:] {
+		if lw > maxLog {
+			maxLog = lw
+		}
+	}
+	// Compute exp(lw - maxLog) and sum
+	probs := make([]float64, len(logWeights))
+	sum := 0.0
+	for i, lw := range logWeights {
+		probs[i] = math.Exp(lw - maxLog)
+		sum += probs[i]
+	}
+	// Normalize
+	for i := range probs {
+		probs[i] /= sum
+	}
+	return probs
+}
+
+// buildObservations groups TestRun records for a single (normalized) test name by commit SHA
+// and returns them sorted chronologically using the commit order slice.
+// runToSHA maps workflow RunID → commit SHA.
+// commitOrder lists SHAs from oldest to newest.
+// Runs whose SHA is not in commitOrder are skipped (e.g. from force-pushes or unrelated branches).
+func buildObservations(testName string, runs []TestRun, commitOrderSlice []string, runToSHA map[int64]string) []CommitObservation {
+	// Build commit index map: SHA → index
+	commitIdx := make(map[string]int, len(commitOrderSlice))
+	for i, sha := range commitOrderSlice {
+		commitIdx[sha] = i
+	}
+
+	// Aggregate pass/fail counts per commit SHA for this test
+	type counts struct{ passes, fails int }
+	bySHA := make(map[string]*counts)
+
+	for _, run := range runs {
+		if run.Skipped {
+			continue
+		}
+		if normalizeTestName(run.Name) != testName {
+			continue
+		}
+		sha, ok := runToSHA[run.RunID]
+		if !ok || sha == "" {
+			continue
+		}
+		if _, inOrder := commitIdx[sha]; !inOrder {
+			continue
+		}
+		if bySHA[sha] == nil {
+			bySHA[sha] = &counts{}
+		}
+		if run.Failed {
+			bySHA[sha].fails++
+		} else {
+			bySHA[sha].passes++
+		}
+	}
+
+	// Convert to slice and sort by commit index
+	obs := make([]CommitObservation, 0, len(bySHA))
+	for sha, c := range bySHA {
+		obs = append(obs, CommitObservation{
+			CommitSHA: sha,
+			CommitIdx: commitIdx[sha],
+			Prior:     1.0, // uniform prior; adjusted by heuristics if enabled
+			Passes:    c.passes,
+			Fails:     c.fails,
+		})
+	}
+	sort.Slice(obs, func(i, j int) bool {
+		return obs[i].CommitIdx < obs[j].CommitIdx
+	})
+	return obs
+}
+
+// runBisect computes posterior probability for each candidate culprit commit.
+// obs must be sorted by CommitIdx ascending (chronological).
+// The model: there is one transition commit c; before c failure prob is p_before,
+// at/after c failure prob is p_after. Both have Beta(1,1) priors.
+// Returns results sorted by Probability descending.
+// Returns nil if there are fewer than 2 commits with observations (can't localize).
+func runBisect(obs []CommitObservation) []BisectResult {
+	if len(obs) < 2 {
+		return nil
+	}
+
+	// Prefix sums of fails and passes in commit order
+	N := len(obs)
+	prefixFails := make([]int, N+1)
+	prefixPasses := make([]int, N+1)
+	for i, o := range obs {
+		prefixFails[i+1] = prefixFails[i] + o.Fails
+		prefixPasses[i+1] = prefixPasses[i] + o.Passes
+	}
+	totalFails := prefixFails[N]
+	totalPasses := prefixPasses[N]
+	totalObs := totalFails + totalPasses
+
+	if totalObs == 0 {
+		return nil
+	}
+
+	// Compute log-posterior for each candidate culprit commit
+	logWeights := make([]float64, N)
+	for i, o := range obs {
+		kBefore := prefixFails[i]
+		nBefore := prefixPasses[i] + prefixFails[i]
+		kAfter := totalFails - kBefore
+		nAfter := totalObs - nBefore
+
+		logWeights[i] = math.Log(o.Prior) +
+			logBetaBinomial(nBefore, kBefore, 1, 1) +
+			logBetaBinomial(nAfter, kAfter, 1, 1)
+	}
+
+	probs := logSumExpNormalize(logWeights)
+
+	results := make([]BisectResult, N)
+	for i, o := range obs {
+		results[i] = BisectResult{
+			CommitSHA:     o.CommitSHA,
+			CommitIdx:     o.CommitIdx,
+			Probability:   probs[i],
+			FailsBefore:   prefixFails[i],
+			PassesBefore:  prefixPasses[i],
+			FailsAfter:    totalFails - prefixFails[i],
+			PassesAfter:   totalPasses - prefixPasses[i],
+			CommitTitle:   o.CommitSHA, // placeholder; overwritten if heuristics fetch metadata
+			HeuristicNote: "",
+		}
+	}
+
+	sort.Slice(results, func(i, j int) bool {
+		return results[i].Probability > results[j].Probability
+	})
+	return results
+}
+
+// commitPriorWeight returns a prior weight multiplier [0.0, 2.0] for a commit,
+// based on its changed files relative to the test name. For now this only reduces
+// the prior for commits that only touch test files or docs/config files.
+// Weight < 1.0 makes the commit less likely to be the culprit.
+// Weight > 1.0 makes it more likely.
+func commitPriorWeight(meta CommitMeta, testName string) (weight float64, note string) {
+	files := meta.Files
+	if len(files) == 0 {
+		return 1.0, ""
+	}
+
+	// Commits only touching non-code paths cannot affect test behavior
+	if allFilesMatchPrefixes(files, []string{".github/", "docs/", "CODEOWNERS"}) ||
+		allFilesMatchSuffixes(files, []string{".md", ".txt", ".yaml", ".yml"}) {
+		return 0.05, "only touches docs/.github/config — deprioritized"
+	}
+
+	// Commits only touching test files for other packages → mild deprioritization
+	if allFilesMatchSuffixes(files, []string{"_test.go"}) {
+		return 0.3, "only touches test files"
+	}
+
+	return 1.0, ""
+}
+
+// allFilesMatchPrefixes returns true if every file in the list starts with one of the given prefixes.
+func allFilesMatchPrefixes(files, prefixes []string) bool {
+	for _, f := range files {
+		matched := false
+		for _, p := range prefixes {
+			if strings.HasPrefix(f, p) {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			return false
+		}
+	}
+	return true
+}
+
+// allFilesMatchSuffixes returns true if every file in the list ends with one of the given suffixes.
+func allFilesMatchSuffixes(files, suffixes []string) bool {
+	for _, f := range files {
+		matched := false
+		for _, s := range suffixes {
+			if strings.HasSuffix(f, s) {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			return false
+		}
+	}
+	return true
+}
+
+// runBisectForTest runs the full bisect pipeline for a single test name.
+// commitMetas is a pre-populated, read-only cache of commit metadata (SHA → CommitMeta).
+func runBisectForTest(cfg BisectConfig, testName string, allRuns []TestRun, commitOrderSlice []string, runToSHA map[int64]string, commitMetas map[string]CommitMeta) TestBisectReport {
+	obs := buildObservations(testName, allRuns, commitOrderSlice, runToSHA)
+
+	totalPasses, totalFails := 0, 0
+	for _, o := range obs {
+		totalPasses += o.Passes
+		totalFails += o.Fails
+	}
+	totalObs := totalPasses + totalFails
+
+	// Check signal thresholds
+	if totalFails < cfg.MinFailures {
+		return TestBisectReport{
+			TestName: testName,
+			Skipped:  true,
+		}
+	}
+	if totalObs < cfg.MinRuns {
+		return TestBisectReport{
+			TestName: testName,
+			Skipped:  true,
+		}
+	}
+
+	// Apply heuristics from the pre-fetched commit metadata cache.
+	for i := range obs {
+		if meta, ok := commitMetas[obs[i].CommitSHA]; ok {
+			obs[i].Prior, obs[i].HeuristicNote = commitPriorWeight(meta, testName)
+		}
+	}
+
+	results := runBisect(obs)
+	if results == nil {
+		return TestBisectReport{
+			TestName: testName,
+			Skipped:  true,
+		}
+	}
+
+	// Build a SHA → note lookup from the obs slice (already computed above).
+	obsNotes := make(map[string]string, len(obs))
+	for _, o := range obs {
+		obsNotes[o.CommitSHA] = o.HeuristicNote
+	}
+
+	// Annotate results with commit metadata
+	for i := range results {
+		results[i].HeuristicNote = obsNotes[results[i].CommitSHA]
+		if meta, ok := commitMetas[results[i].CommitSHA]; ok {
+			results[i].CommitTitle = meta.Title
+			results[i].CommitAuthor = meta.Author
+			if !meta.CommittedAt.IsZero() {
+				results[i].CommitDate = meta.CommittedAt.Format("2006-01-02")
+			}
+		}
+	}
+
+	// Filter suspects below the minimum probability threshold.
+	if cfg.MinProbability > 0 {
+		filtered := results[:0]
+		for _, r := range results {
+			if r.Probability >= cfg.MinProbability {
+				filtered = append(filtered, r)
+			}
+		}
+		results = filtered
+	}
+
+	if len(results) == 0 {
+		return TestBisectReport{
+			TestName: testName,
+			TotalObs: totalObs,
+			Skipped:  true,
+		}
+	}
+
+	return TestBisectReport{
+		TestName:    testName,
+		TopSuspects: results,
+		TotalObs:    totalObs,
+	}
+}
+
+// selectTopFlakyTests selects the top-N flakiest tests that meet minimum signal thresholds.
+// allRuns is the full set of test runs. Returns normalized test names sorted by failure rate desc.
+func selectTopFlakyTests(allRuns []TestRun, cfg BisectConfig) []string {
+	type testStats struct {
+		fails int
+		total int
+	}
+	stats := make(map[string]*testStats)
+	for _, run := range allRuns {
+		if run.Skipped {
+			continue
+		}
+		name := normalizeTestName(run.Name)
+		if stats[name] == nil {
+			stats[name] = &testStats{}
+		}
+		stats[name].total++
+		if run.Failed {
+			stats[name].fails++
+		}
+	}
+
+	type ranked struct {
+		name  string
+		rate  float64
+		fails int
+	}
+	var candidates []ranked
+	for name, s := range stats {
+		if s.fails < cfg.MinFailures || s.total < cfg.MinRuns {
+			continue
+		}
+		candidates = append(candidates, ranked{
+			name:  name,
+			rate:  float64(s.fails) / float64(s.total),
+			fails: s.fails,
+		})
+	}
+
+	sort.Slice(candidates, func(i, j int) bool {
+		if candidates[i].rate != candidates[j].rate {
+			return candidates[i].rate > candidates[j].rate
+		}
+		return candidates[i].fails > candidates[j].fails
+	})
+
+	n := len(candidates)
+	if cfg.TopN > 0 && cfg.TopN < n {
+		n = cfg.TopN
+	}
+
+	names := make([]string, n)
+	for i := range names {
+		names[i] = candidates[i].name
+	}
+	return names
+}
+
+// runBisectAnalysis is the top-level bisect orchestrator called from the generate command.
+// It runs bisect for the top-N flakiest tests and returns their reports.
+func runBisectAnalysis(ctx context.Context, cfg BisectConfig, allRuns []TestRun, workflowRuns []WorkflowRun) ([]TestBisectReport, error) {
+	// Build runID → SHA map from workflow runs
+	runToSHA := make(map[int64]string, len(workflowRuns))
+	var oldestSHA string
+	var oldestTime time.Time
+	for _, wr := range workflowRuns {
+		if wr.HeadSHA == "" {
+			continue
+		}
+		runToSHA[wr.ID] = wr.HeadSHA
+		if oldestTime.IsZero() || wr.CreatedAt.Before(oldestTime) {
+			oldestTime = wr.CreatedAt
+			oldestSHA = wr.HeadSHA
+		}
+	}
+
+	if oldestSHA == "" {
+		return nil, errors.New("no workflow runs with commit SHAs found")
+	}
+
+	// Get commit ordering from git log
+	fmt.Printf("Building commit order from %s..HEAD\n", oldestSHA[:7])
+	commitOrderSlice, err := commitOrder(ctx, oldestSHA)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get commit order: %w", err)
+	}
+	if len(commitOrderSlice) == 0 {
+		return nil, fmt.Errorf("commit order is empty for anchor %s: SHA may not be an ancestor of HEAD (force-push or unrelated branch?)", oldestSHA[:7])
+	}
+	fmt.Printf("Commit range: %d commits\n", len(commitOrderSlice))
+
+	// Select qualifying flaky tests
+	targetTests := selectTopFlakyTests(allRuns, cfg)
+	if cfg.TopN > 0 {
+		fmt.Printf("Selected %d tests for bisect analysis (top %d, min %d failures, min %d runs)\n",
+			len(targetTests), cfg.TopN, cfg.MinFailures, cfg.MinRuns)
+	} else {
+		fmt.Printf("Selected %d tests for bisect analysis (all qualifying, min %d failures, min %d runs)\n",
+			len(targetTests), cfg.MinFailures, cfg.MinRuns)
+	}
+
+	// Pre-fetch commit metadata for all unique SHAs that appear in the run set.
+	// Fetching once here (deduplicated, in parallel) avoids O(tests × commits) serial
+	// subprocess calls — the dominant cost of the bisect pipeline.
+	uniqueSHAs := make([]string, 0, len(runToSHA))
+	seenSHA := make(map[string]struct{}, len(runToSHA))
+	for _, sha := range runToSHA {
+		if _, ok := seenSHA[sha]; !ok {
+			seenSHA[sha] = struct{}{}
+			uniqueSHAs = append(uniqueSHAs, sha)
+		}
+	}
+	commitMetas := fetchCommitMetasParallel(ctx, cfg.Repo, uniqueSHAs, defaultConcurrency)
+	fmt.Printf("Fetched metadata for %d/%d unique commits\n", len(commitMetas), len(uniqueSHAs))
+
+	// Run bisect for each test
+	reports := make([]TestBisectReport, 0, len(targetTests))
+	for _, testName := range targetTests {
+		start := time.Now()
+		report := runBisectForTest(cfg, testName, allRuns, commitOrderSlice, runToSHA, commitMetas)
+		fmt.Printf("  Bisected %s in %s (skipped=%v)\n", testName, time.Since(start).Round(time.Millisecond), report.Skipped)
+		reports = append(reports, report)
+	}
+
+	return reports, nil
+}
+
+// fetchCommitMetasParallel fetches commit metadata for a list of SHAs concurrently,
+// bounded by maxConcurrency. Returns an immutable map of SHA → CommitMeta; failed
+// fetches are logged and omitted (heuristic priors fall back to 1.0).
+func fetchCommitMetasParallel(ctx context.Context, repo string, shas []string, maxConcurrency int) map[string]CommitMeta {
+	metas := make([]CommitMeta, len(shas))
+	ok := make([]bool, len(shas))
+
+	g, ctx := errgroup.WithContext(ctx)
+	g.SetLimit(maxConcurrency)
+	for i, sha := range shas {
+		g.Go(func() error {
+			meta, err := fetchCommitMeta(ctx, repo, sha)
+			if err != nil {
+				fmt.Printf("Warning: failed to fetch metadata for commit %s: %v\n", sha[:7], err)
+				return nil
+			}
+			metas[i] = meta
+			ok[i] = true
+			return nil
+		})
+	}
+	_ = g.Wait()
+
+	commitMetas := make(map[string]CommitMeta, len(shas))
+	for i, sha := range shas {
+		if ok[i] {
+			commitMetas[sha] = metas[i]
+		}
+	}
+	return commitMetas
+}

tools/flakereport/bisect_test.go

@@ -0,0 +1,492 @@
+package flakereport
+
+import (
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLogBetaBinomial(t *testing.T) {
+	t.Run("symmetry: k=0 equals k=n", func(t *testing.T) {
+		// logBetaBinomial(n, 0) == logBetaBinomial(n, n) by symmetry of Beta(1,1)
+		v0 := logBetaBinomial(10, 0, 1, 1)
+		v10 := logBetaBinomial(10, 10, 1, 1)
+		assert.InDelta(t, v0, v10, 1e-10, "logBetaBinomial(10,0,1,1) should equal logBetaBinomial(10,10,1,1)")
+	})
+
+	t.Run("extreme k is more likely than middle k under Beta(1,1)", func(t *testing.T) {
+		// Under a Beta(1,1) (uniform) prior, the marginal likelihood favors extreme k.
+		// Observing all failures (k=n=10) or all passes (k=0) is MORE likely than k=5,
+		// because the posterior can concentrate near p=1 or p=0 to explain the data.
+		vMid := logBetaBinomial(10, 5, 1, 1)
+		v0 := logBetaBinomial(10, 0, 1, 1)
+		assert.Greater(t, v0, vMid, "logBetaBinomial(10,0,1,1) should exceed logBetaBinomial(10,5,1,1) under uniform prior")
+	})
+
+	t.Run("n=0 k=0 is valid", func(t *testing.T) {
+		// No observations: should return a finite value (0 with Beta-Binomial normalization)
+		v := logBetaBinomial(0, 0, 1, 1)
+		assert.False(t, math.IsNaN(v), "logBetaBinomial(0,0,1,1) should not be NaN")
+		assert.False(t, math.IsInf(v, 0), "logBetaBinomial(0,0,1,1) should not be Inf")
+	})
+
+	t.Run("single failure in 1 trial", func(t *testing.T) {
+		v := logBetaBinomial(1, 1, 1, 1)
+		assert.False(t, math.IsNaN(v), "logBetaBinomial(1,1,1,1) should not be NaN")
+		assert.False(t, math.IsInf(v, 0), "logBetaBinomial(1,1,1,1) should not be Inf")
+	})
+}
+
+func TestLogSumExpNormalize(t *testing.T) {
+	t.Run("empty input returns nil", func(t *testing.T) {
+		result := logSumExpNormalize(nil)
+		assert.Nil(t, result)
+	})
+
+	t.Run("single element returns 1.0", func(t *testing.T) {
+		result := logSumExpNormalize([]float64{-5.0})
+		require.Len(t, result, 1)
+		assert.InDelta(t, 1.0, result[0], 1e-10)
+	})
+
+	t.Run("uniform log-weights produce equal probabilities", func(t *testing.T) {
+		logWeights := []float64{-3.0, -3.0, -3.0, -3.0}
+		result := logSumExpNormalize(logWeights)
+		require.Len(t, result, 4)
+		for _, p := range result {
+			assert.InDelta(t, 0.25, p, 1e-10)
+		}
+	})
+
+	t.Run("probabilities sum to 1.0", func(t *testing.T) {
+		logWeights := []float64{-10.0, -5.0, -1.0, -3.0, -7.0}
+		result := logSumExpNormalize(logWeights)
+		require.Len(t, result, 5)
+		sum := 0.0
+		for _, p := range result {
+			sum += p
+		}
+		assert.InDelta(t, 1.0, sum, 1e-10)
+	})
+
+	t.Run("highest log-weight gets highest probability", func(t *testing.T) {
+		logWeights := []float64{-10.0, -5.0, -1.0, -3.0}
+		result := logSumExpNormalize(logWeights)
+		require.Len(t, result, 4)
+		// index 2 has the highest log-weight (-1.0)
+		maxIdx := 0
+		for i, p := range result {
+			if p > result[maxIdx] {
+				maxIdx = i
+			}
+		}
+		assert.Equal(t, 2, maxIdx)
+	})
+
+	t.Run("handles very large negative values without underflow", func(t *testing.T) {
+		logWeights := []float64{-1000.0, -999.0, -998.0}
+		result := logSumExpNormalize(logWeights)
+		require.Len(t, result, 3)
+		sum := 0.0
+		for _, p := range result {
+			assert.False(t, math.IsNaN(p), "probability should not be NaN")
+			assert.False(t, math.IsInf(p, 0), "probability should not be Inf")
+			assert.GreaterOrEqual(t, p, 0.0, "probability should be non-negative")
+			sum += p
+		}
+		assert.InDelta(t, 1.0, sum, 1e-10)
+	})
+}
+
+func TestRunBisect(t *testing.T) {
+	t.Run("returns nil with fewer than 2 observations", func(t *testing.T) {
+		obs := []CommitObservation{
+			{CommitSHA: "abc", CommitIdx: 0, Prior: 1.0, Passes: 10, Fails: 2},
+		}
+		result := runBisect(obs)
+		assert.Nil(t, result)
+	})
+
+	t.Run("returns nil with zero total observations", func(t *testing.T) {
+		obs := []CommitObservation{
+			{CommitSHA: "abc", CommitIdx: 0, Prior: 1.0, Passes: 0, Fails: 0},
+			{CommitSHA: "def", CommitIdx: 1, Prior: 1.0, Passes: 0, Fails: 0},
+		}
+		result := runBisect(obs)
+		assert.Nil(t, result)
+	})
+
+	t.Run("sharp signal: transition at commit index 3", func(t *testing.T) {
+		// First 3 commits: 0 fails out of 30 each
+		// Last 3 commits: 10 fails out of 30 each (1/3 failure rate)
+		// The 4th commit (index 3) should have the highest posterior probability
+		obs := []CommitObservation{
+			{CommitSHA: "sha0", CommitIdx: 0, Prior: 1.0, Passes: 30, Fails: 0},
+			{CommitSHA: "sha1", CommitIdx: 1, Prior: 1.0, Passes: 30, Fails: 0},
+			{CommitSHA: "sha2", CommitIdx: 2, Prior: 1.0, Passes: 30, Fails: 0},
+			{CommitSHA: "sha3", CommitIdx: 3, Prior: 1.0, Passes: 20, Fails: 10},
+			{CommitSHA: "sha4", CommitIdx: 4, Prior: 1.0, Passes: 20, Fails: 10},
+			{CommitSHA: "sha5", CommitIdx: 5, Prior: 1.0, Passes: 20, Fails: 10},
+		}
+		results := runBisect(obs)
+		require.NotNil(t, results)
+		require.Len(t, results, 6)
+
+		// Probabilities should sum to 1.0
+		sum := 0.0
+		for _, r := range results {
+			sum += r.Probability
+		}
+		assert.InDelta(t, 1.0, sum, 1e-10)
+
+		// The top result should be sha3 (index 3) — the transition point
+		require.Equal(t, "sha3", results[0].CommitSHA, "sha3 should be the top suspect")
+		require.Greater(t, results[0].Probability, 0.5, "top suspect should have >50% probability")
+
+		// Results should be sorted by probability descending
+		for i := 1; i < len(results); i++ {
+			assert.LessOrEqual(t, results[i].Probability, results[i-1].Probability,
+				"results should be sorted by probability descending")
+		}
+	})
+
+	t.Run("counts are correctly propagated", func(t *testing.T) {
+		obs := []CommitObservation{
+			{CommitSHA: "sha0", CommitIdx: 0, Prior: 1.0, Passes: 10, Fails: 0},
+			{CommitSHA: "sha1", CommitIdx: 1, Prior: 1.0, Passes: 5, Fails: 5},
+		}
+		results := runBisect(obs)
+		require.NotNil(t, results)
+
+		// Find sha1 result (should be the top suspect since all failures are after it)
+		var sha1Result *BisectResult
+		for i := range results {
+			if results[i].CommitSHA == "sha1" {
+				sha1Result = &results[i]
+				break
+			}
+		}
+		require.NotNil(t, sha1Result)
+		// sha1 is the transition: before sha1 = 10 passes, 0 fails; after/at sha1 = 5 passes, 5 fails
+		assert.Equal(t, 10, sha1Result.PassesBefore)
+		assert.Equal(t, 0, sha1Result.FailsBefore)
+		assert.Equal(t, 5, sha1Result.PassesAfter)
+		assert.Equal(t, 5, sha1Result.FailsAfter)
+	})
+
+	t.Run("two commits uniform signal returns valid probabilities", func(t *testing.T) {
+		obs := []CommitObservation{
+			{CommitSHA: "sha0", CommitIdx: 0, Prior: 1.0, Passes: 5, Fails: 5},
+			{CommitSHA: "sha1", CommitIdx: 1, Prior: 1.0, Passes: 5, Fails: 5},
+		}
+		results := runBisect(obs)
+		require.NotNil(t, results)
+		require.Len(t, results, 2)
+		sum := results[0].Probability + results[1].Probability
+		assert.InDelta(t, 1.0, sum, 1e-10)
+	})
+}
+
+func TestRunBisectInformationlessData(t *testing.T) {
+	t.Run("zero observations across many commits returns nil", func(t *testing.T) {
+		// With no observations at any commit, there is no data to localize a transition.
+		obs := []CommitObservation{
+			{CommitSHA: "sha0", CommitIdx: 0, Prior: 1.0},
+			{CommitSHA: "sha1", CommitIdx: 1, Prior: 1.0},
+			{CommitSHA: "sha2", CommitIdx: 2, Prior: 1.0},
+			{CommitSHA: "sha3", CommitIdx: 3, Prior: 1.0},
+			{CommitSHA: "sha4", CommitIdx: 4, Prior: 1.0},
+		}
+		result := runBisect(obs)
+		assert.Nil(t, result)
+	})
+
+	t.Run("balanced data (equal failures and successes per commit) yields symmetric posterior", func(t *testing.T) {
+		// Each commit has the same number of failures and successes (50% failure rate
+		// throughout), so there is no directional signal about where a transition occurred.
+		//
+		// The Beta-Binomial model does NOT produce posterior == prior in this case: placing
+		// all observations on one side of the transition is more parsimonious (one marginal
+		// distribution instead of two), so the model assigns slightly higher likelihood to
+		// extreme transition points. However, the posterior IS symmetric around the midpoint:
+		// prob[i] == prob[N-1-i] when priors are uniform and per-commit observations are
+		// identical.
+		obs := []CommitObservation{
+			{CommitSHA: "sha0", CommitIdx: 0, Prior: 1.0, Passes: 5, Fails: 5},
+			{CommitSHA: "sha1", CommitIdx: 1, Prior: 1.0, Passes: 5, Fails: 5},
+			{CommitSHA: "sha2", CommitIdx: 2, Prior: 1.0, Passes: 5, Fails: 5},
+			{CommitSHA: "sha3", CommitIdx: 3, Prior: 1.0, Passes: 5, Fails: 5},
+			{CommitSHA: "sha4", CommitIdx: 4, Prior: 1.0, Passes: 5, Fails: 5},
+		}
+		results := runBisect(obs)
+		require.NotNil(t, results)
+		require.Len(t, results, 5)
+
+		// Probabilities must be non-negative and sum to 1.0.
+		sum := 0.0
+		for _, r := range results {
+			assert.GreaterOrEqual(t, r.Probability, 0.0)
+			sum += r.Probability
+		}
+		assert.InDelta(t, 1.0, sum, 1e-10)
+
+		// With uniform priors and identical per-commit observations the distribution has
+		// a specific symmetry: the weight for transition at index i depends only on the
+		// total observations before/after the split. Because the "before" segment for
+		// index i mirrors the "after" segment for index N-1-i+1, the pairs (sha1, sha4)
+		// and (sha2, sha3) must have equal probability. sha0 is unpaired: its "before"
+		// segment is empty while no index has an empty "after" segment.
+		//
+		// Note: sha0 will have a higher probability than the others because placing all
+		// observations on one side of the transition is more parsimonious for the
+		// Beta-Binomial model than splitting them between two equal-rate distributions.
+		// The balanced data does not produce a diffuse uniform posterior.
+		probBySHA := make(map[string]float64, len(results))
+		for _, r := range results {
+			probBySHA[r.CommitSHA] = r.Probability
+		}
+		// No commit should dominate with balanced data (>50% would imply a spurious signal).
+		for sha, p := range probBySHA {
+			assert.Less(t, p, 0.5, "no single commit should dominate with balanced data (sha=%s)", sha)
+		}
+		// Paired commits must have equal probability.
+		assert.InDelta(t, probBySHA["sha1"], probBySHA["sha4"], 1e-10,
+			"symmetric pair: sha1 and sha4 should have equal probability")
+		assert.InDelta(t, probBySHA["sha2"], probBySHA["sha3"], 1e-10,
+			"symmetric pair: sha2 and sha3 should have equal probability")
+	})
+}
+
+func TestBuildObservations(t *testing.T) {
+	commitOrderSlice := []string{"sha-a", "sha-b", "sha-c", "sha-d"}
+	runToSHA := map[int64]string{
+		100: "sha-a",
+		200: "sha-b",
+		300: "sha-c",
+		400: "sha-d",
+		500: "sha-unknown", // not in commitOrder
+	}
+
+	runs := []TestRun{
+		// sha-a: 3 passes, 1 fail for "TestFoo"
+		{Name: "TestFoo", Failed: false, RunID: 100},
+		{Name: "TestFoo", Failed: false, RunID: 100},
+		{Name: "TestFoo", Failed: false, RunID: 100},
+		{Name: "TestFoo", Failed: true, RunID: 100},
+		// sha-b: 2 passes for "TestFoo"
+		{Name: "TestFoo", Failed: false, RunID: 200},
+		{Name: "TestFoo", Failed: false, RunID: 200},
+		// sha-c: 1 pass, 2 fails for "TestFoo"
+		{Name: "TestFoo", Failed: false, RunID: 300},
+		{Name: "TestFoo", Failed: true, RunID: 300},
+		{Name: "TestFoo", Failed: true, RunID: 300},
+		// sha-d: 1 fail for "TestFoo" with retry suffix (should normalize to "TestFoo")
+		{Name: "TestFoo (retry 1)", Failed: true, RunID: 400},
+		// sha-unknown: should be skipped (not in commitOrder)
+		{Name: "TestFoo", Failed: false, RunID: 500},
+		// different test name: should be excluded
+		{Name: "TestBar", Failed: true, RunID: 100},
+		// skipped run: should be excluded
+		{Name: "TestFoo", Failed: false, Skipped: true, RunID: 100},
+		// no SHA mapping: should be excluded
+		{Name: "TestFoo", Failed: false, RunID: 999},
+	}
+
+	obs := buildObservations("TestFoo", runs, commitOrderSlice, runToSHA)
+
+	require.Len(t, obs, 4, "should have one observation per commit with data")
+
+	// Verify chronological ordering
+	for i := 1; i < len(obs); i++ {
+		assert.Less(t, obs[i-1].CommitIdx, obs[i].CommitIdx,
+			"observations should be sorted by commit index")
+	}
+
+	// Find each commit's observation
+	byShA := make(map[string]CommitObservation)
+	for _, o := range obs {
+		byShA[o.CommitSHA] = o
+	}
+
+	// sha-a: 3 passes, 1 fail
+	require.Contains(t, byShA, "sha-a")
+	assert.Equal(t, 3, byShA["sha-a"].Passes)
+	assert.Equal(t, 1, byShA["sha-a"].Fails)
+	assert.Equal(t, 0, byShA["sha-a"].CommitIdx)
+
+	// sha-b: 2 passes, 0 fails
+	require.Contains(t, byShA, "sha-b")
+	assert.Equal(t, 2, byShA["sha-b"].Passes)
+	assert.Equal(t, 0, byShA["sha-b"].Fails)
+	assert.Equal(t, 1, byShA["sha-b"].CommitIdx)
+
+	// sha-c: 1 pass, 2 fails
+	require.Contains(t, byShA, "sha-c")
+	assert.Equal(t, 1, byShA["sha-c"].Passes)
+	assert.Equal(t, 2, byShA["sha-c"].Fails)
+	assert.Equal(t, 2, byShA["sha-c"].CommitIdx)
+
+	// sha-d: 0 passes, 1 fail (the retry suffix is normalized to "TestFoo")
+	require.Contains(t, byShA, "sha-d")
+	assert.Equal(t, 0, byShA["sha-d"].Passes)
+	assert.Equal(t, 1, byShA["sha-d"].Fails)
+	assert.Equal(t, 3, byShA["sha-d"].CommitIdx)
+
+	// sha-unknown should not appear
+	assert.NotContains(t, byShA, "sha-unknown")
+
+	// Prior should default to 1.0
+	for _, o := range obs {
+		assert.InDelta(t, 1.0, o.Prior, 1e-10)
+	}
+}
+
+func TestSelectTopFlakyTests(t *testing.T) {
+	t.Run("excludes tests below MinFailures threshold", func(t *testing.T) {
+		runs := makeRunsForTest("TestLowFails", 100, 2) // 2 failures: below minBisectFailures=5
+		cfg := BisectConfig{TopN: 10, MinFailures: 5, MinRuns: 10}
+		result := selectTopFlakyTests(runs, cfg)
+		assert.NotContains(t, result, "TestLowFails")
+	})
+
+	t.Run("excludes tests below MinRuns threshold", func(t *testing.T) {
+		runs := makeRunsForTest("TestFewRuns", 10, 6) // only 10 total runs: below minBisectRuns=30
+		cfg := BisectConfig{TopN: 10, MinFailures: 5, MinRuns: 30}
+		result := selectTopFlakyTests(runs, cfg)
+		assert.NotContains(t, result, "TestFewRuns")
+	})
+
+	t.Run("ranks by failure rate descending", func(t *testing.T) {
+		// TestHighRate: 20/40 = 50% failure rate
+		// TestLowRate: 10/40 = 25% failure rate
+		var runs []TestRun
+		runs = append(runs, makeRunsForTest("TestHighRate", 40, 20)...)
+		runs = append(runs, makeRunsForTest("TestLowRate", 40, 10)...)
+		cfg := BisectConfig{TopN: 10, MinFailures: 5, MinRuns: 30}
+		result := selectTopFlakyTests(runs, cfg)
+		require.GreaterOrEqual(t, len(result), 2)
+		assert.Equal(t, "TestHighRate", result[0])
+		assert.Equal(t, "TestLowRate", result[1])
+	})
+
+	t.Run("limits to TopN results", func(t *testing.T) {
+		var runs []TestRun
+		for i := range 5 {
+			runs = append(runs, makeRunsForTest("TestFlaky"+string(rune('A'+i)), 40, 10)...)
+		}
+		cfg := BisectConfig{TopN: 3, MinFailures: 5, MinRuns: 30}
+		result := selectTopFlakyTests(runs, cfg)
+		assert.Len(t, result, 3)
+	})
+
+	t.Run("skips skipped runs", func(t *testing.T) {
+		runs := []TestRun{
+			{Name: "TestSkipped", Skipped: true, Failed: false},
+			{Name: "TestSkipped", Skipped: true, Failed: true},
+		}
+		cfg := BisectConfig{TopN: 10, MinFailures: 1, MinRuns: 1}
+		result := selectTopFlakyTests(runs, cfg)
+		assert.NotContains(t, result, "TestSkipped")
+	})
+
+	t.Run("normalizes test names", func(t *testing.T) {
+		// Retry suffix should be stripped; all runs collapse to one test
+		var runs []TestRun
+		for range 30 {
+			runs = append(runs, TestRun{Name: "TestFlaky", Failed: false})
+		}
+		for range 8 {
+			runs = append(runs, TestRun{Name: "TestFlaky (retry 1)", Failed: true})
+		}
+		cfg := BisectConfig{TopN: 10, MinFailures: 5, MinRuns: 30}
+		result := selectTopFlakyTests(runs, cfg)
+		require.Len(t, result, 1)
+		assert.Equal(t, "TestFlaky", result[0])
+	})
+}
+
+func TestCommitPriorWeight(t *testing.T) {
+	t.Run("docs-only commit is deprioritized", func(t *testing.T) {
+		meta := CommitMeta{
+			SHA:   "abc",
+			Files: []string{".github/workflows/ci.yml", "docs/architecture.md"},
+		}
+		weight, note := commitPriorWeight(meta, "TestWorkflowSuite/TestRun")
+		assert.InDelta(t, 0.05, weight, 1e-10)
+		assert.NotEmpty(t, note)
+	})
+
+	t.Run("markdown-only commit is deprioritized", func(t *testing.T) {
+		meta := CommitMeta{
+			SHA:   "abc",
+			Files: []string{"README.md", "CHANGELOG.md"},
+		}
+		weight, note := commitPriorWeight(meta, "TestWorkflowSuite/TestRun")
+		assert.InDelta(t, 0.05, weight, 1e-10)
+		assert.NotEmpty(t, note)
+	})
+
+	t.Run("service/history/ commit is neutral weight", func(t *testing.T) {
+		meta := CommitMeta{
+			SHA:   "abc",
+			Files: []string{"service/history/workflow_executor.go", "service/history/mutable_state.go"},
+		}
+		weight, note := commitPriorWeight(meta, "TestWorkflowSuite/TestContinueAsNew")
+		assert.InDelta(t, 1.0, weight, 1e-10)
+		assert.Empty(t, note)
+	})
+
+	t.Run("test-only commit is mildly deprioritized", func(t *testing.T) {
+		meta := CommitMeta{
+			SHA:   "abc",
+			Files: []string{"service/frontend/nexus_handler_test.go", "service/worker/deployment_test.go"},
+		}
+		weight, note := commitPriorWeight(meta, "TestWorkflowSuite/TestRun")
+		assert.InDelta(t, 0.3, weight, 1e-10)
+		assert.NotEmpty(t, note)
+	})
+
+	t.Run("mixed code and doc files is neutral", func(t *testing.T) {
+		meta := CommitMeta{
+			SHA:   "abc",
+			Files: []string{"service/frontend/handler.go", "README.md"},
+		}
+		weight, note := commitPriorWeight(meta, "TestWorkflowSuite/TestRun")
+		// Not all files are docs/config (handler.go is not), not all are test files
+		// and "service/history/" is not a prefix of service/frontend/
+		assert.InDelta(t, 1.0, weight, 1e-10)
+		assert.Empty(t, note)
+	})
+
+	t.Run("empty files list returns neutral weight", func(t *testing.T) {
+		meta := CommitMeta{SHA: "abc", Files: nil}
+		weight, note := commitPriorWeight(meta, "TestWorkflowSuite/TestRun")
+		assert.InDelta(t, 1.0, weight, 1e-10)
+		assert.Empty(t, note)
+	})
+
+	t.Run("service/frontend/ commit is neutral weight", func(t *testing.T) {
+		meta := CommitMeta{
+			SHA:   "abc",
+			Files: []string{"service/frontend/nexus_handler.go"},
+		}
+		weight, note := commitPriorWeight(meta, "TestNexusSuite/TestOperation")
+		assert.InDelta(t, 1.0, weight, 1e-10)
+		assert.Empty(t, note)
+	})
+}
+
+// makeRunsForTest creates a slice of TestRun with the given total count and failure count.
+// The first `failures` runs are marked as failed, the rest as passed.
+func makeRunsForTest(name string, total, failures int) []TestRun {
+	runs := make([]TestRun, total)
+	for i := range runs {
+		runs[i] = TestRun{
+			Name:   name,
+			Failed: i < failures,
+		}
+	}
+	return runs
+}

tools/flakereport/doc.go

@@ -0,0 +1,76 @@
+// Package flakereport implements Bayesian commit bisection for identifying which commit
+// most likely introduced a flaky test in a CI system.
+//
+// # Problem
+//
+// A flaky test is one whose failure probability changes at some point in the commit
+// history — typically because a code change introduced a race condition, timing
+// sensitivity, or environmental dependency. Given a rolling window of CI runs (each
+// associated with the HEAD commit at the time), we want to rank candidate commits by
+// their posterior probability of being the "transition commit" that caused the flakiness.
+//
+// # Data
+//
+// The raw input is a set of test runs downloaded from GitHub Actions artifacts. Each run
+// records whether a specific test passed or failed, and is tagged with the workflow RunID.
+// RunIDs are mapped to commit SHAs via the GitHub Actions API (WorkflowRun.HeadSHA).
+// Runs are then grouped into CommitObservation records: for each (test, commit SHA) pair
+// we count the number of passing and failing runs.
+//
+// # Model
+//
+// The inference model assumes exactly one transition commit c in the history:
+//
+//   - Before commit c: the test has a background failure probability p_before.
+//   - At commit c and all later commits: the test has an elevated failure probability p_after.
+//
+// Neither p_before nor p_after is known, so both are assigned independent uniform (Beta(1,1))
+// priors. The model treats them as nuisance parameters and marginalizes them out, yielding
+// the Beta-Binomial marginal likelihood:
+//
+//	P(data | transition at c) = BetaBinomial(n_before, k_before; 1, 1)
+//	                           × BetaBinomial(n_after,  k_after;  1, 1)
+//
+// where n_before / k_before are the total runs / failures before commit c, and n_after /
+// k_after are the total runs / failures at and after commit c. The closed form is:
+//
+//	BetaBinomial(n, k; α, β) = Beta(k+α, n-k+β) / Beta(α, β)
+//
+// All arithmetic is performed in log-space to avoid floating-point underflow, with the
+// log-sum-exp trick applied during normalization.
+//
+// # Commit Priors
+//
+// The prior probability that a given commit is the transition commit is not uniform across
+// all commits. Commits that only touch documentation, CI configuration, or test files
+// cannot plausibly affect production code behaviour, so they receive a reduced prior weight
+// (down to 0.05×). Commits that touch source code receive the default weight of 1.0.
+// These heuristic weights are fetched from the GitHub API in parallel and applied before
+// running the inference.
+//
+// # Inference Algorithm
+//
+// For N commits with observations, the algorithm runs in O(N) time using prefix sums:
+//
+//  1. Compute prefix sums of failures and passes across the commit sequence.
+//  2. For each candidate transition commit i, use the prefix sums to split the data into
+//     "before" and "after" segments and evaluate the log marginal likelihood.
+//  3. Add the log prior weight to each log-likelihood.
+//  4. Normalize via log-sum-exp to obtain posterior probabilities that sum to 1.
+//
+// # Output
+//
+// Commits are ranked by posterior probability. Only commits above a configurable minimum
+// probability threshold (default 0.50) are reported. Results are surfaced in the GitHub
+// Actions step summary, a markdown report artifact, and optionally a Slack message
+// listing the "hottest" commits (those with the highest aggregate posterior probability
+// across all analyzed tests).
+//
+// # Limitations
+//
+// The model assumes a single transition in the observed window. If multiple commits each
+// contributed independently to flakiness, or if the failure rate oscillates, the model
+// may identify the wrong commit or produce a diffuse posterior with no strong suspect.
+// The model also requires a minimum of 5 failures and 30 total runs before it will
+// attempt bisection on a test, to avoid over-fitting noisy data.
+package flakereport

tools/flakereport/flakereport.go

@@ -86,6 +86,25 @@ func NewCliApp() *cli.App {
 					Name:  "sha",
 					Usage: "Git commit SHA (for failure messages)",
 				},
+				&cli.BoolFlag{
+					Name:  "bisect",
+					Usage: "Run Bayesian commit bisect on all qualifying flaky tests after generating the report",
+				},
+				&cli.IntFlag{
+					Name:  "bisect-top-n",
+					Value: 0,
+					Usage: "Limit bisect to the N flakiest qualifying tests; 0 = all qualifying tests",
+				},
+				&cli.IntFlag{
+					Name:  "bisect-days",
+					Value: 28,
+					Usage: "Lookback window in days for bisect artifact data (independent of --days)",
+				},
+				&cli.Float64Flag{
+					Name:  "bisect-min-probability",
+					Value: 0.50,
+					Usage: "Only report tests where the top suspect commit has at least this probability (0–1)",
+				},
 			},
 			Action: runGenerateCommand,
 		},
@@ -94,10 +113,11 @@ func NewCliApp() *cli.App {
 	return app
 }
 
-// fetchAndAnalyzeWorkflowRuns fetches workflow runs and counts successes
-func fetchAndAnalyzeWorkflowRuns(ctx context.Context, repo string, workflowID int64, branch string, days int) ([]WorkflowRun, int, error) {
+// fetchAndAnalyzeWorkflowRuns fetches workflow runs between since and until and counts successes.
+// until is zero for an open-ended (up to now) window.
+func fetchAndAnalyzeWorkflowRuns(ctx context.Context, repo string, workflowID int64, branch string, since, until time.Time) ([]WorkflowRun, int, error) {
 	fmt.Println("\n=== Fetching workflow runs ===")
-	runs, err := fetchWorkflowRuns(ctx, repo, workflowID, branch, days)
+	runs, err := fetchWorkflowRuns(ctx, repo, workflowID, branch, since, until)
 	if err != nil {
 		return nil, 0, fmt.Errorf("failed to fetch workflow runs: %w", err)
 	}
@@ -202,7 +222,10 @@ func runGenerateCommand(c *cli.Context) (err error) {
 	runID := c.String("run-id")
 	refName := c.String("ref-name")
 	sha := c.String("sha")
-
+	runBisect := c.Bool("bisect")
+	bisectTopN := c.Int("bisect-top-n")
+	bisectDays := c.Int("bisect-days")
+	bisectMinProbability := c.Float64("bisect-min-probability")
 	// Send failure notification on error
 	defer func() {
 		if err != nil {
@@ -222,7 +245,9 @@ func runGenerateCommand(c *cli.Context) (err error) {
 	defer cancel()
 
 	// Fetch and analyze workflow runs
-	runs, successfulRuns, err := fetchAndAnalyzeWorkflowRuns(ctx, repo, workflowID, branch, days)
+	now := time.Now()
+	reportSince := now.AddDate(0, 0, -days)
+	runs, successfulRuns, err := fetchAndAnalyzeWorkflowRuns(ctx, repo, workflowID, branch, reportSince, time.Time{})
 	if err != nil {
 		return err
 	}
@@ -292,6 +317,36 @@ func runGenerateCommand(c *cli.Context) (err error) {
 		allFailures, allTestRuns, runs, successfulRuns,
 	)
 
+	// Optionally run Bayesian bisect analysis
+	var bisectReports []TestBisectReport
+	if runBisect {
+		fmt.Println("\n=== Running Bayesian bisect analysis ===")
+		// Extend the run set with the incremental window (days→bisectDays) to avoid
+		// re-fetching runs we already have from the report window.
+		bisectRuns := runs
+		if bisectDays > days {
+			bisectSince := now.AddDate(0, 0, -bisectDays)
+			fmt.Printf("Fetching incremental runs for bisect (days %d–%d)...\n", days, bisectDays)
+			extraRuns, _, extraErr := fetchAndAnalyzeWorkflowRuns(ctx, repo, workflowID, branch, bisectSince, reportSince)
+			if extraErr != nil {
+				fmt.Printf("Warning: Failed to fetch bisect runs: %v\n", extraErr)
+			} else {
+				bisectRuns = append(extraRuns, runs...)
+			}
+		}
+		bisectCfg := BisectConfig{
+			Repo:           repo,
+			TopN:           bisectTopN,
+			MinFailures:    minBisectFailures,
+			MinRuns:        minBisectRuns,
+			MinProbability: bisectMinProbability,
+		}
+		bisectReports, err = runBisectAnalysis(ctx, bisectCfg, allTestRuns, bisectRuns)
+		if err != nil {
+			fmt.Printf("Warning: Bisect analysis failed: %v\n", err)
+		}
+	}
+
 	fmt.Println("\n=== Writing report files ===")
 	if err = os.MkdirAll(outputDir, 0755); err != nil {
 		return fmt.Errorf("failed to create output directory: %w", err)
@@ -305,6 +360,9 @@ func runGenerateCommand(c *cli.Context) (err error) {
 	// Write GitHub summary (to GITHUB_STEP_SUMMARY if set, otherwise to output dir)
 	fmt.Println("\n=== Writing GitHub summary ===")
 	summaryContent := generateGitHubSummary(summary, runID, maxLinks)
+	if len(bisectReports) > 0 {
+		summaryContent += generateBisectSummary(bisectReports, repo, bisectMinProbability)
+	}
 	if err := writeGitHubSummary(summaryContent, outputDir); err != nil {
 		fmt.Printf("Warning: Failed to write GitHub summary: %v\n", err)
 	}

tools/flakereport/git.go

@@ -0,0 +1,40 @@
+package flakereport
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// commitOrder returns commit SHAs between oldSHA (exclusive) and HEAD (inclusive),
+// in chronological order (oldest first).
+// Shells out to: git log --reverse --format=%H <oldSHA>..HEAD
+func commitOrder(ctx context.Context, oldSHA string) ([]string, error) {
+	ctxTimeout, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctxTimeout, "git", "log",
+		"--reverse",
+		"--format=%H",
+		fmt.Sprintf("%s..HEAD", oldSHA),
+	)
+
+	output, err := cmd.Output()
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			return nil, fmt.Errorf("git log failed: %w\nstderr: %s", err, string(exitErr.Stderr))
+		}
+		return nil, fmt.Errorf("failed to run git log: %w", err)
+	}
+
+	var commits []string
+	for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") {
+		line = strings.TrimSpace(line)
+		if line != "" {
+			commits = append(commits, line)
+		}
+	}
+	return commits, nil
+}

tools/flakereport/github.go

@@ -13,21 +13,25 @@ import (
 	"time"
 )
 
-// fetchWorkflowRuns retrieves all completed workflow runs within date range
-// Implements proper pagination to fix the 100-run limit bug
-func fetchWorkflowRuns(ctx context.Context, repo string, workflowID int64, branch string, sinceDays int) ([]WorkflowRun, error) {
+// fetchWorkflowRuns retrieves all completed workflow runs within a date range.
+// since is the oldest bound (inclusive); until is the newest bound (zero means open-ended).
+// Implements proper pagination to fix the 100-run limit bug.
+func fetchWorkflowRuns(ctx context.Context, repo string, workflowID int64, branch string, since, until time.Time) ([]WorkflowRun, error) {
 	var allRuns []WorkflowRun
-	sinceDate := time.Now().AddDate(0, 0, -sinceDays).Format("2006-01-02")
 
-	fmt.Printf("Fetching workflow runs since %s...\n", sinceDate)
+	createdFilter := ">=" + since.Format("2006-01-02")
+	if !until.IsZero() {
+		createdFilter = since.Format("2006-01-02") + ".." + until.Format("2006-01-02")
+	}
+	fmt.Printf("Fetching workflow runs created %s...\n", createdFilter)
 
 	page := 1
 	for {
 		ctxTimeout, cancel := context.WithTimeout(ctx, 30*time.Second)
 
 		cmd := exec.CommandContext(ctxTimeout, "gh", "api",
-			fmt.Sprintf("/repos/%s/actions/workflows/%d/runs?branch=%s&created=>=%s&per_page=100&page=%d",
-				repo, workflowID, branch, sinceDate, page),
+			fmt.Sprintf("/repos/%s/actions/workflows/%d/runs?branch=%s&created=%s&per_page=100&page=%d",
+				repo, workflowID, branch, createdFilter, page),
 		)
 
 		output, err := cmd.Output()
@@ -230,3 +234,59 @@ func buildGitHubURL(repo, runID, jobID string) string {
 	}
 	return baseURL
 }
+
+// fetchCommitMeta fetches commit title, author, and changed file list for a single commit SHA.
+// Uses: GET /repos/{owner}/{repo}/commits/{sha}
+func fetchCommitMeta(ctx context.Context, repo, sha string) (CommitMeta, error) {
+	ctxTimeout, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctxTimeout, "gh", "api",
+		fmt.Sprintf("/repos/%s/commits/%s", repo, sha),
+	)
+
+	output, err := cmd.Output()
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			return CommitMeta{SHA: sha}, fmt.Errorf("gh api failed for commit %s: %w\nstderr: %s", sha, err, string(exitErr.Stderr))
+		}
+		return CommitMeta{SHA: sha}, fmt.Errorf("failed to execute gh command for commit %s: %w", sha, err)
+	}
+
+	var response struct {
+		SHA    string `json:"sha"`
+		Commit struct {
+			Message string `json:"message"`
+			Author  struct {
+				Name string    `json:"name"`
+				Date time.Time `json:"date"`
+			} `json:"author"`
+		} `json:"commit"`
+		Files []struct {
+			Filename string `json:"filename"`
+		} `json:"files"`
+	}
+
+	if err := json.Unmarshal(output, &response); err != nil {
+		return CommitMeta{SHA: sha}, fmt.Errorf("failed to parse commit response for %s: %w", sha, err)
+	}
+
+	// Extract just the first line of the commit message as the title
+	title := response.Commit.Message
+	if idx := strings.IndexByte(title, '\n'); idx >= 0 {
+		title = title[:idx]
+	}
+
+	files := make([]string, 0, len(response.Files))
+	for _, f := range response.Files {
+		files = append(files, f.Filename)
+	}
+
+	return CommitMeta{
+		SHA:         sha,
+		Title:       title,
+		Author:      response.Commit.Author.Name,
+		CommittedAt: response.Commit.Author.Date,
+		Files:       files,
+	}, nil
+}

tools/flakereport/slack.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"sort"
 	"strings"
 	"time"
 )
@@ -129,6 +130,83 @@ func buildSuccessMessage(summary *ReportSummary, runID, repo string, days int) *
 	return msg
 }
 
+// addBisectSection appends a Bayesian bisect section to an existing Slack message.
+// TODO seankane: after validating this methodology can identify problematic commits
+// add the section to the Slack message.
+func (msg *SlackMessage) addBisectSection(reports []TestBisectReport, repo string) {
+	// Count qualifying
+	qualifying := 0
+	for _, r := range reports {
+		if !r.Skipped {
+			qualifying++
+		}
+	}
+	if qualifying == 0 {
+		return
+	}
+
+	// Find hot commits (top suspect in 2+ tests)
+	type commitCount struct {
+		title string
+		count int
+	}
+	hotCommits := make(map[string]*commitCount)
+	for _, r := range reports {
+		if r.Skipped || len(r.TopSuspects) == 0 {
+			continue
+		}
+		top := r.TopSuspects[0]
+		if _, ok := hotCommits[top.CommitSHA]; !ok {
+			hotCommits[top.CommitSHA] = &commitCount{title: top.CommitTitle}
+		}
+		hotCommits[top.CommitSHA].count++
+	}
+
+	var lines []string
+	for sha, cc := range hotCommits {
+		if cc.count >= 2 {
+			shortSHA := sha
+			if len(shortSHA) > 7 {
+				shortSHA = shortSHA[:7]
+			}
+			commitURL := fmt.Sprintf("https://github.com/%s/commit/%s", repo, sha)
+			title := cc.title
+			if title == sha || title == "" {
+				title = shortSHA
+			}
+			lines = append(lines, fmt.Sprintf("• <%s|%s> — %d tests — %s", commitURL, shortSHA, cc.count, title))
+		}
+	}
+
+	if len(lines) == 0 {
+		// No multi-test suspects; just report the count
+		msg.Blocks = append(msg.Blocks, SlackBlock{
+			Type: "section",
+			Text: &SlackText{
+				Type: "mrkdwn",
+				Text: fmt.Sprintf("*🔍 Commit Suspects (Bayesian)*\n%d tests analyzed. See GitHub summary for details.", qualifying),
+			},
+		})
+		return
+	}
+
+	// Sort lines for deterministic output
+	sort.Strings(lines)
+	if len(lines) > slackMaxListItems {
+		lines = lines[:slackMaxListItems]
+	}
+
+	text := fmt.Sprintf("*🔍 Hot Commits (Bayesian, implicated in 2+ tests):*\n%s\n\nSee GitHub summary for full details.",
+		strings.Join(lines, "\n"))
+	msg.Blocks = append(msg.Blocks, SlackBlock{
+		Type: "section",
+		Text: &SlackText{
+			Type: "mrkdwn",
+			Text: truncateToSlackLimit(text, 2900),
+		},
+	})
+}
+
 // buildFailureMessage creates failure notification
 func buildFailureMessage(runID, refName, sha, repo string) *SlackMessage {
 	msg := &SlackMessage{

tools/flakereport/types.go

@@ -75,6 +75,7 @@ type WorkflowRun struct {
 	Status     string    `json:"status"`
 	Conclusion string    `json:"conclusion"`
 	HeadBranch string    `json:"head_branch"`
+	HeadSHA    string    `json:"head_sha"`
 }
 
 // WorkflowArtifact represents a downloadable artifact
@@ -90,3 +91,55 @@ type ArtifactsResponse struct {
 	TotalCount int                `json:"total_count"`
 	Artifacts  []WorkflowArtifact `json:"artifacts"`
 }
+
+// CommitObservation holds aggregated pass/fail data for a single (test, commit) pair.
+type CommitObservation struct {
+	CommitSHA     string
+	CommitIdx     int     // chronological index (0 = oldest)
+	Prior         float64 // prior weight (1.0 = uniform; adjusted by heuristics)
+	HeuristicNote string  // reason for prior adjustment, if any
+	Passes        int
+	Fails         int
+}
+
+// BisectResult is one candidate culprit commit with its posterior probability.
+type BisectResult struct {
+	CommitSHA     string
+	CommitIdx     int
+	Probability   float64 // posterior P(this commit introduced the flakiness)
+	PassesBefore  int
+	FailsBefore   int
+	PassesAfter   int
+	FailsAfter    int
+	CommitTitle   string
+	CommitAuthor  string
+	CommitDate    string // formatted date of the commit, e.g. "2024-01-15"
+	HeuristicNote string // e.g. "only touches .github/ — deprioritized"
+}
+
+// TestBisectReport is the full bisect output for a single test.
+type TestBisectReport struct {
+	TestName    string
+	TopSuspects []BisectResult // sorted by Probability descending
+	TotalObs    int            // total observations (pass + fail) used
+	Skipped     bool           // true if below signal or confidence threshold
+}
+
+// CommitMeta holds changed-file info fetched from the GitHub API.
+// GET /repos/{owner}/{repo}/commits/{sha}
+type CommitMeta struct {
+	SHA         string
+	Title       string
+	Author      string
+	CommittedAt time.Time
+	Files       []string // relative paths of changed files
+}
+
+// BisectConfig holds configuration for a bisect analysis run.
+type BisectConfig struct {
+	Repo           string
+	TopN           int // max tests to analyze; 0 = all qualifying tests
+	MinFailures    int
+	MinRuns        int
+	MinProbability float64 // only report tests whose top suspect exceeds this (0–1); 0 = report all
+}

tools/flakereport/writer.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -100,6 +101,89 @@ func generateGitHubSummary(summary *ReportSummary, runID string, maxLinks int) s
 	return content
 }
 
+// countQualifyingBisectReports returns the number of non-skipped reports.
+func countQualifyingBisectReports(reports []TestBisectReport) int {
+	n := 0
+	for _, r := range reports {
+		if !r.Skipped {
+			n++
+		}
+	}
+	return n
+}
+
+// escapeTableCell replaces pipe characters so they don't corrupt GFM table rows.
+func escapeTableCell(s string) string {
+	return strings.ReplaceAll(s, "|", "|")
+}
+
+// writeBisectTable writes all suspect (test, commit) pairs into a single flat table.
+func writeBisectTable(sb *strings.Builder, reports []TestBisectReport, repo string) {
+	sb.WriteString("| Test | Prob | Commit | Date | Author | Before | After | Note |\n")
+	sb.WriteString("|------|------|--------|------|--------|--------|-------|------|\n")
+	for _, r := range reports {
+		if r.Skipped || len(r.TopSuspects) == 0 {
+			continue
+		}
+		for _, s := range r.TopSuspects {
+			shortSHA := s.CommitSHA
+			if len(shortSHA) > 7 {
+				shortSHA = shortSHA[:7]
+			}
+			commitURL := fmt.Sprintf("https://github.com/%s/commit/%s", repo, s.CommitSHA)
+			title := s.CommitTitle
+			if title == s.CommitSHA || title == "" {
+				title = shortSHA
+			}
+			beforeStr := fmt.Sprintf("%d/%d (%.0f%%)", s.FailsBefore, s.PassesBefore+s.FailsBefore,
+				pct(s.FailsBefore, s.PassesBefore+s.FailsBefore))
+			afterStr := fmt.Sprintf("%d/%d (%.0f%%)", s.FailsAfter, s.PassesAfter+s.FailsAfter,
+				pct(s.FailsAfter, s.PassesAfter+s.FailsAfter))
+			fmt.Fprintf(sb, "| `%s` | %.1f%% | [%s](%s) %s | %s | %s | %s | %s | %s |\n",
+				escapeTableCell(r.TestName), s.Probability*100, shortSHA, commitURL, escapeTableCell(title),
+				s.CommitDate, escapeTableCell(s.CommitAuthor), beforeStr, afterStr, escapeTableCell(s.HeuristicNote))
+		}
+	}
+	sb.WriteString("\n")
+}
+
+// generateBisectSummary creates the markdown section for bisect results to append to the GitHub summary.
+func generateBisectSummary(reports []TestBisectReport, repo string, minProb float64) string {
+	qualifying := countQualifyingBisectReports(reports)
+
+	skipped := len(reports) - qualifying
+	threshold := fmt.Sprintf("%.0f%%", minProb*100)
+
+	var sb strings.Builder
+	sb.WriteString("\n## Bayesian Commit Suspects\n\n")
+
+	if qualifying == 0 {
+		sb.WriteString("No actionable commit suspects found")
+		if skipped > 0 {
+			sb.WriteString(fmt.Sprintf(" — %d tests analyzed but none above %s confidence", skipped, threshold))
+		}
+		sb.WriteString("\n")
+		return sb.String()
+	}
+
+	sb.WriteString(fmt.Sprintf("%d tests with actionable suspects (≥%s confidence)", qualifying, threshold))
+	if skipped > 0 {
+		sb.WriteString(fmt.Sprintf(", %d below confidence threshold", skipped))
+	}
+	sb.WriteString("\n\n")
+
+	writeBisectTable(&sb, reports, repo)
+	return sb.String()
+}
+
+// pct returns percentage of num/denom, returning 0 if denom is 0.
+func pct(num, denom int) float64 {
+	if denom == 0 {
+		return 0
+	}
+	return float64(num) / float64(denom) * 100.0
+}
+
 // writeGitHubSummary writes markdown summary to GITHUB_STEP_SUMMARY (if set)
 // and always writes to outputDir/github-report.md.
 func writeGitHubSummary(content string, outputDir string) error {