mirror of
https://github.com/hanzoai/tasks.git
synced 2026-10-04 02:04:03 +00:00
Sean KaneandGitHub
cd6a8c39c7
## What changed? Added a Bayesian commit bisect analysis to the `flakereport` tool that automatically identifies which commits most likely introduced test flakiness. Inspired by this Python tool: https://github.com/hauntsaninja/git_bayesect Sample run of the report can be found [here](https://github.com/temporalio/temporal/actions/runs/23914956307). This change uses the set of test runs which are mapped to commit history to infer with Bayesian statistics the probability that a commit introduced a flaky test or tests. There is a fuller explanation in `doc.go`. A tl;dr: The algorithm ranks each commit (as a transition) in the history by assigning equal p_before and p_after and using a Beta-Binomial marginal likelihood to determine the posterior probability. There is a heuristic to lower the prior probability for commits that only touch documentation or ci configuration. ## Why? This can help us identify the actual commit that introduced a problem and also help us get commit authors to validate their work instead of engineers without all of the context for a change. ## How did you test it? - [ ] built - [X] run locally and tested manually - [ ] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s) ## Potential risks Minimal, reporting only
41 lines
1016 B
Go
41 lines
1016 B
Go
package flakereport
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os/exec"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// commitOrder returns commit SHAs between oldSHA (exclusive) and HEAD (inclusive),
|
|
// in chronological order (oldest first).
|
|
// Shells out to: git log --reverse --format=%H <oldSHA>..HEAD
|
|
func commitOrder(ctx context.Context, oldSHA string) ([]string, error) {
|
|
ctxTimeout, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancel()
|
|
|
|
cmd := exec.CommandContext(ctxTimeout, "git", "log",
|
|
"--reverse",
|
|
"--format=%H",
|
|
fmt.Sprintf("%s..HEAD", oldSHA),
|
|
)
|
|
|
|
output, err := cmd.Output()
|
|
if err != nil {
|
|
if exitErr, ok := err.(*exec.ExitError); ok {
|
|
return nil, fmt.Errorf("git log failed: %w\nstderr: %s", err, string(exitErr.Stderr))
|
|
}
|
|
return nil, fmt.Errorf("failed to run git log: %w", err)
|
|
}
|
|
|
|
var commits []string
|
|
for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line != "" {
|
|
commits = append(commits, line)
|
|
}
|
|
}
|
|
return commits, nil
|
|
}
|