Files
tasks/tools/flakereport/report.go
Stephan BehnkeandGitHub 40be0f40c8 Fix Crash and CI Breakers reporting (#9478)
## What changed?

Fixed Crash and CI Breakers reporting in flakereport.

## Why?

Misleading aggregation.

## How did you test it?

```
go run ./cmd/tools/flakereport generate --days 7 --output-dir out
```

<details><summary>GitHub (abridged)</summary>
<p>

### Crashes

| Test | Crash Rate | Last Failure | Links |
|------|------------|-------------|-------|
| `unit-test` | 4.0% (1/25) | 3h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22966733319)
[2](https://github.com/temporalio/temporal/actions/runs/22923165266) |
| `functional-test` | 3.5% (35/1003) | 1h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22970016748/job/66684243853)
[2](https://github.com/temporalio/temporal/actions/runs/22970016748/job/66684243820)
[3](https://github.com/temporalio/temporal/actions/runs/22969997600/job/66684218138)
|


### CI Breakers (Failed All Retries)

| Test | CI Break Rate | Last Failure | Links |
|------|------------|-------------|-------|
|
`TestNexusStateReplicationTestSuite/EnableTransitionHistory/TestNexusCallbackReplicated`
| **50.0% (13/26)** | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674252859)
[2](https://github.com/temporalio/temporal/actions/runs/22931602170/job/66576965326)
[3](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593587)
|
|
`TestFuncClustersTestSuite/DisableTransitionHistory/TestForceMigration_ClosedWorkflow`
| **30.8% (8/26)** | 1h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22923524988/job/66528148344)
[2](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396416)
[3](https://github.com/temporalio/temporal/actions/runs/22929738731/job/66548562937)
|
|
`TestFuncClustersTestSuite/DisableTransitionHistory/TestForceMigration_ResetWorkflow`
| **19.2% (5/26)** | 19h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22928356811/job/66544280229)
[2](https://github.com/temporalio/temporal/actions/runs/22923524988/job/66528148344)
[3](https://github.com/temporalio/temporal/actions/runs/22929738731/job/66548562947)
|
|
`TestFuncClustersTestSuite/EnableTransitionHistory/TestForceMigration_ClosedWorkflow`
| **19.2% (5/26)** | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674252859)
[2](https://github.com/temporalio/temporal/actions/runs/22931602170/job/66576965326)
[3](https://github.com/temporalio/temporal/actions/runs/22929738731/job/66548563003)
|
|
`TestFuncClustersTestSuite/EnableTransitionHistory/TestForceMigration_ResetWorkflow`
| **15.4% (4/26)** | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22928356811/job/66544280299)
[2](https://github.com/temporalio/temporal/actions/runs/22941944027/job/66585445737)
[3](https://github.com/temporalio/temporal/actions/runs/22926291741/job/66537603391)
|
|
`TestDeploymentVersionSuiteV2/TestReactivationSignalCache_Deduplication_Reset`
| **11.5% (3/26)** | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717)
[2](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454)
[3](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006)
|
| `TestScheduleFunctionalSuite/TestListSchedulesReturnsWorkflowStatus` |
**7.7% (2/26)** | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717)
[2](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006)
|
| `TestWorkerDeploymentSuiteV2/TestSetCurrentVersion_Batching` | **7.7%
(2/26)** | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717)
[2](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006)
|
| `TestClientDataConverterTestSuite` | **7.7% (2/26)** | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600)
[2](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006)
|
|
`TestVersioning3FunctionalSuiteV0/TestTransitionDuringTransientTask_WithSignal`
| **7.7% (2/26)** | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22967653083/job/66676176237)
[2](https://github.com/temporalio/temporal/actions/runs/22967653083/job/66676176283)
|
|
`TestWorkerDeploymentSuiteV2/TestResourceExhaustedErrors_Converted_To_ReadableMessage`
| **7.7% (2/26)** | 19h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454)
[2](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600)
|
|
`TestDeploymentVersionSuiteV0/TestVersionMissingTaskQueues_ValidSetCurrentVersion`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestVersioning3FunctionalSuiteV2/TestPinnedCaN_UpgradeOnCaN_NormalWFT_PinnedOverride_WithSuggest/ForceTaskForwardNoPollForwardAllowSync`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestVersioning3FunctionalSuiteV0/TestPinnedCaN_UpgradeOnCaN_NormalWFT_WithSuggest/ForceTaskForwardNoPollForwardForceAsync`
| 3.8% (1/26) | 21h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717)
|
| `TestDeploymentVersionSuiteV2/TestVersionScavenger_DeleteOnAdd` | 3.8%
(1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestVersioning3FunctionalSuiteV2/TestTransitionDuringTransientTask_WithSignal`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
| `TestVersioning3FunctionalSuiteV2/TestVersionedQueueUnload` | 3.8%
(1/26) | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22967653083/job/66676176237)
|
| `TestScheduleFunctionalSuite/TestNextTimeCache` | 3.8% (1/26) | 19h
ago |
[1](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454)
|
|
`TestVersioningFunctionalSuite/TestDispatchActivityFail/NoTaskForwardForcePollForwardForceAsync`
| 3.8% (1/26) | 23h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600)
|
| `TestPrioritySuite/TestSubqueue_Migration` | 3.8% (1/26) | 21h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717)
|
|
`TestAdvancedVisibilitySuite/TestWorkerTaskReachability_Unversioned_InTaskQueue`
| 3.8% (1/26) | 21h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717)
|
| `TestDeploymentVersionSuiteV2/TestDeleteVersion_DrainingVersion` |
3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestDeploymentVersionSuiteV0/TestStartWorkflowExecution_ReactivateVersionOnPinned_WithConflictPolicy`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestTaskQueueStats_Classic_Suite/TestCurrentAbsorbsUnversionedBacklog_WhenRampingToUnversioned/ForceTaskForwardForcePollForwardAllowSync`
| 3.8% (1/26) | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22967653083/job/66676176237)
|
|
`TestWorkflowUpdateSuite/SpeculativeWorkflowTask_QueryFailureClearsWFContext`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestTaskQueueStats_Classic_Suite/TestMultipleTasks_WithMatchingBehavior_ValidateStats/ForceTaskForwardForcePollForwardForceAsync`
| 3.8% (1/26) | 19h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454)
|
|
`TestTaskQueueStats_Pri_Suite/TestRampingAndCurrentAbsorbUnversionedBacklog/ForceTaskForwardNoPollForwardForceAsync`
| 3.8% (1/26) | 19h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454)
|
|
`TestVersioningFunctionalSuite/TestDispatchActivityTimeoutCrossTq/NoTaskForwardForcePollForwardAllowSync`
| 3.8% (1/26) | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006)
|
| `TestWorkerDeploymentSuiteV0/TestDrainRollbackedVersion` | 3.8% (1/26)
| 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006)
|
|
`TestWorkerDeploymentSuiteV0/TestSetWorkerDeploymentRampingVersion_SetRampingAsCurrent`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestStandaloneActivityTestSuite/TestHeartbeat/ActivityRetriesOnHeartbeatTimeout`
| 3.8% (1/26) | 23h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600)
|
|
`TestTaskQueueStats_Classic_Suite/TestMultipleTasks_WithMatchingBehavior_ValidateStats/NoTaskForwardForcePollForwardForceAsync`
| 3.8% (1/26) | 23h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600)
|
|
`TestTaskQueueStats_Pri_Suite/TestMultipleTasks_WithMatchingBehavior_ValidateStats/ForceTaskForwardForcePollForwardAllowSync`
| 3.8% (1/26) | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22967653083/job/66676176237)
|
|
`TestWorkerDeploymentSuiteV0/TestSetRampingVersion_Concurrent_DifferentVersions_NoUnexpectedErrors`
| 3.8% (1/26) | 19h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454)
|
|
`TestDeploymentVersionSuiteV0/TestVersionMissingTaskQueues_ValidSetRampingVersion`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestDeploymentVersionSuiteV2/TestVersionIgnoresDrainageSignalWhenCurrentOrRamping`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestVersioning3FunctionalSuiteV0/TestPinnedCaN_UpgradeOnCaN_TransientWFT_WithSuggest/ForceTaskForwardForcePollForwardAllowSync`
| 3.8% (1/26) | 19h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454)
|
|
`TestStandaloneActivityTestSuite/TestDescribeActivityExecution_DeadlineExceeded/NoCallerDeadline`
| 3.8% (1/26) | 1h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22970016748/job/66684243820)
|
|
`TestAdvancedVisibilitySuiteLegacy/TestUpsertWorkflowExecutionSearchAttributes`
| 3.8% (1/26) | 21h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717)
|
|
`TestVersioning3FunctionalSuiteV2/TestPinnedCaN_UpgradeOnCaN_NormalWFT_NoSuggest/ForceTaskForwardForcePollForwardForceAsync`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestDeploymentVersionSuiteV0/TestVersionIgnoresDrainageSignalWhenCurrentOrRamping`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestVersioning3FunctionalSuiteV2/TestAutoUpgradeCaN_UpgradeOnCaN/ForceTaskForwardForcePollForwardAllowSync`
| 3.8% (1/26) | 21h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717)
|
|
`TestDeploymentVersionSuiteV0/TestUpdateWorkflowExecutionOptions_SetImpliedPinnedSuccess`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestDeploymentVersionSuiteV0/TestReactivationSignalCache_Deduplication_StartWorkflow`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestDeploymentVersionSuiteV2/TestVersionMissingTaskQueues_ValidSetRampingVersion`
| 3.8% (1/26) | 28h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605)
|
|
`TestDeploymentVersionSuiteV0/TestReactivationSignalCache_Deduplication_UpdateOptions`
| 3.8% (1/26) | 122h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22775553239/job/66068056068)
|
|
`TestNexusApiTestSuiteWithTemporalFailures/TestNexusCancelOperation_Outcomes/handler_timeout/ByNamespaceAndTaskQueue`
| 3.8% (1/26) | 23h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600)
|
|
`TestDeploymentVersionSuiteV0/TestStartWorkflowExecution_ReactivateVersionOnPinned`
| 3.8% (1/26) | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006)
|
| `TestCronTestSuite/TestCronWorkflow` | 3.8% (1/26) | 2h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22967653083/job/66676176283)
|
| `TestWorkerDeploymentSuiteV0/TestDeploymentVersionLimits` | 3.8%
(1/26) | 23h ago |
[1](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600)
|

</p>
</details> 

<details><summary>Slack</summary>
<p>

Flaky Tests Report - Last 7 Days

*CI Success Rate:* 9/26 (34.62%)
*Total Test Runs:* 1134261
*Total Failures:* 3774
*Failure Rate:* 3.33 per 1000 tests

*CI Breakers:* 50
*Crashes:* 2
*Flaky Tests:* 416
*Timeouts:* 0

*CI Breakers (top 10):*
• 50.0% (13/26 runs):
`TestNexusStateReplicationTestSuite/EnableTransitionHistory/TestNexusCallbackReplicated`
• 30.8% (8/26 runs):
`TestFuncClustersTestSuite/DisableTransitionHistory/TestForceMigration_ClosedWorkflow`
• 19.2% (5/26 runs):
`TestFuncClustersTestSuite/DisableTransitionHistory/TestForceMigration_ResetWorkflow`
• 19.2% (5/26 runs):
`TestFuncClustersTestSuite/EnableTransitionHistory/TestForceMigration_ClosedWorkflow`
• 15.4% (4/26 runs):
`TestFuncClustersTestSuite/EnableTransitionHistory/TestForceMigration_ResetWorkflow`
• 11.5% (3/26 runs):
`TestDeploymentVersionSuiteV2/TestReactivationSignalCache_Deduplication_Reset`
• 7.7% (2/26 runs):
`TestWorkerDeploymentSuiteV2/TestResourceExhaustedErrors_Converted_To_ReadableMessage`
• 7.7% (2/26 runs): `TestClientDataConverterTestSuite`
• 7.7% (2/26 runs):
`TestWorkerDeploymentSuiteV2/TestSetCurrentVersion_Batching`
• 7.7% (2/26 runs):
`TestVersioning3FunctionalSuiteV0/TestTransitionDuringTransientTask_WithSignal`

*Flaky Tests (top 10):*
• 100.0% (4 failures): `DATA RACE: Data race detected`
• 100.0% (24 failures): `TestClientDataConverterTestSuite`
• 100.0% (4 failures): `PANIC: Fail in goroutine after
TestWorkerDeploymentSuiteV2/TestConcurrentPollers_DifferentTaskQueues_SameVersion_SetCurrentVersion
has completed — in
TestDeploymentVersionSuiteV2/TestBatchUpdateWorkflowExecutionOptions_SetPinnedThenUnset`
• 100.0% (36 failures): `PANIC: Fail in goroutine after
TestWorkerDeploymentSuiteV2/TestConcurrentPollers_DifferentTaskQueues_SameVersion_SetCurrentVersion
has completed — in
TestWorkerDeploymentSuiteV2/TestConcurrentPollers_DifferentTaskQueues_SameVersion_SetCurrentVersion`
• 100.0% (4 failures): `DATA RACE: Data race detected — in
go.uber.org/zap/zaptest.TestingWriter.Write`
• 49.3% (187 failures):
`TestVersioning3FunctionalSuiteV2/TestTransitionDuringTransientTask_WithoutSignal`
• 48.9% (183 failures):
`TestVersioning3FunctionalSuiteV2/TestTransitionDuringTransientTask_WithSignal`
• 47.5% (161 failures):
`TestVersioning3FunctionalSuiteV0/TestTransitionDuringTransientTask_WithSignal`
• 47.4% (172 failures):
`TestVersioning3FunctionalSuiteV0/TestTransitionDuringTransientTask_WithoutSignal`
• 27.2% (71 failures):
`TestFuncClustersTestSuite/DisableTransitionHistory/TestForceMigration_ClosedWorkflow`


</p>
</details>
2026-03-11 22:24:47 +00:00

103 lines
2.8 KiB
Go

package flakereport
import (
"fmt"
"math"
"strings"
"time"
)
const boldFlakeRateThreshold = 5.0
// hoursAgo formats a timestamp as "Xh ago" relative to now.
func hoursAgo(t time.Time) string {
h := math.Round(time.Since(t).Hours())
if h < 1 {
h = 1
}
return fmt.Sprintf("%dh ago", int(h))
}
// formatReportLines returns a plain-text bullet line per report.
func formatReportLines(reports []TestReport) []string {
var lines []string
for _, r := range reports {
pct := 0.0
if r.TotalRuns > 0 {
pct = float64(r.FailureCount) / float64(r.TotalRuns) * 100.0
}
lines = append(lines, fmt.Sprintf("• %.1f%% (%d/%d): `%s`",
pct, r.FailureCount, r.TotalRuns, r.TestName))
}
return lines
}
// formatLinks formats GitHub URLs as numbered markdown links
func formatLinks(urls []string, maxLinks int) string {
linkCount := len(urls)
if linkCount > maxLinks {
linkCount = maxLinks
}
var parts []string
for i := 0; i < linkCount; i++ {
parts = append(parts, fmt.Sprintf("[%d](%s)", i+1, urls[i]))
}
return strings.Join(parts, " ")
}
// generateSuiteBreakdownTable creates a markdown table of per-suite flake data
func generateSuiteBreakdownTable(suiteReports []SuiteReport) string {
if len(suiteReports) == 0 {
return ""
}
var sb strings.Builder
sb.WriteString("| Suite | Flake Rate | Last Failure |\n")
sb.WriteString("|-------|------------|-------------|\n")
for _, sr := range suiteReports {
lastFailure := "-"
if sr.FailedRuns > 0 && !sr.LastFailure.IsZero() {
lastFailure = hoursAgo(sr.LastFailure)
}
rate := fmt.Sprintf("%.1f%% (%d/%d)", sr.FlakeRate, sr.FailedRuns, sr.TotalRuns)
if sr.FlakeRate > boldFlakeRateThreshold {
rate = "**" + rate + "**"
}
sb.WriteString(fmt.Sprintf("| `%s` | %s | %s |\n", sr.SuiteName, rate, lastFailure))
}
return sb.String()
}
// generateTestReportTable creates a markdown table of test reports with rate column.
func generateTestReportTable(reports []TestReport, rateHeader string, maxLinks int) string {
if len(reports) == 0 {
return ""
}
var sb strings.Builder
sb.WriteString(fmt.Sprintf("| Test | %s | Last Failure | Links |\n", rateHeader))
sb.WriteString("|------|------------|-------------|-------|\n")
for _, report := range reports {
pct := 0.0
if report.TotalRuns > 0 {
pct = float64(report.FailureCount) / float64(report.TotalRuns) * 100.0
}
links := formatLinks(report.GitHubURLs, maxLinks)
lastFailure := "N/A"
if !report.LastFailure.IsZero() {
lastFailure = hoursAgo(report.LastFailure)
}
rate := fmt.Sprintf("%.1f%% (%d/%d)", pct, report.FailureCount, report.TotalRuns)
if pct > boldFlakeRateThreshold {
rate = "**" + rate + "**"
}
sb.WriteString(fmt.Sprintf("| `%s` | %s | %s | %s |\n",
report.TestName, rate, lastFailure, links))
}
return sb.String()
}