mirror of
https://github.com/hanzoai/tasks.git
synced 2026-11-03 03:56:04 +00:00
## What changed? Fixed Crash and CI Breakers reporting in flakereport. ## Why? Misleading aggregation. ## How did you test it? ``` go run ./cmd/tools/flakereport generate --days 7 --output-dir out ``` <details><summary>GitHub (abridged)</summary> <p> ### Crashes | Test | Crash Rate | Last Failure | Links | |------|------------|-------------|-------| | `unit-test` | 4.0% (1/25) | 3h ago | [1](https://github.com/temporalio/temporal/actions/runs/22966733319) [2](https://github.com/temporalio/temporal/actions/runs/22923165266) | | `functional-test` | 3.5% (35/1003) | 1h ago | [1](https://github.com/temporalio/temporal/actions/runs/22970016748/job/66684243853) [2](https://github.com/temporalio/temporal/actions/runs/22970016748/job/66684243820) [3](https://github.com/temporalio/temporal/actions/runs/22969997600/job/66684218138) | ### CI Breakers (Failed All Retries) | Test | CI Break Rate | Last Failure | Links | |------|------------|-------------|-------| | `TestNexusStateReplicationTestSuite/EnableTransitionHistory/TestNexusCallbackReplicated` | **50.0% (13/26)** | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674252859) [2](https://github.com/temporalio/temporal/actions/runs/22931602170/job/66576965326) [3](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593587) | | `TestFuncClustersTestSuite/DisableTransitionHistory/TestForceMigration_ClosedWorkflow` | **30.8% (8/26)** | 1h ago | [1](https://github.com/temporalio/temporal/actions/runs/22923524988/job/66528148344) [2](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396416) [3](https://github.com/temporalio/temporal/actions/runs/22929738731/job/66548562937) | | `TestFuncClustersTestSuite/DisableTransitionHistory/TestForceMigration_ResetWorkflow` | **19.2% (5/26)** | 19h ago | [1](https://github.com/temporalio/temporal/actions/runs/22928356811/job/66544280229) [2](https://github.com/temporalio/temporal/actions/runs/22923524988/job/66528148344) [3](https://github.com/temporalio/temporal/actions/runs/22929738731/job/66548562947) | | `TestFuncClustersTestSuite/EnableTransitionHistory/TestForceMigration_ClosedWorkflow` | **19.2% (5/26)** | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674252859) [2](https://github.com/temporalio/temporal/actions/runs/22931602170/job/66576965326) [3](https://github.com/temporalio/temporal/actions/runs/22929738731/job/66548563003) | | `TestFuncClustersTestSuite/EnableTransitionHistory/TestForceMigration_ResetWorkflow` | **15.4% (4/26)** | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22928356811/job/66544280299) [2](https://github.com/temporalio/temporal/actions/runs/22941944027/job/66585445737) [3](https://github.com/temporalio/temporal/actions/runs/22926291741/job/66537603391) | | `TestDeploymentVersionSuiteV2/TestReactivationSignalCache_Deduplication_Reset` | **11.5% (3/26)** | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717) [2](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454) [3](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006) | | `TestScheduleFunctionalSuite/TestListSchedulesReturnsWorkflowStatus` | **7.7% (2/26)** | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717) [2](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006) | | `TestWorkerDeploymentSuiteV2/TestSetCurrentVersion_Batching` | **7.7% (2/26)** | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717) [2](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006) | | `TestClientDataConverterTestSuite` | **7.7% (2/26)** | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600) [2](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006) | | `TestVersioning3FunctionalSuiteV0/TestTransitionDuringTransientTask_WithSignal` | **7.7% (2/26)** | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22967653083/job/66676176237) [2](https://github.com/temporalio/temporal/actions/runs/22967653083/job/66676176283) | | `TestWorkerDeploymentSuiteV2/TestResourceExhaustedErrors_Converted_To_ReadableMessage` | **7.7% (2/26)** | 19h ago | [1](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454) [2](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600) | | `TestDeploymentVersionSuiteV0/TestVersionMissingTaskQueues_ValidSetCurrentVersion` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestVersioning3FunctionalSuiteV2/TestPinnedCaN_UpgradeOnCaN_NormalWFT_PinnedOverride_WithSuggest/ForceTaskForwardNoPollForwardAllowSync` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestVersioning3FunctionalSuiteV0/TestPinnedCaN_UpgradeOnCaN_NormalWFT_WithSuggest/ForceTaskForwardNoPollForwardForceAsync` | 3.8% (1/26) | 21h ago | [1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717) | | `TestDeploymentVersionSuiteV2/TestVersionScavenger_DeleteOnAdd` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestVersioning3FunctionalSuiteV2/TestTransitionDuringTransientTask_WithSignal` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestVersioning3FunctionalSuiteV2/TestVersionedQueueUnload` | 3.8% (1/26) | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22967653083/job/66676176237) | | `TestScheduleFunctionalSuite/TestNextTimeCache` | 3.8% (1/26) | 19h ago | [1](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454) | | `TestVersioningFunctionalSuite/TestDispatchActivityFail/NoTaskForwardForcePollForwardForceAsync` | 3.8% (1/26) | 23h ago | [1](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600) | | `TestPrioritySuite/TestSubqueue_Migration` | 3.8% (1/26) | 21h ago | [1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717) | | `TestAdvancedVisibilitySuite/TestWorkerTaskReachability_Unversioned_InTaskQueue` | 3.8% (1/26) | 21h ago | [1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717) | | `TestDeploymentVersionSuiteV2/TestDeleteVersion_DrainingVersion` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestDeploymentVersionSuiteV0/TestStartWorkflowExecution_ReactivateVersionOnPinned_WithConflictPolicy` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestTaskQueueStats_Classic_Suite/TestCurrentAbsorbsUnversionedBacklog_WhenRampingToUnversioned/ForceTaskForwardForcePollForwardAllowSync` | 3.8% (1/26) | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22967653083/job/66676176237) | | `TestWorkflowUpdateSuite/SpeculativeWorkflowTask_QueryFailureClearsWFContext` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestTaskQueueStats_Classic_Suite/TestMultipleTasks_WithMatchingBehavior_ValidateStats/ForceTaskForwardForcePollForwardForceAsync` | 3.8% (1/26) | 19h ago | [1](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454) | | `TestTaskQueueStats_Pri_Suite/TestRampingAndCurrentAbsorbUnversionedBacklog/ForceTaskForwardNoPollForwardForceAsync` | 3.8% (1/26) | 19h ago | [1](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454) | | `TestVersioningFunctionalSuite/TestDispatchActivityTimeoutCrossTq/NoTaskForwardForcePollForwardAllowSync` | 3.8% (1/26) | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006) | | `TestWorkerDeploymentSuiteV0/TestDrainRollbackedVersion` | 3.8% (1/26) | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006) | | `TestWorkerDeploymentSuiteV0/TestSetWorkerDeploymentRampingVersion_SetRampingAsCurrent` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestStandaloneActivityTestSuite/TestHeartbeat/ActivityRetriesOnHeartbeatTimeout` | 3.8% (1/26) | 23h ago | [1](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600) | | `TestTaskQueueStats_Classic_Suite/TestMultipleTasks_WithMatchingBehavior_ValidateStats/NoTaskForwardForcePollForwardForceAsync` | 3.8% (1/26) | 23h ago | [1](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600) | | `TestTaskQueueStats_Pri_Suite/TestMultipleTasks_WithMatchingBehavior_ValidateStats/ForceTaskForwardForcePollForwardAllowSync` | 3.8% (1/26) | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22967653083/job/66676176237) | | `TestWorkerDeploymentSuiteV0/TestSetRampingVersion_Concurrent_DifferentVersions_NoUnexpectedErrors` | 3.8% (1/26) | 19h ago | [1](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454) | | `TestDeploymentVersionSuiteV0/TestVersionMissingTaskQueues_ValidSetRampingVersion` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestDeploymentVersionSuiteV2/TestVersionIgnoresDrainageSignalWhenCurrentOrRamping` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestVersioning3FunctionalSuiteV0/TestPinnedCaN_UpgradeOnCaN_TransientWFT_WithSuggest/ForceTaskForwardForcePollForwardAllowSync` | 3.8% (1/26) | 19h ago | [1](https://github.com/temporalio/temporal/actions/runs/22931281792/job/66553396454) | | `TestStandaloneActivityTestSuite/TestDescribeActivityExecution_DeadlineExceeded/NoCallerDeadline` | 3.8% (1/26) | 1h ago | [1](https://github.com/temporalio/temporal/actions/runs/22970016748/job/66684243820) | | `TestAdvancedVisibilitySuiteLegacy/TestUpsertWorkflowExecutionSearchAttributes` | 3.8% (1/26) | 21h ago | [1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717) | | `TestVersioning3FunctionalSuiteV2/TestPinnedCaN_UpgradeOnCaN_NormalWFT_NoSuggest/ForceTaskForwardForcePollForwardForceAsync` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestDeploymentVersionSuiteV0/TestVersionIgnoresDrainageSignalWhenCurrentOrRamping` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestVersioning3FunctionalSuiteV2/TestAutoUpgradeCaN_UpgradeOnCaN/ForceTaskForwardForcePollForwardAllowSync` | 3.8% (1/26) | 21h ago | [1](https://github.com/temporalio/temporal/actions/runs/22929090805/job/66546593717) | | `TestDeploymentVersionSuiteV0/TestUpdateWorkflowExecutionOptions_SetImpliedPinnedSuccess` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestDeploymentVersionSuiteV0/TestReactivationSignalCache_Deduplication_StartWorkflow` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestDeploymentVersionSuiteV2/TestVersionMissingTaskQueues_ValidSetRampingVersion` | 3.8% (1/26) | 28h ago | [1](https://github.com/temporalio/temporal/actions/runs/22912627598/job/66489344605) | | `TestDeploymentVersionSuiteV0/TestReactivationSignalCache_Deduplication_UpdateOptions` | 3.8% (1/26) | 122h ago | [1](https://github.com/temporalio/temporal/actions/runs/22775553239/job/66068056068) | | `TestNexusApiTestSuiteWithTemporalFailures/TestNexusCancelOperation_Outcomes/handler_timeout/ByNamespaceAndTaskQueue` | 3.8% (1/26) | 23h ago | [1](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600) | | `TestDeploymentVersionSuiteV0/TestStartWorkflowExecution_ReactivateVersionOnPinned` | 3.8% (1/26) | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22967233794/job/66674253006) | | `TestCronTestSuite/TestCronWorkflow` | 3.8% (1/26) | 2h ago | [1](https://github.com/temporalio/temporal/actions/runs/22967653083/job/66676176283) | | `TestWorkerDeploymentSuiteV0/TestDeploymentVersionLimits` | 3.8% (1/26) | 23h ago | [1](https://github.com/temporalio/temporal/actions/runs/22923165266/job/66526932600) | </p> </details> <details><summary>Slack</summary> <p> Flaky Tests Report - Last 7 Days *CI Success Rate:* 9/26 (34.62%) *Total Test Runs:* 1134261 *Total Failures:* 3774 *Failure Rate:* 3.33 per 1000 tests *CI Breakers:* 50 *Crashes:* 2 *Flaky Tests:* 416 *Timeouts:* 0 *CI Breakers (top 10):* • 50.0% (13/26 runs): `TestNexusStateReplicationTestSuite/EnableTransitionHistory/TestNexusCallbackReplicated` • 30.8% (8/26 runs): `TestFuncClustersTestSuite/DisableTransitionHistory/TestForceMigration_ClosedWorkflow` • 19.2% (5/26 runs): `TestFuncClustersTestSuite/DisableTransitionHistory/TestForceMigration_ResetWorkflow` • 19.2% (5/26 runs): `TestFuncClustersTestSuite/EnableTransitionHistory/TestForceMigration_ClosedWorkflow` • 15.4% (4/26 runs): `TestFuncClustersTestSuite/EnableTransitionHistory/TestForceMigration_ResetWorkflow` • 11.5% (3/26 runs): `TestDeploymentVersionSuiteV2/TestReactivationSignalCache_Deduplication_Reset` • 7.7% (2/26 runs): `TestWorkerDeploymentSuiteV2/TestResourceExhaustedErrors_Converted_To_ReadableMessage` • 7.7% (2/26 runs): `TestClientDataConverterTestSuite` • 7.7% (2/26 runs): `TestWorkerDeploymentSuiteV2/TestSetCurrentVersion_Batching` • 7.7% (2/26 runs): `TestVersioning3FunctionalSuiteV0/TestTransitionDuringTransientTask_WithSignal` *Flaky Tests (top 10):* • 100.0% (4 failures): `DATA RACE: Data race detected` • 100.0% (24 failures): `TestClientDataConverterTestSuite` • 100.0% (4 failures): `PANIC: Fail in goroutine after TestWorkerDeploymentSuiteV2/TestConcurrentPollers_DifferentTaskQueues_SameVersion_SetCurrentVersion has completed — in TestDeploymentVersionSuiteV2/TestBatchUpdateWorkflowExecutionOptions_SetPinnedThenUnset` • 100.0% (36 failures): `PANIC: Fail in goroutine after TestWorkerDeploymentSuiteV2/TestConcurrentPollers_DifferentTaskQueues_SameVersion_SetCurrentVersion has completed — in TestWorkerDeploymentSuiteV2/TestConcurrentPollers_DifferentTaskQueues_SameVersion_SetCurrentVersion` • 100.0% (4 failures): `DATA RACE: Data race detected — in go.uber.org/zap/zaptest.TestingWriter.Write` • 49.3% (187 failures): `TestVersioning3FunctionalSuiteV2/TestTransitionDuringTransientTask_WithoutSignal` • 48.9% (183 failures): `TestVersioning3FunctionalSuiteV2/TestTransitionDuringTransientTask_WithSignal` • 47.5% (161 failures): `TestVersioning3FunctionalSuiteV0/TestTransitionDuringTransientTask_WithSignal` • 47.4% (172 failures): `TestVersioning3FunctionalSuiteV0/TestTransitionDuringTransientTask_WithoutSignal` • 27.2% (71 failures): `TestFuncClustersTestSuite/DisableTransitionHistory/TestForceMigration_ClosedWorkflow` </p> </details>
103 lines
2.8 KiB
Go
103 lines
2.8 KiB
Go
package flakereport
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const boldFlakeRateThreshold = 5.0
|
|
|
|
// hoursAgo formats a timestamp as "Xh ago" relative to now.
|
|
func hoursAgo(t time.Time) string {
|
|
h := math.Round(time.Since(t).Hours())
|
|
if h < 1 {
|
|
h = 1
|
|
}
|
|
return fmt.Sprintf("%dh ago", int(h))
|
|
}
|
|
|
|
// formatReportLines returns a plain-text bullet line per report.
|
|
func formatReportLines(reports []TestReport) []string {
|
|
var lines []string
|
|
for _, r := range reports {
|
|
pct := 0.0
|
|
if r.TotalRuns > 0 {
|
|
pct = float64(r.FailureCount) / float64(r.TotalRuns) * 100.0
|
|
}
|
|
lines = append(lines, fmt.Sprintf("• %.1f%% (%d/%d): `%s`",
|
|
pct, r.FailureCount, r.TotalRuns, r.TestName))
|
|
}
|
|
return lines
|
|
}
|
|
|
|
// formatLinks formats GitHub URLs as numbered markdown links
|
|
func formatLinks(urls []string, maxLinks int) string {
|
|
linkCount := len(urls)
|
|
if linkCount > maxLinks {
|
|
linkCount = maxLinks
|
|
}
|
|
var parts []string
|
|
for i := 0; i < linkCount; i++ {
|
|
parts = append(parts, fmt.Sprintf("[%d](%s)", i+1, urls[i]))
|
|
}
|
|
return strings.Join(parts, " ")
|
|
}
|
|
|
|
// generateSuiteBreakdownTable creates a markdown table of per-suite flake data
|
|
func generateSuiteBreakdownTable(suiteReports []SuiteReport) string {
|
|
if len(suiteReports) == 0 {
|
|
return ""
|
|
}
|
|
|
|
var sb strings.Builder
|
|
sb.WriteString("| Suite | Flake Rate | Last Failure |\n")
|
|
sb.WriteString("|-------|------------|-------------|\n")
|
|
|
|
for _, sr := range suiteReports {
|
|
lastFailure := "-"
|
|
if sr.FailedRuns > 0 && !sr.LastFailure.IsZero() {
|
|
lastFailure = hoursAgo(sr.LastFailure)
|
|
}
|
|
rate := fmt.Sprintf("%.1f%% (%d/%d)", sr.FlakeRate, sr.FailedRuns, sr.TotalRuns)
|
|
if sr.FlakeRate > boldFlakeRateThreshold {
|
|
rate = "**" + rate + "**"
|
|
}
|
|
sb.WriteString(fmt.Sprintf("| `%s` | %s | %s |\n", sr.SuiteName, rate, lastFailure))
|
|
}
|
|
|
|
return sb.String()
|
|
}
|
|
|
|
// generateTestReportTable creates a markdown table of test reports with rate column.
|
|
func generateTestReportTable(reports []TestReport, rateHeader string, maxLinks int) string {
|
|
if len(reports) == 0 {
|
|
return ""
|
|
}
|
|
|
|
var sb strings.Builder
|
|
sb.WriteString(fmt.Sprintf("| Test | %s | Last Failure | Links |\n", rateHeader))
|
|
sb.WriteString("|------|------------|-------------|-------|\n")
|
|
|
|
for _, report := range reports {
|
|
pct := 0.0
|
|
if report.TotalRuns > 0 {
|
|
pct = float64(report.FailureCount) / float64(report.TotalRuns) * 100.0
|
|
}
|
|
links := formatLinks(report.GitHubURLs, maxLinks)
|
|
lastFailure := "N/A"
|
|
if !report.LastFailure.IsZero() {
|
|
lastFailure = hoursAgo(report.LastFailure)
|
|
}
|
|
rate := fmt.Sprintf("%.1f%% (%d/%d)", pct, report.FailureCount, report.TotalRuns)
|
|
if pct > boldFlakeRateThreshold {
|
|
rate = "**" + rate + "**"
|
|
}
|
|
sb.WriteString(fmt.Sprintf("| `%s` | %s | %s | %s |\n",
|
|
report.TestName, rate, lastFailure, links))
|
|
}
|
|
|
|
return sb.String()
|
|
}
|