hanzoai/tasks
2
0
Fork 0
mirror of https://github.com/hanzoai/tasks.git synced 2026-10-04 02:04:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

flakes: improve flaky test report to link to the 3 most recent failures (#8613)

Browse Source 
## What changed?
- Updated tringa SQL query to fetch all individual test failures instead
of aggregated results
- Modified Python script to group test failures and display up to 3
(configurable) links per test
- Added `--max-links` parameter to control how many failure links are
shown per test (default: 3)
- Changed markdown output format from single link to multiple numbered
links: `TestName: [1](url1) [2](url2) [3](url3)`

### Sample Output:
|Name with Url|Count|
| -------- | ------- |

|`TestWorkflowAliasSearchAttributeTestSuite/TestWorkflowAliasSearchAttribute_CustomSearchAttributeOverride`:
[1](https://github.com/temporalio/temporal/actions/runs/19240366865/job/55001542397)
[2](https://github.com/temporalio/temporal/actions/runs/19240366865/job/55001542356)
[3](https://github.com/temporalio/temporal/actions/runs/19240366865/job/55001542323)|87|
|`TestTransientTaskSuite/TestTransientWorkflowTaskHistorySize`:
[1](https://github.com/temporalio/temporal/actions/runs/19240366865/job/55001542322)
[2](https://github.com/temporalio/temporal/actions/runs/19240366865/job/55001542318)
[3](https://github.com/temporalio/temporal/actions/runs/19240366865/job/55001542298)|19|

|`TestTaskQueueSuite/TestWholeQueueLimit_TighterThanPerKeyDefault_IsEnforced`:
[1](https://github.com/temporalio/temporal/actions/runs/19240366865/job/55001542340)
[2](https://github.com/temporalio/temporal/actions/runs/19152120651/job/54744877613)
[3](https://github.com/temporalio/temporal/actions/runs/19152120651/job/54744877611)|15|

## Why?
- Makes the flaky test report more actionable and useful for engineers

## How did you test it?
- [x] built
- [x] run locally and tested manually
- [ ] covered by existing tests
- [ ] added new unit test(s)
- [ ] added new functional test(s)

## Potential risks
- SQL query now returns more data (all individual failures vs aggregated
counts), which may increase data processing time
This commit is contained in:
Sean Kane
2025-11-10 14:29:55 -07:00
committed by GitHub
parent c278efc8c3
commit 3c5b0d5e13
2 changed files with 129 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/flaky-tests-report.yml
+9 -2
View File
@@ -11,6 +11,11 @@ on:
required: false
default: '7'
type: string
max_links:
description: 'Maximum number of failure links to show per test'
required: false
default: '3'
type: string
permissions:
contents: read
@@ -64,7 +69,7 @@ jobs:
mkdir -p tools/flakes/out
tringa --json --since-days "$DAYS_PARAM" repo sql \
'select classname, name, count(*) as failure_count, max(artifact) as artifact from test where passed = false and skipped = false group by classname, name order by failure_count desc' \
'select classname, name, artifact from test where passed = false and skipped = false order by classname, name, artifact desc' \
--branch main \
--workflow-id 80591745 \
https://github.com/temporalio/temporal > tools/flakes/out/out.json
@@ -81,6 +86,7 @@ jobs:
RUN_ID: ${{ github.run_id }}
REF_NAME: ${{ github.ref_name }}
SHA: ${{ github.sha }}
MAX_LINKS_PARAM: ${{ github.event.inputs.max_links || '3' }}
run: |
set -x
cd tools/flakes && uv run main.py \
@@ -89,7 +95,8 @@ jobs:
--slack-webhook "$SLACK_WEBHOOK" \
--run-id "$RUN_ID" \
--ref-name "$REF_NAME" \
--sha "$SHA"
--sha "$SHA" \
--max-links "$MAX_LINKS_PARAM"
- name: Upload generated reports
uses: actions/upload-artifact@v4
tools/flakes/main.py
+120 -38
View File
@@ -8,11 +8,9 @@ from typing import Any, Dict
import requests
def process_tests(data, pattern, output_file: str):
lines = []
lines.append(f"|{pattern}, Name with Url|Count|\n")
lines.append("| -------- | ------- |\n")
def process_tests(data, pattern, output_file: str, max_links: int = 3):
# Group data by test name and collect artifacts for tests matching pattern
test_groups = {}
for item in data:
name_parts = item["name"].split("/")
if len(name_parts) < 2:
@@ -20,6 +18,10 @@ def process_tests(data, pattern, output_file: str):
if not item["name"].endswith(pattern):
continue
test_name = item["name"]
if test_name not in test_groups:
test_groups[test_name] = []
parts = item["artifact"].split("--")
if len(parts) > 0 and len(parts[1]) > 0 and len(parts[2]) > 0:
p2 = parts[2]
@@ -32,39 +34,99 @@ def process_tests(data, pattern, output_file: str):
else:
job_url = item["artifact"]
lines.append(f"|[{item['name']}]({job_url})|{item['failure_count']}|\n")
test_groups[test_name].append(job_url)
with open(output_file, "w") as outfile:
outfile.writelines(lines)
# Transform into list with counts and multiple links
transformed = []
for test_name, artifacts in test_groups.items():
failure_count = len(artifacts)
# Get up to max_links most recent artifacts (already sorted desc from SQL)
recent_artifacts = artifacts[:max_links]
transformed.append({
"name": test_name,
"count": failure_count,
"artifacts": recent_artifacts,
})
def process_crash(data, pattern, output_file: str):
# Sort by failure count descending
transformed.sort(key=lambda x: x["count"], reverse=True)
# Write markdown table
lines = []
lines.append(f"|{pattern}, Name with Url|Count|\n")
lines.append("| -------- | ------- |\n")
for item in data:
if "crash" in item["name"]:
job_url = ""
parts = item["artifact"].split("--")
if len(parts) > 0 and len(parts[1]) > 0:
job_url = (
f"https://github.com/temporalio/temporal/actions/runs/{parts[1]}"
)
else:
job_url = item["artifact"]
lines.append(f"|[{item['name']}]({job_url})|{item['failure_count']}|\n")
for item in transformed:
# Format: `TestName`: [1](url1) [2](url2) [3](url3)
links = " ".join([f"[{i+1}]({url})" for i, url in enumerate(item["artifacts"])])
lines.append(f"|`{item['name']}`: {links}|{item['count']}|\n")
with open(output_file, "w") as outfile:
outfile.writelines(lines)
def process_flaky(data, output_file: str):
def process_crash(data, pattern, output_file: str, max_links: int = 3):
# Group data by test name and collect artifacts for crash tests
test_groups = {}
for item in data:
if "crash" not in item["name"]:
continue
test_name = item["name"]
if test_name not in test_groups:
test_groups[test_name] = []
parts = item["artifact"].split("--")
if len(parts) > 0 and len(parts[1]) > 0:
job_url = (
f"https://github.com/temporalio/temporal/actions/runs/{parts[1]}"
)
else:
job_url = item["artifact"]
test_groups[test_name].append(job_url)
# Transform into list with counts and multiple links
transformed = []
for test_name, artifacts in test_groups.items():
failure_count = len(artifacts)
# Get up to max_links most recent artifacts (already sorted desc from SQL)
recent_artifacts = artifacts[:max_links]
transformed.append({
"name": test_name,
"count": failure_count,
"artifacts": recent_artifacts,
})
# Sort by failure count descending
transformed.sort(key=lambda x: x["count"], reverse=True)
# Write markdown table
lines = []
lines.append(f"|{pattern}, Name with Url|Count|\n")
lines.append("| -------- | ------- |\n")
for item in transformed:
# Format: `TestName`: [1](url1) [2](url2) [3](url3)
links = " ".join([f"[{i+1}]({url})" for i, url in enumerate(item["artifacts"])])
lines.append(f"|`{item['name']}`: {links}|{item['count']}|\n")
with open(output_file, "w") as outfile:
outfile.writelines(lines)
def process_flaky(data, output_file: str, max_links: int = 3):
# Group data by test name and collect artifacts
test_groups = {}
for item in data:
name_parts = item["name"].split("/")
if len(name_parts) < 2:
continue
test_name = item["name"]
if test_name not in test_groups:
test_groups[test_name] = []
parts = item["artifact"].split("--")
if len(parts) > 0 and len(parts[1]) > 0 and len(parts[2]) > 0:
p2 = parts[2]
@@ -77,23 +139,35 @@ def process_flaky(data, output_file: str):
else:
job_url = item["artifact"]
transformed.append(
{
"name": item["name"], # the 'name' field from the original
"job_url": job_url, # the newly constructed job URL
"count": item["failure_count"], # store the failure_count
}
)
test_groups[test_name].append(job_url)
if len(transformed) > 10:
break
# Transform into list with counts and multiple links
transformed = []
for test_name, artifacts in test_groups.items():
failure_count = len(artifacts)
# Get up to max_links most recent artifacts (already sorted desc from SQL)
recent_artifacts = artifacts[:max_links]
transformed.append({
"name": test_name,
"count": failure_count,
"artifacts": recent_artifacts,
})
# Sort by failure count descending
transformed.sort(key=lambda x: x["count"], reverse=True)
# Limit to top 10 flaky tests
transformed = transformed[:10]
# Write both markdown table (for GitHub) and plain text (for Slack)
lines = []
lines.append("|Name with Url|Count|\n")
lines.append("| -------- | ------- |\n")
for item in transformed:
lines.append(f"|[{item['name']}]({item['job_url']})|{item['count']}|\n")
# Format: `TestName`: [1](url1) [2](url2) [3](url3)
links = " ".join([f"[{i+1}]({url})" for i, url in enumerate(item["artifacts"])])
lines.append(f"|`{item['name']}`: {links}|{item['count']}|\n")
with open(output_file, "w") as outfile:
outfile.writelines(lines)
@@ -340,7 +414,7 @@ def write_github_actions_summary(summary_content: str) -> None:
print(f"⚠️ Warning: Could not write GitHub Actions summary: {e}", file=sys.stderr)
def process_json_file(input_filename: str):
def process_json_file(input_filename: str, max_links: int = 3):
with open(input_filename, "r") as file:
# Load the file content as JSON
data = json.load(file)
@@ -348,10 +422,10 @@ def process_json_file(input_filename: str):
# Create output directory if it doesn't exist
os.makedirs("out", exist_ok=True)
process_flaky(data, "out/flaky.txt")
process_tests(data, "(timeout)", "out/timeout.txt")
process_tests(data, "(retry 2)", "out/retry.txt")
process_crash(data, "(crash)", "out/crash.txt")
process_flaky(data, "out/flaky.txt", max_links)
process_tests(data, "(timeout)", "out/timeout.txt", max_links)
process_tests(data, "(retry 2)", "out/retry.txt", max_links)
process_crash(data, "(crash)", "out/crash.txt", max_links)
def create_argument_parser() -> argparse.ArgumentParser:
@@ -379,6 +453,14 @@ def create_argument_parser() -> argparse.ArgumentParser:
parser.add_argument("--ref-name", help="Git branch name")
parser.add_argument("--sha", help="Git commit SHA")
# Display options
parser.add_argument(
"--max-links",
type=int,
default=3,
help="Maximum number of failure links to show per test (default: 3)",
)
return parser
@@ -469,7 +551,7 @@ def main():
# Try to process the JSON file and handle both success and failure cases
try:
process_json_file(args.file)
process_json_file(args.file, args.max_links)
print(f"Successfully processed {args.file}")
handle_success_case(args)