mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.6
0897b30e30
The SQLLogicTest CI job now tracks individual failing tests rather than
only checking aggregate thresholds. Changes:
- Add `tests/sqllogic/known_failures.txt` — a committed list of known
failure IDs (`test_name:position`), one per line. Initially empty;
to be populated from a CI run's `current_failures.txt` artifact.
- Add `tests/sqllogic/generate_known_failures.py` — standalone script
to extract failure IDs from a `complete-clickhouse` `report.json`.
- Update `ci/jobs/sqllogic_test.py`:
- After running complete-test, extract all current failures with
details (query, reason) and save as `current_failures.txt` artifact.
- Compare current failures against `known_failures.txt` baseline.
- Report new failures (regressions) at the top of the HTML report
with test file, line number, SQL query, and failure reason.
- Report fixed tests (tests that now pass but were in the baseline).
- Fail the job if new failures are detected (when baseline is populated).
- Keep the existing threshold check as a fallback.
Workflow to populate the baseline:
1. Merge this PR.
2. After the next master CI run, download the `current_failures.txt` artifact.
3. Copy it to `tests/sqllogic/known_failures.txt` and commit.
Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
59 lines
1.6 KiB
Python
59 lines
1.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract known failures from a SQLLogicTest complete-clickhouse report.json.
|
|
|
|
Usage:
|
|
python3 generate_known_failures.py <report.json> [--output <file>]
|
|
|
|
Reads the report.json produced by the complete-clickhouse stage and prints
|
|
one failure ID per line (test_name:position), sorted. The output can be
|
|
redirected to tests/sqllogic/known_failures.txt.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
|
|
|
|
def extract_failure_ids(report):
|
|
"""Extract sorted failure IDs from a report dict."""
|
|
failures = []
|
|
for test_name, test_data in report.get("tests", {}).items():
|
|
for position, req in test_data.get("requests", {}).items():
|
|
if req.get("status") == "error":
|
|
failures.append(f"{test_name}:{position}")
|
|
failures.sort()
|
|
return failures
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Extract known failures from a SQLLogicTest report.json"
|
|
)
|
|
parser.add_argument("report_json", help="Path to report.json file")
|
|
parser.add_argument(
|
|
"--output",
|
|
"-o",
|
|
help="Output file (default: stdout)",
|
|
default=None,
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
with open(args.report_json, "r", encoding="utf-8") as f:
|
|
report = json.load(f)
|
|
|
|
failure_ids = extract_failure_ids(report)
|
|
|
|
out = open(args.output, "w", encoding="utf-8") if args.output else sys.stdout
|
|
try:
|
|
for fid in failure_ids:
|
|
out.write(fid + "\n")
|
|
finally:
|
|
if out is not sys.stdout:
|
|
out.close()
|
|
|
|
print(f"Extracted {len(failure_ids)} failures", file=sys.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|