mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Nikita TaranovandClaude Opus 4.8
f2f97a22c4
The unit-test Bugfix Validation job was allow_failure=False, so an inconclusive ERROR (the before-binary couldn't compile because the test references PR-introduced code, or it crashed before any test ran, or submodules were missing) blocked merge just like a definitive failed-to-reproduce. "We couldn't determine" is not grounds to block. Make the job allow_failure=True (like the FT/IT bugfix jobs) and move the merge decision into new_tests_check.py: for a unit-only bug-fix PR, block iff the job reported a definitive FAIL (the added test passes on the merge-base too, so it doesn't catch the bug). A reproduction (OK/XFAIL), an inconclusive ERROR, or a skip does not block. Validated on real CI before this change: reproduce=OK (#108833), fail-to-reproduce=FAIL (#108946), compile-inconclusive=ERROR (#108947). With this change the ERROR case (#108947) no longer blocks; only the FAIL case (#108946) does. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
70 lines
2.1 KiB
Python
70 lines
2.1 KiB
Python
"""
|
|
Tests for the unit-test branch of the bug-fix gate in
|
|
`ci.jobs.scripts.workflow_hooks.new_tests_check`.
|
|
|
|
The unit-test Bugfix Validation job is `allow_failure`, so the merge decision for a
|
|
unit-only bug-fix PR is made here: block iff that job definitively FAILED to reproduce
|
|
(the added test passes on the merge-base too). A reproduction (OK/XFAIL), an inconclusive
|
|
ERROR (the before-binary could not be compiled or crashed before any test), or a skip
|
|
must NOT block — "we couldn't determine" is not grounds to block merge.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../.."))
|
|
|
|
from ci.defs.defs import JobNames
|
|
from ci.jobs.scripts.workflow_hooks import new_tests_check as ntc
|
|
from ci.praktika.result import Result
|
|
|
|
|
|
class _FakeWorkflowResult:
|
|
def __init__(self, ut_status):
|
|
self.results = []
|
|
if ut_status is not None:
|
|
self.results.append(
|
|
Result(name=JobNames.BUGFIX_VALIDATE_UT, status=ut_status)
|
|
)
|
|
|
|
|
|
class _FakeInfo:
|
|
workflow_name = "PR"
|
|
|
|
|
|
def _patch(monkeypatch, ut_status):
|
|
monkeypatch.setattr(ntc, "Info", lambda: _FakeInfo())
|
|
monkeypatch.setattr(
|
|
ntc.Result, "from_fs", staticmethod(lambda name: _FakeWorkflowResult(ut_status))
|
|
)
|
|
|
|
|
|
def test_refuted_only_when_job_failed(monkeypatch):
|
|
_patch(monkeypatch, Result.Status.FAIL)
|
|
assert ntc.unit_bugfix_validation_refuted() is True
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"status",
|
|
[
|
|
Result.Status.OK, # reproduced, or skipped (nothing to validate)
|
|
Result.Status.ERROR, # inconclusive — before-binary couldn't compile / crashed
|
|
Result.Status.SKIPPED, # job filtered out
|
|
],
|
|
)
|
|
def test_not_refuted_for_non_fail(monkeypatch, status):
|
|
_patch(monkeypatch, status)
|
|
assert ntc.unit_bugfix_validation_refuted() is False
|
|
|
|
|
|
def test_not_refuted_when_job_absent(monkeypatch):
|
|
# No unit Bugfix Validation job in the workflow result -> do not block.
|
|
_patch(monkeypatch, None)
|
|
assert ntc.unit_bugfix_validation_refuted() is False
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(pytest.main([__file__, "-v"]))
|