mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
zeekay
9a90943b8f
13,838 upstream commits, forward from the 26.6.1.1 fork point. 129 conflicts: 37 resolved mechanically (take upstream's file, re-apply the brand sweep) and 92 by hand where our fork and upstream had both changed the same code. The rule throughout: upstream owns the engine, we own the naming and a small set of deliberate deviations. Never resolve by discarding an upstream change — a dropped rename or a missed new parameter becomes a compile error a thousand files away. Deviations verified intact after the merge: ENABLE_XRAY still defaults OFF (CMakeLists.txt:105) X-Datastore-User read by the server (authenticateUserByHTTP.cpp) X-ClickHouse-* in src/ 0 hanzo/research present system.*_log retention bound present ArrowFlight rip held against upstream re-adding it Two things worth naming, because neither shows up as a conflict: src/CMakeLists.txt gained an UNGUARDED `add_object_library(datastore_storages_arrowflight Storages/ArrowFlight)` from upstream, for a directory our gRPC rip deleted. It sat outside every conflict hunk, so a resolver reading only the markers would sail straight past it into a CMake configure error. Removed; the Server/ArrowFlight sibling at line 474 stays because it is inside `if (TARGET ch_contrib::arrow_flight)`. The brand sweep had rewritten github.com/ClickHouse/ClickHouse issue links to github.com/Datastore/Datastore — a dead host. Those point at upstream's own tracker and are restored, alongside the contributors list, sql.clickhouse.com and the ClickHouse Inc copyright: attribution is theirs, not ours to rename. base/glibc-compatibility/memcpy/memcpy.h is deleted, not kept: upstream dropped it for LLVM-libc's, and our only change to it was a comment rebrand.
260 lines
9.5 KiB
Python
260 lines
9.5 KiB
Python
from praktika import Workflow
|
|
|
|
from ci.defs.defs import (
|
|
BASE_BRANCH,
|
|
DOCKERS,
|
|
SECRETS,
|
|
ArtifactConfigs,
|
|
JobNames,
|
|
)
|
|
from ci.defs.job_configs import JobConfigs
|
|
from ci.jobs.scripts.workflow_hooks.filter_job import should_skip_job
|
|
from ci.jobs.scripts.workflow_hooks.trusted import can_be_tested
|
|
|
|
ALL_FUNCTIONAL_TESTS = [job.name for job in JobConfigs.functional_tests_jobs]
|
|
|
|
CORE_BLOCKING_JOB_NAMES = [
|
|
job.name
|
|
for job in JobConfigs.functional_tests_jobs
|
|
if any(
|
|
substr in job.name
|
|
for substr in (
|
|
"_debug, parallel",
|
|
"_binary, parallel",
|
|
"_binary, sequential",
|
|
"_asan_ubsan, distributed plan, parallel",
|
|
"_asan_ubsan, db disk, distributed plan, sequential",
|
|
"_tsan, parallel",
|
|
)
|
|
)
|
|
] + [
|
|
job.name
|
|
for job in JobConfigs.integration_test_jobs_required
|
|
if "_asan_ubsan, db disk, old analyzer" in job.name
|
|
] + [
|
|
job.name
|
|
for job in JobConfigs.unittest_jobs
|
|
]
|
|
|
|
STYLE_AND_FAST_TESTS = [
|
|
JobNames.STYLE_CHECK,
|
|
JobNames.FAST_TEST,
|
|
*[j.name for j in JobConfigs.tidy_build_arm_jobs],
|
|
]
|
|
|
|
CODE_REVIEW_BLOCKING_JOBS = [
|
|
JobNames.STYLE_CHECK,
|
|
JobNames.FAST_TEST,
|
|
]
|
|
|
|
REGULAR_BUILD_NAMES = [job.name for job in JobConfigs.build_jobs]
|
|
|
|
PLAIN_FUNCTIONAL_TEST_JOB = [
|
|
j for j in JobConfigs.functional_tests_jobs if "amd_debug, parallel" in j.name
|
|
][0]
|
|
|
|
workflow = Workflow.Config(
|
|
name="PR",
|
|
event=Workflow.Event.PULL_REQUEST,
|
|
base_branches=[BASE_BRANCH],
|
|
jobs=[
|
|
JobConfigs.style_check,
|
|
JobConfigs.code_review.set_run_after(CODE_REVIEW_BLOCKING_JOBS),
|
|
JobConfigs.docs_job,
|
|
JobConfigs.docs_job_mintlify,
|
|
JobConfigs.fast_test,
|
|
JobConfigs.ci_tests.set_run_after(CORE_BLOCKING_JOB_NAMES),
|
|
*JobConfigs.darwin_fast_test_jobs,
|
|
*JobConfigs.tidy_build_arm_jobs,
|
|
*[job.set_run_after(STYLE_AND_FAST_TESTS) for job in JobConfigs.build_jobs],
|
|
*[
|
|
job.set_run_after(STYLE_AND_FAST_TESTS)
|
|
for job in JobConfigs.extra_validation_build_jobs
|
|
],
|
|
*[
|
|
job.set_run_after(REGULAR_BUILD_NAMES)
|
|
for job in JobConfigs.release_build_jobs
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.special_build_jobs
|
|
],
|
|
*[job.set_run_after(STYLE_AND_FAST_TESTS) for job in JobConfigs.build_llvm_coverage_job],
|
|
# TODO: stabilize new jobs and remove set_allow_failure
|
|
JobConfigs.lightweight_functional_tests_job,
|
|
*[j.set_allow_failure() for j in JobConfigs.stateless_tests_targeted_pr_jobs],
|
|
JobConfigs.integration_test_targeted_pr_jobs[0].set_allow_failure(),
|
|
JobConfigs.ast_fuzzer_targeted_pr_jobs[0].set_allow_failure(),
|
|
JobConfigs.ast_fuzzer_targeted_pr_jobs[1].set_allow_failure(),
|
|
*JobConfigs.stateless_tests_flaky_pr_jobs,
|
|
*JobConfigs.integration_test_asan_flaky_pr_jobs,
|
|
# Per-arch Bugfix Validation Checks (functional + integration tests on
|
|
# both amd64 and aarch64). Each per-arch variant has
|
|
# `allow_failure=True` so an individual FAIL doesn't block PR merge -
|
|
# the aggregate decision (validate iff at least one arch passed) lives
|
|
# in the `new_tests_check.py` workflow post-hook below.
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.bugfix_validation_ft_pr_jobs
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.bugfix_validation_it_jobs
|
|
],
|
|
# Unit-test (gtest) bugfix validation: a single AMD-only job (allow_failure)
|
|
# that builds a merge-base "before" binary and runs the touched suite against it.
|
|
# It is not part of the per-arch FT/IT aggregation; instead new_tests_check.py
|
|
# blocks the unit case iff this job reported a definitive FAIL (failed to
|
|
# reproduce) — a reproduction or an inconclusive ERROR does not block.
|
|
# Like the sibling FT/IT jobs, it is deferred behind the core blocking jobs.
|
|
JobConfigs.bugfix_validation_ut_job.set_run_after(CORE_BLOCKING_JOB_NAMES),
|
|
*[
|
|
j.set_run_after(
|
|
CORE_BLOCKING_JOB_NAMES
|
|
if j.name not in CORE_BLOCKING_JOB_NAMES
|
|
else []
|
|
)
|
|
for j in JobConfigs.functional_tests_jobs
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.functional_tests_jobs_azure
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.functional_test_llvm_coverage_jobs
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.functional_test_excluded_from_llvm_job
|
|
],
|
|
*[
|
|
job.set_run_after(
|
|
CORE_BLOCKING_JOB_NAMES if job.name not in CORE_BLOCKING_JOB_NAMES else []
|
|
)
|
|
for job in JobConfigs.integration_test_jobs_required[:]
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.integration_test_jobs_non_required
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.integration_test_llvm_coverage_jobs
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.integration_test_excluded_from_llvm_job
|
|
],
|
|
*JobConfigs.unittest_jobs,
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.unittest_llvm_coverage_job
|
|
],
|
|
JobConfigs.docker_server.set_run_after(
|
|
CORE_BLOCKING_JOB_NAMES
|
|
),
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.install_check_jobs
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.compatibility_test_jobs
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.stress_test_jobs
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.upgrade_test_jobs
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.ast_fuzzer_jobs
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.buzz_fuzzer_jobs
|
|
],
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.performance_comparison_with_master_head_jobs
|
|
],
|
|
# ClickBench runs on PRs only when files in its digest change
|
|
# (see `clickbench_jobs.digest_config`), so the cost is bounded.
|
|
*[
|
|
job.set_run_after(CORE_BLOCKING_JOB_NAMES)
|
|
for job in JobConfigs.clickbench_jobs
|
|
],
|
|
JobConfigs.llvm_coverage_job,
|
|
JobConfigs.sqllogic_test_master_job.set_run_after(
|
|
CORE_BLOCKING_JOB_NAMES
|
|
),
|
|
JobConfigs.sqlstorm_test_job.set_run_after(
|
|
CORE_BLOCKING_JOB_NAMES
|
|
),
|
|
# Keeper stress (PR): 3 no-fault scenarios (prod-mix, read-multi, write-multi),
|
|
# default backend only, 15 min each. Runs when src/Coordination or stress test files change.
|
|
JobConfigs.keeper_stress_job
|
|
.set_name("Keeper Stress Tests (PR)")
|
|
.set_timeout(3 * 3600),
|
|
*JobConfigs.toolchain_build_jobs,
|
|
],
|
|
artifacts=[
|
|
*ArtifactConfigs.unittests_binaries,
|
|
*ArtifactConfigs.clickhouse_binaries,
|
|
*ArtifactConfigs.clickhouse_debians,
|
|
*ArtifactConfigs.clickhouse_rpms,
|
|
*ArtifactConfigs.clickhouse_tgzs,
|
|
ArtifactConfigs.fuzzers,
|
|
ArtifactConfigs.fuzzers_corpus,
|
|
*ArtifactConfigs.llvm_profdata_file,
|
|
ArtifactConfigs.llvm_coverage_info_file,
|
|
ArtifactConfigs.toolchain_pgo_bolt_amd,
|
|
ArtifactConfigs.toolchain_pgo_bolt_arm,
|
|
],
|
|
dockers=DOCKERS,
|
|
enable_dockers_manifest_merge=True,
|
|
secrets=SECRETS,
|
|
enable_job_filtering_by_changes=True,
|
|
enable_cache=True,
|
|
enable_report=True,
|
|
enable_cidb=True,
|
|
enable_merge_ready_status=True,
|
|
enable_gh_summary_comment=True,
|
|
enable_commit_status_on_failure=False,
|
|
enable_open_issues_check=True,
|
|
enable_slack_feed=True,
|
|
pre_hooks=[
|
|
can_be_tested,
|
|
"python3 ./ci/jobs/scripts/workflow_hooks/store_data.py",
|
|
"python3 ./ci/jobs/scripts/workflow_hooks/pr_labels_and_category.py",
|
|
"python3 ./ci/jobs/scripts/workflow_hooks/version_log.py",
|
|
"python3 ./ci/jobs/scripts/workflow_hooks/team_notifications.py",
|
|
],
|
|
workflow_filter_hooks=[should_skip_job],
|
|
post_hooks=[
|
|
"python3 ./ci/jobs/scripts/workflow_hooks/pr_body_check.py",
|
|
"python3 ./ci/jobs/scripts/workflow_hooks/feature_docs.py",
|
|
"python3 ./ci/jobs/scripts/workflow_hooks/new_tests_check.py",
|
|
"python3 ./ci/jobs/scripts/workflow_hooks/can_be_merged.py",
|
|
"python3 ./ci/jobs/scripts/workflow_hooks/check_report_messages.py",
|
|
],
|
|
job_aliases={
|
|
"integration": JobConfigs.integration_test_jobs_non_required[
|
|
0
|
|
].name, # plain integration test job, no old analyzer, no dist plan
|
|
"fast": "Fast test",
|
|
"functional": PLAIN_FUNCTIONAL_TEST_JOB.name,
|
|
"build_debug": "Build (amd_debug)",
|
|
"build": "Build (amd_binary)",
|
|
},
|
|
runs_on_label_prefix="pr-",
|
|
)
|
|
|
|
WORKFLOWS = [
|
|
workflow,
|
|
]
|