mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.6
bc65bc7d5e
The ACME TLS tests (`test_acme_tls/test_single_node.py` and `test_acme_tls/test_multi_node.py`) were flaky on ARM CI runners, failing with "Failed to get expected certificate issuer" after the 60-second timeout. https://s3.amazonaws.com/clickhouse-test-reports/json.html?REF=master&sha=2c2f86d0b7c17e67fe6b5fe2cbc295d93c709b10&name_0=MasterCI&name_1=Integration%20tests%20%28arm_binary%2C%20distributed%20plan%2C%201%2F4%29 Evidence from the CI logs shows both tests passed on retry (single-node in 24s, multi-node in 31s) but failed on the first attempt when the system was under heavier load from parallel test workers. On ARM, RSA 4096-bit key generation (done twice: account key + certificate key) is significantly slower, and combined with unnecessary delays in the ACME client scheduling, the total time exceeded the 60-second test timeout. Changes in `ACME::Client::refreshCertificatesTask`: 1. Lock contention delay reduced from 10s to 5s (3 locations). When another node holds the certificate order lock, this is normal multi-node behavior, not an error. Using `REFRESH_TASK_AFTER_ERROR_MS` (10s) was excessive; changed to `REFRESH_TASK_HAPPY_PATH_MS` (5s). 2. Post-finalize delay reduced from 5s to 1s. After submitting the CSR to finalize the ACME order, waiting 5s before checking if the certificate is ready was unnecessarily long. Changed to `REFRESH_TASK_IN_A_SECOND` (1s). 3. Order status polling interval fixed from configurable interval to 1s. When actively polling for order completion (status "pending" or "processing"), the code used `refresh_certificates_task_interval_ms` which defaults to 1 hour in production. This was also a production bug: after order finalization, ClickHouse would wait up to 1 hour before checking if the certificate was ready. Changed to `REFRESH_TASK_IN_A_SECOND` (1s). Test timeout increased from 60s to 120s in both tests to provide a safety margin for slow ARM runners. Co-Authored-By: Claude Opus 4.6 <[email protected]>
69 lines
1.9 KiB
Python
69 lines
1.9 KiB
Python
import logging
|
|
import time
|
|
import requests
|
|
|
|
import pytest
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
logging.getLogger().setLevel(logging.INFO)
|
|
logging.getLogger().addHandler(logging.StreamHandler())
|
|
|
|
single_replica_cluster = ClickHouseCluster(__file__)
|
|
node = single_replica_cluster.add_instance(
|
|
"node_acme",
|
|
main_configs=["configs/config.xml"],
|
|
stay_alive=True,
|
|
with_zookeeper=True,
|
|
with_letsencrypt_pebble=True,
|
|
|
|
ipv4_address="10.5.11.11", # never copy-paste this line
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def started_single_replica_cluster():
|
|
try:
|
|
single_replica_cluster.start()
|
|
yield single_replica_cluster
|
|
finally:
|
|
single_replica_cluster.shutdown()
|
|
|
|
|
|
def test_acme_authorization(started_single_replica_cluster):
|
|
# Let Pebble know where to find our server
|
|
requests.post(
|
|
'http://10.5.11.3:8055/add-a',
|
|
json={'host': 'single.integration-tests.clickhouse.com', 'addresses': ['10.5.11.11']}
|
|
)
|
|
|
|
for _ in range(120):
|
|
time.sleep(1)
|
|
|
|
curl_result = node.exec_in_container(
|
|
[
|
|
"bash",
|
|
"-c",
|
|
"curl -k -v 'https://127.0.0.1:3443' 2>&1 | grep issuer: || true",
|
|
]
|
|
)
|
|
|
|
if not curl_result:
|
|
continue
|
|
|
|
assert "CN=Pebble Intermediate CA" in curl_result
|
|
|
|
zk = started_single_replica_cluster.get_kazoo_client("zoo1")
|
|
zk.start()
|
|
|
|
assert zk.exists("/clickhouse/acme")
|
|
assert zk.exists("/clickhouse/acme/10.5.11.2")
|
|
assert zk.exists("/clickhouse/acme/10.5.11.2/account_private_key")
|
|
assert zk.exists("/clickhouse/acme/10.5.11.2/challenges")
|
|
assert zk.exists("/clickhouse/acme/10.5.11.2/domains")
|
|
assert len(zk.get_children("/clickhouse/acme/10.5.11.2/domains")) == 1
|
|
|
|
return
|
|
|
|
raise Exception("Failed to get expected certificate issuer")
|