mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Sonnet 5
c12cd88178
4 new test directories (generic settings-profile constraints, table data-drop size guards, startup-script dictionary creation, and Delta Lake concurrent shuffle reads) plus AWS STS assume-role coverage added to the existing test_storage_delta and test_storage_s3_queue suites, ported from ClickHouse's private/Cloud fork. None of these depend on SharedMergeTree, Shared Catalog, or any other Cloud-only feature. test_storage_s3_queue/test_sts_smoke.py needed a small, generic extension to the create_table() helper in helpers/s3_queue_common.py to pass through an extra_credentials() clause and support schema-less CREATE TABLE (empty format string) - matches the private fork's helper exactly. Every test here was run against a real local build via `python -m ci.praktika run "integration" --test ...` and confirmed passing before being added. A larger initial candidate list (17 directories, including test_packed_io, test_cache_warmer, test_prewarm_cache_merges, and additions to test_check_table, test_database_glue, test_default_compression_codec, test_replicated_database, and test_s3_cluster) was pared down after actually running them surfaced either Cloud-only dependencies invisible to code review, or real unexplained behavioral differences (S3 request counts, CHECK TABLE corruption detection on packed parts, a disks-CLI packed-io command mismatch) that need further investigation before they can be safely ported. Co-Authored-By: Claude Sonnet 5 <[email protected]>
94 lines
2.8 KiB
Python
94 lines
2.8 KiB
Python
import logging
|
|
|
|
import pytest
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
from test_storage_delta.test import (
|
|
get_spark,
|
|
write_delta_from_file,
|
|
upload_directory,
|
|
create_initial_data_file,
|
|
randomize_table_name,
|
|
)
|
|
from test_storage_s3.test_sts import run_s3_mocks
|
|
|
|
@pytest.fixture(scope="module")
|
|
def started_cluster():
|
|
try:
|
|
cluster = ClickHouseCluster(__file__)
|
|
cluster.add_instance(
|
|
"s3_with_environment_credentials",
|
|
with_minio=True,
|
|
env_variables={
|
|
"AWS_ACCESS_KEY_ID": "aws",
|
|
"AWS_SECRET_ACCESS_KEY": "aws123",
|
|
},
|
|
main_configs=[
|
|
"configs/config.d/use_environment_credentials.xml",
|
|
],
|
|
user_configs=[
|
|
"configs/allow_server_credentials.xml",
|
|
],
|
|
)
|
|
|
|
sts = cluster.add_instance(
|
|
name="sts.amazonaws.com",
|
|
hostname="sts.amazonaws.com",
|
|
image="clickhouse/python-bottle",
|
|
tag="latest",
|
|
stay_alive=True,
|
|
)
|
|
sts.stop_clickhouse(kill=True)
|
|
cluster.spark_session = get_spark()
|
|
|
|
logging.info("Starting cluster...")
|
|
cluster.start()
|
|
|
|
if int(cluster.instances["s3_with_environment_credentials"].query("SELECT count() FROM system.table_engines WHERE name = 'DeltaLake'").strip()) == 0:
|
|
pytest.skip(
|
|
"DeltaLake engine is not available"
|
|
)
|
|
|
|
logging.info("Cluster started")
|
|
|
|
logging.info("S3 bucket created")
|
|
run_s3_mocks(cluster)
|
|
|
|
yield cluster
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def test_sts_smoke(started_cluster):
|
|
spark = started_cluster.spark_session
|
|
minio_client = started_cluster.minio_client
|
|
TABLE_NAME = randomize_table_name("test_sts_smoke")
|
|
bucket = started_cluster.minio_bucket
|
|
|
|
if not minio_client.bucket_exists(bucket):
|
|
minio_client.make_bucket(bucket)
|
|
|
|
node_name = "s3_with_environment_credentials"
|
|
instance = started_cluster.instances[node_name]
|
|
parquet_data_path = create_initial_data_file(
|
|
started_cluster,
|
|
instance,
|
|
"SELECT toUInt64(number), toString(number) FROM numbers(100)",
|
|
TABLE_NAME,
|
|
node_name=node_name,
|
|
)
|
|
|
|
write_delta_from_file(spark, parquet_data_path, f"/{TABLE_NAME}")
|
|
upload_directory(minio_client, bucket, f"/{TABLE_NAME}", "")
|
|
|
|
assert 100 == int(
|
|
instance.query(
|
|
f"""
|
|
SELECT count() FROM deltaLake(
|
|
'http://{started_cluster.minio_host}:{started_cluster.minio_port}/{started_cluster.minio_bucket}/{TABLE_NAME}/',
|
|
extra_credentials(role_arn = 'arn::role', role_session_name = 'miniorole'),
|
|
SETTINGS allow_experimental_delta_kernel_rs=1)
|
|
"""
|
|
)
|
|
)
|