mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Raúl MarínandClaude Opus 4.8
0a2022fdf7
Add a session setting `s3_allow_server_credentials_in_user_queries` (default off in clickhouse-server, no effect in clickhouse-local). When off, S3 access driven by user SQL cannot resolve the server's own ambient credentials (environment, IMDS, IRSA, `role_arn` STS, AWS config files, or the GCP OAuth metadata service) at the single credential-resolution chokepoint; a request with no explicit keys and no NOSIGN is refused with ACCESS_DENIED. Covers the `s3`/`s3Cluster` functions, the `S3`/`S3Queue` engines, named collections, dynamic S3 disks, BACKUP/RESTORE TO S3, DataLake reads, and Glue/BigLake catalogs. A constraint can pin it for untrusted users. Operator- provisioned static credentials in config are intentionally out of scope. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
108 lines
3.2 KiB
Python
108 lines
3.2 KiB
Python
import os
|
|
|
|
import pytest
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
from helpers.mock_servers import start_mock_servers
|
|
|
|
METADATA_SERVER_HOSTNAME = "resolver"
|
|
METADATA_SERVER_PORT = 8080
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
|
|
node = cluster.add_instance(
|
|
"node",
|
|
with_minio=True,
|
|
main_configs=["configs/use_environment_credentials.xml"],
|
|
user_configs=["configs/allow_server_credentials.xml"],
|
|
env_variables={
|
|
"AWS_EC2_METADATA_SERVICE_ENDPOINT": f"{METADATA_SERVER_HOSTNAME}:{METADATA_SERVER_PORT}",
|
|
},
|
|
)
|
|
|
|
|
|
def start_metadata_server():
|
|
script_dir = os.path.join(os.path.dirname(__file__), "metadata_servers")
|
|
start_mock_servers(
|
|
cluster,
|
|
script_dir,
|
|
[
|
|
(
|
|
"simple_server.py",
|
|
METADATA_SERVER_HOSTNAME,
|
|
METADATA_SERVER_PORT,
|
|
)
|
|
],
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="module", autouse=True)
|
|
def start_cluster():
|
|
try:
|
|
cluster.start()
|
|
start_metadata_server()
|
|
yield
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def test_credentials_from_metadata():
|
|
node.query(
|
|
f"INSERT INTO FUNCTION s3('http://{cluster.minio_host}:{cluster.minio_port}/{cluster.minio_bucket}/test1.jsonl') SELECT * FROM numbers(100) SETTINGS s3_truncate_on_insert = 1"
|
|
)
|
|
|
|
assert (
|
|
"100"
|
|
== node.query(
|
|
f"SELECT count() FROM s3('http://{cluster.minio_host}:{cluster.minio_port}/{cluster.minio_bucket}/test1.jsonl')"
|
|
).strip()
|
|
)
|
|
|
|
expected_logs = [
|
|
"Calling EC2MetadataService to get token failed, falling back to a less secure way",
|
|
"Getting default credentials for ec2 instance from resolver:8080",
|
|
"Calling EC2MetadataService resource, /latest/meta-data/iam/security-credentials returned credential string myrole",
|
|
"Calling EC2MetadataService resource /latest/meta-data/iam/security-credentials/myrole",
|
|
"Successfully pulled credentials from EC2MetadataService with access key",
|
|
]
|
|
|
|
node.query("SYSTEM FLUSH LOGS")
|
|
for expected_msg in expected_logs:
|
|
assert node.contains_in_log(
|
|
"AWSEC2InstanceProfileConfigLoader: " + expected_msg
|
|
)
|
|
|
|
|
|
def test_credentials_provider_caching():
|
|
node.query(
|
|
f"INSERT INTO FUNCTION s3('http://{cluster.minio_host}:{cluster.minio_port}/{cluster.minio_bucket}/test_caching.jsonl') SELECT * FROM numbers(10) SETTINGS s3_truncate_on_insert = 1"
|
|
)
|
|
|
|
for _ in range(10):
|
|
result = node.query(
|
|
f"SELECT count() FROM s3('http://{cluster.minio_host}:{cluster.minio_port}/{cluster.minio_bucket}/test_caching.jsonl')"
|
|
).strip()
|
|
assert result == "10"
|
|
|
|
added = int(
|
|
node.query(
|
|
"SELECT value FROM system.events WHERE event = 'S3CachedCredentialsProvidersAdded'"
|
|
).strip() or "0"
|
|
)
|
|
reused = int(
|
|
node.query(
|
|
"SELECT value FROM system.events WHERE event = 'S3CachedCredentialsProvidersReused'"
|
|
).strip() or "0"
|
|
)
|
|
|
|
assert added > 0
|
|
assert reused > 0
|
|
assert reused >= added
|
|
|
|
current_cached = int(
|
|
node.query(
|
|
"SELECT value FROM system.metrics WHERE name = 'S3CachedCredentialsProviders'"
|
|
).strip() or "0"
|
|
)
|
|
assert current_cached > 0
|