mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Groene AIandClaude Opus 4.8
5381e15b35
Follow-up to #106189. The Remote/RemoteSecure storage engines build a cluster on the fly from user-supplied addresses via the same parseRemoteFunctionArguments as the remote/remoteSecure table functions, which calls RemoteHostFilter::checkHostAndPort for every parsed host. For the engines this runs during CREATE, so a host outside <remote_url_allow_hosts> is rejected with UNACCEPTABLE_URL before the table is created. This adds a regression test that locks in that behavior for both Remote and RemoteSecure (disallowed bare host, glob expanding to a disallowed host, disallowed port; allowed host succeeds). Co-Authored-By: Claude Opus 4.8 <[email protected]>
354 lines
13 KiB
Python
354 lines
13 KiB
Python
import pytest
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
node1 = cluster.add_instance("node1", main_configs=["configs/config_with_hosts.xml"])
|
|
node2 = cluster.add_instance(
|
|
"node2", main_configs=["configs/config_with_only_primary_hosts.xml"]
|
|
)
|
|
node3 = cluster.add_instance(
|
|
"node3", main_configs=["configs/config_with_only_regexp_hosts.xml"]
|
|
)
|
|
node4 = cluster.add_instance(
|
|
"node4",
|
|
main_configs=[],
|
|
user_configs=["configs/allow_server_credentials.xml"],
|
|
) # No `remote_url_allow_hosts` at all.
|
|
node5 = cluster.add_instance(
|
|
"node5",
|
|
main_configs=["configs/config_without_allowed_hosts.xml"],
|
|
user_configs=["configs/allow_server_credentials.xml"],
|
|
)
|
|
node6 = cluster.add_instance("node6", main_configs=["configs/config_for_remote.xml"])
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def start_cluster():
|
|
try:
|
|
cluster.start()
|
|
yield cluster
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def test_config_with_hosts(start_cluster):
|
|
assert (
|
|
node1.query(
|
|
"CREATE TABLE table_test_1_1 (word String) Engine=URL('http://host:80', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
assert (
|
|
node1.query(
|
|
"CREATE TABLE table_test_1_2 (word String) Engine=URL('https://yandex.ru', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
assert "not allowed" in node1.query_and_get_error(
|
|
"CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', CSV)"
|
|
)
|
|
assert "not allowed" in node1.query_and_get_error(
|
|
"CREATE TABLE table_test_1_4 (word String) Engine=URL('https://yandex2.ru', CSV)"
|
|
)
|
|
node1.query("DROP TABLE table_test_1_1")
|
|
node1.query("DROP TABLE table_test_1_2")
|
|
|
|
|
|
def test_config_with_only_primary_hosts(start_cluster):
|
|
assert (
|
|
node2.query(
|
|
"CREATE TABLE table_test_2_1 (word String) Engine=URL('https://host:80', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
assert (
|
|
node2.query(
|
|
"CREATE TABLE table_test_2_2 (word String) Engine=URL('https://host:123', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
assert (
|
|
node2.query(
|
|
"CREATE TABLE table_test_2_3 (word String) Engine=URL('https://yandex.ru', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
assert (
|
|
node2.query(
|
|
"CREATE TABLE table_test_2_4 (word String) Engine=URL('https://yandex.ru:87', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
assert "not allowed" in node2.query_and_get_error(
|
|
"CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', CSV)"
|
|
)
|
|
assert "not allowed" in node2.query_and_get_error(
|
|
"CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host:234', CSV)"
|
|
)
|
|
assert "not allowed" in node2.query_and_get_error(
|
|
"CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', CSV)"
|
|
)
|
|
|
|
node2.query("DROP TABLE table_test_2_1")
|
|
node2.query("DROP TABLE table_test_2_2")
|
|
node2.query("DROP TABLE table_test_2_3")
|
|
node2.query("DROP TABLE table_test_2_4")
|
|
|
|
|
|
def test_config_with_only_regexp_hosts(start_cluster):
|
|
assert (
|
|
node3.query(
|
|
"CREATE TABLE table_test_3_1 (word String) Engine=URL('https://host:80', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
assert (
|
|
node3.query(
|
|
"CREATE TABLE table_test_3_2 (word String) Engine=URL('https://yandex.ru', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
assert "not allowed" in node3.query_and_get_error(
|
|
"CREATE TABLE table_test_3_3 (word String) Engine=URL('https://host', CSV)"
|
|
)
|
|
assert "not allowed" in node3.query_and_get_error(
|
|
"CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', CSV)"
|
|
)
|
|
node3.query("DROP TABLE table_test_3_1")
|
|
node3.query("DROP TABLE table_test_3_2")
|
|
|
|
|
|
def test_config_without_allowed_hosts_section(start_cluster):
|
|
assert (
|
|
node4.query(
|
|
"CREATE TABLE table_test_4_1 (word String) Engine=URL('https://host:80', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
assert (
|
|
node4.query(
|
|
"CREATE TABLE table_test_4_2 (word String) Engine=S3('https://host:80/bucket/key', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
assert (
|
|
node4.query(
|
|
"CREATE TABLE table_test_4_3 (word String) Engine=URL('https://host', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
assert (
|
|
node4.query(
|
|
"CREATE TABLE table_test_4_4 (word String) Engine=URL('https://yandex.ru', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
assert (
|
|
node4.query(
|
|
"CREATE TABLE table_test_4_5 (word String) Engine=URL('ftp://something.com', CSV)"
|
|
)
|
|
== ""
|
|
)
|
|
node4.query("DROP TABLE table_test_4_1")
|
|
node4.query("DROP TABLE table_test_4_2")
|
|
node4.query("DROP TABLE table_test_4_3")
|
|
node4.query("DROP TABLE table_test_4_4")
|
|
node4.query("DROP TABLE table_test_4_5")
|
|
|
|
|
|
def test_config_without_allowed_hosts(start_cluster):
|
|
assert "not allowed" in node5.query_and_get_error(
|
|
"CREATE TABLE table_test_5_1 (word String) Engine=URL('https://host:80', CSV)"
|
|
)
|
|
assert "not allowed" in node5.query_and_get_error(
|
|
"CREATE TABLE table_test_5_2 (word String) Engine=S3('https://host:80/bucket/key', CSV)"
|
|
)
|
|
assert "not allowed" in node5.query_and_get_error(
|
|
"CREATE TABLE table_test_5_3 (word String) Engine=URL('https://host', CSV)"
|
|
)
|
|
assert "not allowed" in node5.query_and_get_error(
|
|
"CREATE TABLE table_test_5_4 (word String) Engine=URL('https://yandex.ru', CSV)"
|
|
)
|
|
assert "not allowed" in node5.query_and_get_error(
|
|
"CREATE TABLE table_test_5_5 (word String) Engine=URL('ftp://something.com', CSV)"
|
|
)
|
|
|
|
|
|
def test_mysql_dictionary_host_filter(start_cluster):
|
|
"""MySQL dictionary source should respect RemoteHostFilter for inline DDL params."""
|
|
# node5 has an empty <remote_url_allow_hosts> section, so all hosts are denied.
|
|
# CREATE DICTIONARY succeeds because dictionary loading is async.
|
|
# The host filter check runs in the factory lambda at load time, before
|
|
# any TCP connection is attempted, so no running MySQL server is required.
|
|
node5.query("DROP DICTIONARY IF EXISTS default.test_mysql_host_filter")
|
|
node5.query(
|
|
"""
|
|
CREATE DICTIONARY default.test_mysql_host_filter (id UInt64, val String)
|
|
PRIMARY KEY id
|
|
SOURCE(MYSQL(HOST '10.0.0.1' PORT 3306 DB 'test' TABLE 'tbl'
|
|
USER 'root' PASSWORD 'pass'))
|
|
LAYOUT(FLAT()) LIFETIME(MIN 0 MAX 1)
|
|
"""
|
|
)
|
|
# dictGet forces synchronous loading; the host filter check should throw "not allowed"
|
|
# before any MySQL TCP connection is attempted.
|
|
error = node5.query_and_get_error(
|
|
"SELECT dictGet('default.test_mysql_host_filter', 'val', toUInt64(1))"
|
|
)
|
|
assert "not allowed" in error, f"Expected host filter error, got: {error}"
|
|
node5.query("DROP DICTIONARY IF EXISTS default.test_mysql_host_filter")
|
|
|
|
|
|
def test_mysql_dictionary_replica_host_filter(start_cluster):
|
|
"""MySQL dictionary source with REPLICA(...) syntax should respect RemoteHostFilter."""
|
|
# Covers the replica branch: if (config.has(settings_config_prefix + ".replica"))
|
|
node5.query("DROP DICTIONARY IF EXISTS default.test_mysql_replica_filter")
|
|
node5.query(
|
|
"""
|
|
CREATE DICTIONARY default.test_mysql_replica_filter (id UInt64, val String)
|
|
PRIMARY KEY id
|
|
SOURCE(MYSQL(
|
|
USER 'root' PASSWORD 'pass' DB 'test' TABLE 'tbl'
|
|
REPLICA(PRIORITY 1 HOST '10.0.0.1' PORT 3306)
|
|
REPLICA(PRIORITY 2 HOST '10.0.0.2' PORT 3307)
|
|
))
|
|
LAYOUT(FLAT()) LIFETIME(MIN 0 MAX 1)
|
|
"""
|
|
)
|
|
error = node5.query_and_get_error(
|
|
"SELECT dictGet('default.test_mysql_replica_filter', 'val', toUInt64(1))"
|
|
)
|
|
assert "not allowed" in error, f"Expected host filter error, got: {error}"
|
|
node5.query("DROP DICTIONARY IF EXISTS default.test_mysql_replica_filter")
|
|
|
|
|
|
def test_table_function_remote(start_cluster):
|
|
assert "not allowed in configuration file" not in node6.query_and_get_error(
|
|
"SELECT * FROM remoteSecure('example01-01-{1|2}', system, events)",
|
|
settings={
|
|
"connections_with_failover_max_tries": 1,
|
|
"connect_timeout_with_failover_ms": 1000,
|
|
"connect_timeout_with_failover_secure_ms": 1000,
|
|
"connect_timeout": 1,
|
|
"send_timeout": 1,
|
|
},
|
|
)
|
|
assert "not allowed in configuration file" not in node6.query_and_get_error(
|
|
"SELECT * FROM remoteSecure('example01-01-1,example01-02-1', system, events)",
|
|
settings={
|
|
"connections_with_failover_max_tries": 1,
|
|
"connect_timeout_with_failover_ms": 1000,
|
|
"connect_timeout_with_failover_secure_ms": 1000,
|
|
"connect_timeout": 1,
|
|
"send_timeout": 1,
|
|
},
|
|
)
|
|
assert "not allowed in configuration file" not in node6.query_and_get_error(
|
|
"SELECT * FROM remote('example01-0{1,2}-1', system, events",
|
|
settings={
|
|
"connections_with_failover_max_tries": 1,
|
|
"connect_timeout_with_failover_ms": 1000,
|
|
"connect_timeout_with_failover_secure_ms": 1000,
|
|
"connect_timeout": 1,
|
|
"send_timeout": 1,
|
|
},
|
|
)
|
|
assert "not allowed in configuration file" not in node6.query_and_get_error(
|
|
"SELECT * FROM remote('example01-0{1,2}-{1|2}', system, events)",
|
|
settings={
|
|
"connections_with_failover_max_tries": 1,
|
|
"connect_timeout_with_failover_ms": 1000,
|
|
"connect_timeout_with_failover_secure_ms": 1000,
|
|
"connect_timeout": 1,
|
|
"send_timeout": 1,
|
|
},
|
|
)
|
|
assert "not allowed in configuration file" not in node6.query_and_get_error(
|
|
"SELECT * FROM remoteSecure('example01-{01..02}-{1|2}', system, events)",
|
|
settings={
|
|
"connections_with_failover_max_tries": 1,
|
|
"connect_timeout_with_failover_ms": 1000,
|
|
"connect_timeout_with_failover_secure_ms": 1000,
|
|
"connect_timeout": 1,
|
|
"send_timeout": 1,
|
|
},
|
|
)
|
|
assert "not allowed" in node6.query_and_get_error(
|
|
"SELECT * FROM remoteSecure('example01-01-1,example01-03-1', system, events)",
|
|
settings={
|
|
"connections_with_failover_max_tries": 1,
|
|
"connect_timeout_with_failover_ms": 1000,
|
|
"connect_timeout_with_failover_secure_ms": 1000,
|
|
"connect_timeout": 1,
|
|
"send_timeout": 1,
|
|
},
|
|
)
|
|
assert "not allowed" in node6.query_and_get_error(
|
|
"SELECT * FROM remote('example01-01-{1|3}', system, events)",
|
|
settings={
|
|
"connections_with_failover_max_tries": 1,
|
|
"connect_timeout_with_failover_ms": 1000,
|
|
"connect_timeout_with_failover_secure_ms": 1000,
|
|
"connect_timeout": 1,
|
|
"send_timeout": 1,
|
|
},
|
|
)
|
|
assert "not allowed" in node6.query_and_get_error(
|
|
"SELECT * FROM remoteSecure('example01-0{1,3}-1', system, metrics)",
|
|
settings={
|
|
"connections_with_failover_max_tries": 1,
|
|
"connect_timeout_with_failover_ms": 1000,
|
|
"connect_timeout_with_failover_secure_ms": 1000,
|
|
"connect_timeout": 1,
|
|
"send_timeout": 1,
|
|
},
|
|
)
|
|
assert node6.query("SELECT * FROM remote('localhost', system, events)") != ""
|
|
assert node6.query("SELECT * FROM remoteSecure('localhost', system, metrics)") != ""
|
|
assert (
|
|
'URL "localhost:800" is not allowed in configuration file'
|
|
in node6.query_and_get_error(
|
|
"SELECT * FROM remoteSecure('localhost:800', system, events)"
|
|
)
|
|
)
|
|
assert (
|
|
'URL "localhost:800" is not allowed in configuration file'
|
|
in node6.query_and_get_error(
|
|
"SELECT * FROM remote('localhost:800', system, metrics)"
|
|
)
|
|
)
|
|
|
|
|
|
def test_storage_engine_remote(start_cluster):
|
|
# The persistent Remote/RemoteSecure engines must enforce remote_url_allow_hosts, like the
|
|
# remote/remoteSecure table functions: the check runs at CREATE time, before any connection.
|
|
for engine in ["Remote", "RemoteSecure"]:
|
|
# Disallowed host: rejected with UNACCEPTABLE_URL.
|
|
assert "not allowed in configuration file" in node6.query_and_get_error(
|
|
f"CREATE TABLE test_remote_engine (dummy UInt8) ENGINE = {engine}('example01-01-3', system, one)"
|
|
)
|
|
# A glob that expands to an allowed and a disallowed host is rejected: every expanded host
|
|
# is checked, not only the first one.
|
|
assert "not allowed in configuration file" in node6.query_and_get_error(
|
|
f"CREATE TABLE test_remote_engine (dummy UInt8) ENGINE = {engine}('example01-01-{{1,3}}', system, one)"
|
|
)
|
|
# Allowed host, but a disallowed explicit port is still rejected.
|
|
assert (
|
|
'URL "localhost:800" is not allowed in configuration file'
|
|
in node6.query_and_get_error(
|
|
f"CREATE TABLE test_remote_engine (dummy UInt8) ENGINE = {engine}('localhost:800', system, one)"
|
|
)
|
|
)
|
|
# Allowed host: the table is created (system.tables reports it as Distributed).
|
|
node6.query(
|
|
f"CREATE TABLE test_remote_engine (dummy UInt8) ENGINE = {engine}('example01-01-1', system, one)"
|
|
)
|
|
assert (
|
|
node6.query(
|
|
"SELECT engine FROM system.tables WHERE database = currentDatabase() AND name = 'test_remote_engine'"
|
|
).strip()
|
|
== "Distributed"
|
|
)
|
|
node6.query("DROP TABLE test_remote_engine")
|