mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.8
295ced78d1
Follow-up to #105171, which added `ruff` with a deliberately narrow, bug-class-only rule set and left the unused-name and star-import rules off pending a dedicated cleanup pass. This enables that second batch in `pyproject.toml` and fixes every violation across the tree (the bulk in integration tests): - `F401` unused-import and `F841` unused-variable: removed, preserving any side-effecting right-hand side (e.g. `result = node.query(...)` becomes `node.query(...)`). Two `try: import ...` feature probes are kept with `# noqa: F401`. - `F541` f-string-missing-placeholders: dropped the redundant `f` prefix. - `F403`/`F405` star imports: every `from module import *` is expanded into explicit imports of the names actually used, taken from each name's real origin, so stdlib/third-party names import from their real source rather than through a test "convenience bundle". The now-empty `tests/integration/helpers/kafka/common_direct.py` bundle is removed and its three remaining consumers import `_VarintBytes`/`KafkaAdminClient` directly. Eliminating the star imports surfaced a real bug the wildcard had masked from `F821`: `generate_presets.py` in `test_prometheus_protocols` called an undefined `execute_instant_query_with_http_api`; fixed to the existing `execute_query_via_http_api`. No `black`/`isort` reformatting was applied (neither is enforced in the style check), so the diff is limited to the lint fixes themselves. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
209 lines
8.4 KiB
Python
209 lines
8.4 KiB
Python
#!/usr/bin/env python3
|
|
import time
|
|
import pytest
|
|
import os
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
@pytest.fixture(scope="module")
|
|
def started_cluster():
|
|
try:
|
|
cluster = ClickHouseCluster(__file__)
|
|
cluster.add_instance(
|
|
"node1",
|
|
main_configs=["config.xml"],
|
|
user_configs=["users.xml"],
|
|
stay_alive=True,
|
|
with_hms_catalog=True,
|
|
)
|
|
|
|
cluster.start()
|
|
time.sleep(10)
|
|
|
|
yield cluster
|
|
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
def test_hive_catalog_url_parsing(started_cluster):
|
|
node = started_cluster.instances["node1"]
|
|
|
|
password = os.environ.get('MINIO_PASSWORD', '[HIDDEN]')
|
|
|
|
test_databases = [
|
|
'test_valid_url', 'test_missing_protocol', 'test_missing_port',
|
|
'test_invalid_port', 'test_port_zero', 'test_port_too_large',
|
|
'test_empty_port', 'test_complex_path'
|
|
]
|
|
|
|
for db_name in test_databases:
|
|
node.query(f"DROP DATABASE IF EXISTS {db_name}")
|
|
|
|
try:
|
|
node.query(f"""
|
|
CREATE DATABASE test_hms_support_check ENGINE = DataLakeCatalog('thrift://hive:9083', 'minio', '{password}')
|
|
SETTINGS catalog_type = 'hive',
|
|
warehouse = 'test_warehouse',
|
|
storage_endpoint = 'http://minio1:9001/warehouse-hms/data/'
|
|
""")
|
|
node.query("DROP DATABASE IF EXISTS test_hms_support_check")
|
|
except Exception as e:
|
|
if "compiled without USE_HIVE" in str(e) or "compiled without USE_AVRO" in str(e):
|
|
pytest.skip("HMS catalog not available: ClickHouse compiled without required features")
|
|
if "Invalid URL format" in str(e):
|
|
pass
|
|
|
|
try:
|
|
node.query(f"""
|
|
CREATE DATABASE test_valid_url ENGINE = DataLakeCatalog('thrift://hive:9083', 'minio', '{password}')
|
|
SETTINGS catalog_type = 'hive',
|
|
warehouse = 'test_warehouse',
|
|
storage_endpoint = 'http://minio1:9001/warehouse-hms/data/'
|
|
""")
|
|
node.query("DROP DATABASE IF EXISTS test_valid_url")
|
|
except Exception as e:
|
|
if "Invalid URL format" in str(e):
|
|
pytest.fail("Valid URL should not fail URL parsing")
|
|
|
|
try:
|
|
node.query(f"""
|
|
CREATE DATABASE test_missing_protocol ENGINE = DataLakeCatalog('thrift:hive:9083', 'minio', '{password}')
|
|
SETTINGS catalog_type = 'hive',
|
|
warehouse = 'test_warehouse',
|
|
storage_endpoint = 'http://minio1:9001/warehouse-hms/data/'
|
|
""")
|
|
node.query("select * FROM test_missing_protocol.abc")
|
|
pytest.fail("Missing protocol separator should fail")
|
|
except Exception as e:
|
|
error_msg = str(e)
|
|
if "Invalid URL format: missing protocol separator" not in error_msg:
|
|
pytest.fail(f"Expected protocol separator error, got: {error_msg}")
|
|
finally:
|
|
node.query("DROP DATABASE IF EXISTS test_missing_protocol")
|
|
|
|
try:
|
|
node.query(f"""
|
|
CREATE DATABASE test_missing_port ENGINE = DataLakeCatalog('thrift://hive-metastore', 'minio', '{password}')
|
|
SETTINGS catalog_type = 'hive',
|
|
warehouse = 'test_warehouse',
|
|
storage_endpoint = 'http://minio1:9001/warehouse-hms/data/'
|
|
""")
|
|
node.query("select * FROM test_missing_port.abc")
|
|
pytest.fail("Missing port should fail")
|
|
except Exception as e:
|
|
error_msg = str(e)
|
|
if "Invalid URL format: missing port number" not in error_msg:
|
|
pytest.fail(f"Expected missing port error, got: {error_msg}")
|
|
finally:
|
|
node.query("DROP DATABASE IF EXISTS test_missing_port")
|
|
|
|
try:
|
|
node.query(f"""
|
|
CREATE DATABASE test_invalid_port ENGINE = DataLakeCatalog('thrift://hive-metastore:abc', 'minio', '{password}')
|
|
SETTINGS catalog_type = 'hive',
|
|
warehouse = 'test_warehouse',
|
|
storage_endpoint = 'http://minio1:9001/warehouse-hms/data/'
|
|
""")
|
|
node.query("select * FROM test_invalid_port.abc")
|
|
pytest.fail("Invalid port should fail")
|
|
except Exception as e:
|
|
error_msg = str(e)
|
|
if "Invalid port number: 'abc'" not in error_msg:
|
|
pytest.fail(f"Expected invalid port error, got: {error_msg}")
|
|
finally:
|
|
node.query("DROP DATABASE IF EXISTS test_invalid_port")
|
|
|
|
try:
|
|
node.query(f"""
|
|
CREATE DATABASE test_port_zero ENGINE = DataLakeCatalog('thrift://hive-metastore:0', 'minio', '{password}')
|
|
SETTINGS catalog_type = 'hive',
|
|
warehouse = 'test_warehouse',
|
|
storage_endpoint = 'http://minio1:9001/warehouse-hms/data/'
|
|
""")
|
|
node.query("select * FROM test_port_zero.abc")
|
|
pytest.fail("Port zero should fail")
|
|
except Exception as e:
|
|
error_msg = str(e)
|
|
if "Port number out of valid range (1-65535): 0" not in error_msg:
|
|
pytest.fail(f"Expected port range error, got: {error_msg}")
|
|
finally:
|
|
node.query("DROP DATABASE IF EXISTS test_port_zero")
|
|
|
|
try:
|
|
node.query(f"""
|
|
CREATE DATABASE test_port_too_large ENGINE = DataLakeCatalog('thrift://hive-metastore:70000', 'minio', '{password}')
|
|
SETTINGS catalog_type = 'hive',
|
|
warehouse = 'test_warehouse',
|
|
storage_endpoint = 'http://minio1:9001/warehouse-hms/data/'
|
|
""")
|
|
node.query("select * FROM test_port_too_large.abc")
|
|
pytest.fail("Port too large should fail")
|
|
except Exception as e:
|
|
error_msg = str(e)
|
|
if "Port number out of valid range (1-65535): 70000" not in error_msg:
|
|
pytest.fail(f"Expected port range error, got: {error_msg}")
|
|
finally:
|
|
node.query("DROP DATABASE IF EXISTS test_port_too_large")
|
|
|
|
try:
|
|
node.query(f"""
|
|
CREATE DATABASE test_empty_port ENGINE = DataLakeCatalog('thrift://hive-metastore:', 'minio', '{password}')
|
|
SETTINGS catalog_type = 'hive',
|
|
warehouse = 'test_warehouse',
|
|
storage_endpoint = 'http://minio1:9001/warehouse-hms/data/'
|
|
""")
|
|
node.query("select * FROM test_empty_port.abc")
|
|
pytest.fail("Empty port should fail")
|
|
except Exception as e:
|
|
error_msg = str(e)
|
|
if "Invalid port number: ''" not in error_msg:
|
|
pytest.fail(f"Expected empty port error, got: {error_msg}")
|
|
finally:
|
|
node.query("DROP DATABASE IF EXISTS test_empty_port")
|
|
|
|
try:
|
|
node.query(f"""
|
|
CREATE DATABASE test_complex_path ENGINE = DataLakeCatalog('thrift://hive-metastore:9083/metastore/db/table', 'minio', '{password}')
|
|
SETTINGS catalog_type = 'hive',
|
|
warehouse = 'test_warehouse',
|
|
storage_endpoint = 'http://minio1:9001/warehouse-hms/data/'
|
|
""")
|
|
node.query("DROP DATABASE IF EXISTS test_complex_path")
|
|
except Exception as e:
|
|
if "Invalid URL format" in str(e):
|
|
pytest.fail("Complex path URL should not fail URL parsing")
|
|
finally:
|
|
node.query("DROP DATABASE IF EXISTS test_complex_path")
|
|
|
|
|
|
def test_check_database(started_cluster):
|
|
node = started_cluster.instances["node1"]
|
|
|
|
password = os.environ.get('MINIO_PASSWORD', '[HIDDEN]')
|
|
|
|
node.query("DROP DATABASE IF EXISTS test_hms_check_db")
|
|
|
|
try:
|
|
node.query(f"""
|
|
CREATE DATABASE test_hms_check_db ENGINE = DataLakeCatalog('thrift://hive:9083', 'minio', '{password}')
|
|
SETTINGS catalog_type = 'hive',
|
|
warehouse = 'test_warehouse',
|
|
storage_endpoint = 'http://minio1:9001/warehouse-hms/data/'
|
|
""")
|
|
node.query("CHECK DATABASE test_hms_check_db")
|
|
|
|
node.query(
|
|
"SYSTEM ENABLE FAILPOINT check_database_datalake_negative"
|
|
)
|
|
|
|
assert "fault when checking database" in node.query_and_get_error(
|
|
"CHECK DATABASE test_hms_check_db"
|
|
)
|
|
except Exception as e:
|
|
if "compiled without USE_HIVE" in str(e) or "compiled without USE_AVRO" in str(e):
|
|
pytest.skip("HMS catalog not available: ClickHouse compiled without required features")
|
|
if "Invalid URL format" in str(e):
|
|
pass
|
|
finally:
|
|
node.query("SYSTEM DISABLE FAILPOINT check_database_datalake_negative")
|
|
node.query("DROP DATABASE IF EXISTS test_hms_check_db")
|