mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.6
92b04b3b12
Remove the old Arrow-based Parquet writer (used when `output_format_parquet_use_custom_encoder` = 0) and the old Arrow-based Parquet reader (used when `input_format_parquet_use_native_reader_v3` = 0). Both newer implementations have been the default for an extended period and perform reliably. The custom encoder has been default since 24.5, and the native reader v3 since 25.11. Changes: - Mark `output_format_parquet_use_custom_encoder` and `input_format_parquet_use_native_reader_v3` as obsolete settings - Remove the `writeUsingArrow` code path and Arrow library dependencies from `ParquetBlockOutputFormat` - Remove the entire `ParquetBlockInputFormat` class (old Arrow-based reader) and `ArrowParquetSchemaReader` - Simplify format registration to always use `ParquetV3BlockInputFormat` and `NativeParquetSchemaReader` - PREWHERE support checker for Parquet now always returns true - Update ~50 test files to remove references to obsolete settings - Update documentation Closes https://github.com/ClickHouse/ClickHouse/issues/100126 Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
103 lines
3.1 KiB
Python
103 lines
3.1 KiB
Python
import os
|
|
import time
|
|
|
|
import pyarrow.parquet as pq
|
|
import pytest
|
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
path_to_userfiles = "/var/lib/clickhouse/user_files/"
|
|
node = cluster.add_instance("node", external_dirs=[path_to_userfiles])
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def start_cluster():
|
|
try:
|
|
cluster.start()
|
|
yield cluster
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
def check_page_index(file_path):
|
|
metadata = pq.read_metadata(file_path)
|
|
assert (
|
|
metadata
|
|
), "pyarrow.parquet library can't read parquet file written by Clickhouse"
|
|
return metadata.row_group(0).column(0).has_offset_index
|
|
|
|
|
|
def delete_if_exists(file_path):
|
|
if os.path.exists(file_path):
|
|
os.remove(file_path)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"query, expected_result",
|
|
[
|
|
(
|
|
"SELECT number, number+1 FROM system.numbers LIMIT 100 "
|
|
"INTO OUTFILE '{file_name}' FORMAT Parquet "
|
|
"SETTINGS output_format_parquet_write_page_index = true;",
|
|
True,
|
|
),
|
|
(
|
|
"SELECT number, number+1 FROM system.numbers LIMIT 100 "
|
|
"INTO OUTFILE '{file_name}' FORMAT Parquet "
|
|
"SETTINGS output_format_parquet_write_page_index = false;",
|
|
False,
|
|
),
|
|
(
|
|
"SELECT number, number+1 FROM system.numbers LIMIT 100 "
|
|
"INTO OUTFILE '{file_name}' FORMAT Parquet;",
|
|
True,
|
|
),
|
|
],
|
|
)
|
|
def test_parquet_page_index_select_into_outfile(query, expected_result, start_cluster):
|
|
file_name = f"export{time.time()}.parquet"
|
|
query = query.format(file_name=file_name)
|
|
delete_if_exists(file_name)
|
|
assert node.query(query) == ""
|
|
assert (
|
|
check_page_index(file_name) == expected_result
|
|
), "Page offset index have wrong value"
|
|
delete_if_exists(file_name)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"query, expected_result",
|
|
[
|
|
(
|
|
"INSERT INTO TABLE FUNCTION file('{file_name}') "
|
|
"SELECT number, number+1 FROM system.numbers LIMIT 100 "
|
|
"SETTINGS output_format_parquet_write_page_index=true FORMAT Parquet",
|
|
True,
|
|
),
|
|
(
|
|
"INSERT INTO TABLE FUNCTION file('{file_name}') "
|
|
"SELECT number, number+1 FROM system.numbers LIMIT 100 "
|
|
"SETTINGS output_format_parquet_write_page_index=false FORMAT Parquet",
|
|
False,
|
|
),
|
|
(
|
|
"INSERT INTO TABLE FUNCTION file('{file_name}') "
|
|
"SELECT number, number+1 FROM system.numbers LIMIT 100 FORMAT Parquet",
|
|
True,
|
|
),
|
|
],
|
|
)
|
|
def test_parquet_page_index_insert_into_table_function_file(
|
|
query, expected_result, start_cluster
|
|
):
|
|
file_name = f"export{time.time()}.parquet"
|
|
query = query.format(file_name=file_name)
|
|
file_path = f"{cluster.instances_dir}{path_to_userfiles}{file_name}"
|
|
delete_if_exists(file_path)
|
|
assert node.query(query) == ""
|
|
assert (
|
|
check_page_index(file_path) == expected_result
|
|
), "Page offset index have wrong value"
|
|
delete_if_exists(file_path)
|