mirror of
https://github.com/hanzo-ds/driver.git
synced 2026-10-30 20:32:40 +00:00
Kim Gustyr
11a09af20d
All numbers on one machine and environment, attributed in the docs: Apple M2 Pro, ClickHouse 25.12.11.4 in Docker, Python 3.11.11. Methodology change: scripts end their imports with 'import timing' (new perf/timing.py) which starts the clock and reports wall time and max RSS at exit. Interpreter startup and imports are excluded from timings; importing the numpy/pyarrow stack costs ~0.5 s and dominated sub-second cells under whole-process timing. New typed-projection PyArrow scripts (21-24) cover analytic extracts: int and string column subsets, clickhouse-driver vs clickhouse-connect. All perf scripts carry PEP 723 blocks with pinned versions. Scripts 15-17 updated to the current clickhouse-connect streaming API. beep boop
37 lines
1.0 KiB
Python
37 lines
1.0 KiB
Python
# /// script
|
|
# dependencies = [
|
|
# "requests==2.34.2",
|
|
# ]
|
|
# ///
|
|
from datetime import date
|
|
import sys
|
|
import requests
|
|
|
|
import timing
|
|
|
|
|
|
def get_python_type(ch_type):
|
|
if ch_type.startswith('Int') or ch_type.startswith('UInt'):
|
|
return int
|
|
|
|
elif ch_type == 'String' or ch_type.startswith('FixedString'):
|
|
return None
|
|
|
|
elif ch_type == 'Date':
|
|
return lambda value: date(*[int(x) for x in value.split('-')])
|
|
|
|
raise ValueError(f'Unsupported type: "{ch_type}"')
|
|
|
|
|
|
resp = requests.get('http://localhost:8123', params={'query': 'describe table perftest.ontime FORMAT TSV'})
|
|
ch_types = [x.split('\t')[1] for x in resp.text.split('\n') if x]
|
|
python_types = [get_python_type(x) for x in ch_types]
|
|
|
|
query = "SELECT * FROM perftest.ontime WHERE FlightDate < '{}' FORMAT TSV".format(sys.argv[1])
|
|
resp = requests.get('http://localhost:8123/', stream=True, params={'query': query})
|
|
|
|
data = []
|
|
|
|
for line in resp.iter_lines(chunk_size=10000):
|
|
data.append([cls(x) if cls else x for x, cls in zip(line.decode('utf-8').split('\t'), python_types)])
|