mirror of
https://github.com/hanzo-ds/driver.git
synced 2026-10-30 20:32:40 +00:00
Kim Gustyr
11a09af20d
All numbers on one machine and environment, attributed in the docs: Apple M2 Pro, ClickHouse 25.12.11.4 in Docker, Python 3.11.11. Methodology change: scripts end their imports with 'import timing' (new perf/timing.py) which starts the clock and reports wall time and max RSS at exit. Interpreter startup and imports are excluded from timings; importing the numpy/pyarrow stack costs ~0.5 s and dominated sub-second cells under whole-process timing. New typed-projection PyArrow scripts (21-24) cover analytic extracts: int and string column subsets, clickhouse-driver vs clickhouse-connect. All perf scripts carry PEP 723 blocks with pinned versions. Scripts 15-17 updated to the current clickhouse-connect streaming API. beep boop
33 lines
1.4 KiB
Python
33 lines
1.4 KiB
Python
# /// script
|
|
# dependencies = [
|
|
# "clickhouse-driver[arrow,numpy]==0.2.11",
|
|
# ]
|
|
# ///
|
|
import sys
|
|
from clickhouse_driver import Client
|
|
|
|
# Imported lazily at query time: preload before the clock starts.
|
|
import numpy
|
|
import pandas
|
|
import pyarrow
|
|
|
|
import timing
|
|
|
|
cols = [
|
|
'UniqueCarrier', 'Carrier', 'TailNum', 'FlightNum', 'Origin', 'OriginCityName', 'OriginState',
|
|
'OriginStateFips', 'OriginStateName', 'Dest', 'DestCityName', 'DestState', 'DestStateFips',
|
|
'DestStateName', 'DepartureDelayGroups', 'DepTimeBlk', 'ArrTimeBlk', 'CancellationCode',
|
|
'FirstDepTime', 'TotalAddGTime', 'LongestAddGTime', 'DivAirportLandings', 'DivReachedDest',
|
|
'DivActualElapsedTime', 'DivArrDelay', 'DivDistance', 'Div1Airport', 'Div1WheelsOn', 'Div1TotalGTime',
|
|
'Div1LongestGTime', 'Div1WheelsOff', 'Div1TailNum', 'Div2Airport', 'Div2WheelsOn', 'Div2TotalGTime',
|
|
'Div2LongestGTime', 'Div2WheelsOff', 'Div2TailNum', 'Div3Airport', 'Div3WheelsOn', 'Div3TotalGTime',
|
|
'Div3LongestGTime', 'Div3WheelsOff', 'Div3TailNum', 'Div4Airport', 'Div4WheelsOn', 'Div4TotalGTime',
|
|
'Div4LongestGTime', 'Div4WheelsOff', 'Div4TailNum', 'Div5Airport', 'Div5WheelsOn', 'Div5TotalGTime',
|
|
'Div5LongestGTime', 'Div5WheelsOff', 'Div5TailNum'
|
|
]
|
|
|
|
query = "SELECT {} FROM perftest.ontime WHERE FlightDate < '{}'".format(', '.join(cols), sys.argv[1])
|
|
client = Client('localhost', settings={'use_numpy': True})
|
|
|
|
table = client.query_arrow(query)
|