mirror of
https://github.com/hanzo-ds/driver.git
synced 2026-10-30 20:32:40 +00:00
Kim Gustyr
11a09af20d
All numbers on one machine and environment, attributed in the docs: Apple M2 Pro, ClickHouse 25.12.11.4 in Docker, Python 3.11.11. Methodology change: scripts end their imports with 'import timing' (new perf/timing.py) which starts the clock and reports wall time and max RSS at exit. Interpreter startup and imports are excluded from timings; importing the numpy/pyarrow stack costs ~0.5 s and dominated sub-second cells under whole-process timing. New typed-projection PyArrow scripts (21-24) cover analytic extracts: int and string column subsets, clickhouse-driver vs clickhouse-connect. All perf scripts carry PEP 723 blocks with pinned versions. Scripts 15-17 updated to the current clickhouse-connect streaming API. beep boop
31 lines
1.3 KiB
Python
31 lines
1.3 KiB
Python
# /// script
|
|
# dependencies = [
|
|
# "clickhouse-driver[arrow,numpy]==0.2.11",
|
|
# ]
|
|
# ///
|
|
import sys
|
|
from clickhouse_driver import Client
|
|
|
|
# Imported lazily at query time: preload before the clock starts.
|
|
import numpy
|
|
import pandas
|
|
import pyarrow
|
|
|
|
import timing
|
|
|
|
cols = [
|
|
'Year', 'Quarter', 'Month', 'DayofMonth', 'DayOfWeek', 'AirlineID', 'OriginAirportID', 'OriginAirportSeqID',
|
|
'OriginCityMarketID', 'OriginWac', 'DestAirportID', 'DestAirportSeqID', 'DestCityMarketID', 'DestWac',
|
|
'CRSDepTime', 'DepTime', 'DepDelay', 'DepDelayMinutes', 'DepDel15', 'TaxiOut', 'WheelsOff', 'WheelsOn',
|
|
'TaxiIn', 'CRSArrTime', 'ArrTime', 'ArrDelay', 'ArrDelayMinutes', 'ArrDel15', 'ArrivalDelayGroups',
|
|
'Cancelled', 'Diverted', 'CRSElapsedTime', 'ActualElapsedTime', 'AirTime', 'Flights', 'Distance',
|
|
'DistanceGroup', 'CarrierDelay', 'WeatherDelay', 'NASDelay', 'SecurityDelay', 'LateAircraftDelay',
|
|
'Div1AirportID', 'Div1AirportSeqID', 'Div2AirportID', 'Div2AirportSeqID', 'Div3AirportID',
|
|
'Div3AirportSeqID', 'Div4AirportID', 'Div4AirportSeqID', 'Div5AirportID', 'Div5AirportSeqID'
|
|
]
|
|
|
|
query = "SELECT {} FROM perftest.ontime WHERE FlightDate < '{}'".format(', '.join(cols), sys.argv[1])
|
|
client = Client('localhost', settings={'use_numpy': True})
|
|
|
|
table = client.query_arrow(query)
|