mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
fastio
771fb179ef
Store vectors at the target length instead of building them per query with arrayResize/materialize, whose allocation and copy dominated the previous wall-clock time and masked the kernel cost. Cover all five metrics on the native and mixed-type, const and non-const paths.
106 lines
4.5 KiB
XML
106 lines
4.5 KiB
XML
<test>
|
|
<!--
|
|
This benchmark isolates the distance *kernel* (the hot SIMD loop), which is
|
|
what PR #101310 optimizes. The earlier version computed
|
|
L2Distance(arrayResize(v, N), arrayResize(materialize(v), N))
|
|
whose wall-clock time was dominated by the per-query allocation/copy of the
|
|
temporary Array columns produced by `arrayResize`/`materialize`, not by the
|
|
kernel - so a faster kernel could not move the measured number.
|
|
|
|
Here the arrays are stored already at the target length (one table per size),
|
|
and the measured query calls the distance function directly. No `arrayResize`,
|
|
no `materialize`, no wrapping subquery - so the kernel is the dominant cost.
|
|
-->
|
|
<substitutions>
|
|
<substitution>
|
|
<name>array_size</name>
|
|
<values>
|
|
<value>256</value>
|
|
<value>1024</value>
|
|
<value>4096</value>
|
|
</values>
|
|
</substitution>
|
|
</substitutions>
|
|
|
|
<settings>
|
|
<max_threads>1</max_threads>
|
|
</settings>
|
|
|
|
<!-- Two distinct Float32 vectors per row (so cosine is well-defined). -->
|
|
<create_query>
|
|
CREATE TABLE array_distance_f32_{array_size}
|
|
(
|
|
id UInt64,
|
|
a Array(Float32),
|
|
b Array(Float32)
|
|
)
|
|
ENGINE = Memory
|
|
</create_query>
|
|
|
|
<!-- Quantised (UInt8) storage vector against a Float64 query vector: the
|
|
canonical mixed-type ANN layout that PR #101310 adds SIMD support for. -->
|
|
<create_query>
|
|
CREATE TABLE array_distance_mixed_{array_size}
|
|
(
|
|
id UInt64,
|
|
u8 Array(UInt8),
|
|
f64 Array(Float64)
|
|
)
|
|
ENGINE = Memory
|
|
</create_query>
|
|
|
|
<fill_query>
|
|
INSERT INTO array_distance_f32_{array_size}
|
|
SELECT
|
|
number,
|
|
arrayMap(i -> toFloat32((number * 9 + i) % 1000) * 0.001, range({array_size})),
|
|
arrayMap(i -> toFloat32((number * 7 + i + 13) % 1000) * 0.001, range({array_size}))
|
|
FROM numbers(100000)
|
|
</fill_query>
|
|
|
|
<fill_query>
|
|
INSERT INTO array_distance_mixed_{array_size}
|
|
SELECT
|
|
number,
|
|
arrayMap(i -> toUInt8((number + i) % 256), range({array_size})),
|
|
arrayMap(i -> toFloat64((number * 3 + i) % 997) * 0.5, range({array_size}))
|
|
FROM numbers(100000)
|
|
</fill_query>
|
|
|
|
<!-- ===== Native Float32, non-const x non-const path (all five metrics) ===== -->
|
|
<query>SELECT sum(L1Distance(a, b)) FROM array_distance_f32_{array_size}</query>
|
|
<query>SELECT sum(L2Distance(a, b)) FROM array_distance_f32_{array_size}</query>
|
|
<query>SELECT sum(L2SquaredDistance(a, b)) FROM array_distance_f32_{array_size}</query>
|
|
<query>SELECT sum(LinfDistance(a, b)) FROM array_distance_f32_{array_size}</query>
|
|
<query>SELECT sum(cosineDistance(a, b)) FROM array_distance_f32_{array_size}</query>
|
|
|
|
<!-- ===== Native Float32, const x non-const path =====
|
|
The needle is a single cached vector reused for every row. For L2/cosine this
|
|
exercises the hand-written AVX-512 intrinsic path (already present on master),
|
|
so these queries are expected to stay flat - they are the baseline reference. -->
|
|
<query>
|
|
WITH (SELECT a FROM array_distance_f32_{array_size} LIMIT 1) AS needle
|
|
SELECT sum(L2Distance(needle, b)) FROM array_distance_f32_{array_size}
|
|
</query>
|
|
<query>
|
|
WITH (SELECT a FROM array_distance_f32_{array_size} LIMIT 1) AS needle
|
|
SELECT sum(cosineDistance(needle, b)) FROM array_distance_f32_{array_size}
|
|
</query>
|
|
|
|
<!-- ===== Mixed UInt8 x Float64, non-const x non-const path =====
|
|
Exercises executeDistanceMixed / the is_common_mixed_pair specialisation. -->
|
|
<query>SELECT sum(L1Distance(u8, f64)) FROM array_distance_mixed_{array_size}</query>
|
|
<query>SELECT sum(L2Distance(u8, f64)) FROM array_distance_mixed_{array_size}</query>
|
|
<query>SELECT sum(cosineDistance(u8, f64)) FROM array_distance_mixed_{array_size}</query>
|
|
|
|
<!-- ===== Mixed UInt8 x Float64, const x non-const path =====
|
|
Exercises executeDistanceConstMixed. -->
|
|
<query>
|
|
WITH (SELECT u8 FROM array_distance_mixed_{array_size} LIMIT 1) AS needle
|
|
SELECT sum(L2Distance(needle, f64)) FROM array_distance_mixed_{array_size}
|
|
</query>
|
|
|
|
<drop_query>DROP TABLE array_distance_f32_{array_size}</drop_query>
|
|
<drop_query>DROP TABLE array_distance_mixed_{array_size}</drop_query>
|
|
</test>
|