mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Amos Bird
64095c8663
During projection MATERIALIZE (mutation/merge), every small block from the pipeline independently calls projection.calculate(), each creating a separate temp MergeTree part. For a 28M-row part with index_granularity=8192, this produces ~3500 temp parts requiring 4 levels of tree merge with ~390 rounds. Add a pre-calculate squashing step that accumulates source blocks up to min_insert_block_size_rows before calling calculate(). This reduces temp parts from ~3500 to ~30, cutting merge overhead from 4 levels / 390 rounds to 1 level / 3 rounds. The optimization applies to both MergeTask and MutateTask.
34 lines
1002 B
XML
34 lines
1002 B
XML
<test>
|
|
|
|
<settings>
|
|
<mutations_sync>2</mutations_sync>
|
|
</settings>
|
|
|
|
<create_query>
|
|
CREATE TABLE projection_materialize (
|
|
key UInt64,
|
|
val UInt64
|
|
) ENGINE = MergeTree()
|
|
ORDER BY key
|
|
</create_query>
|
|
|
|
<!-- Insert 50M rows and merge into a single part -->
|
|
<fill_query>
|
|
INSERT INTO projection_materialize SELECT number, rand() FROM numbers(50000000)
|
|
</fill_query>
|
|
<fill_query>
|
|
OPTIMIZE TABLE projection_materialize FINAL
|
|
</fill_query>
|
|
|
|
<!-- Add projection without materializing -->
|
|
<fill_query>
|
|
ALTER TABLE projection_materialize ADD PROJECTION agg_proj (SELECT key % 1000 AS k, sum(val) GROUP BY k)
|
|
</fill_query>
|
|
|
|
<!-- Each iteration: clear then re-materialize the projection -->
|
|
<query>ALTER TABLE projection_materialize CLEAR PROJECTION agg_proj, MATERIALIZE PROJECTION agg_proj</query>
|
|
|
|
<drop_query>DROP TABLE IF EXISTS projection_materialize</drop_query>
|
|
|
|
</test>
|