mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
zeekay
9a90943b8f
13,838 upstream commits, forward from the 26.6.1.1 fork point. 129 conflicts: 37 resolved mechanically (take upstream's file, re-apply the brand sweep) and 92 by hand where our fork and upstream had both changed the same code. The rule throughout: upstream owns the engine, we own the naming and a small set of deliberate deviations. Never resolve by discarding an upstream change — a dropped rename or a missed new parameter becomes a compile error a thousand files away. Deviations verified intact after the merge: ENABLE_XRAY still defaults OFF (CMakeLists.txt:105) X-Datastore-User read by the server (authenticateUserByHTTP.cpp) X-ClickHouse-* in src/ 0 hanzo/research present system.*_log retention bound present ArrowFlight rip held against upstream re-adding it Two things worth naming, because neither shows up as a conflict: src/CMakeLists.txt gained an UNGUARDED `add_object_library(datastore_storages_arrowflight Storages/ArrowFlight)` from upstream, for a directory our gRPC rip deleted. It sat outside every conflict hunk, so a resolver reading only the markers would sail straight past it into a CMake configure error. Removed; the Server/ArrowFlight sibling at line 474 stays because it is inside `if (TARGET ch_contrib::arrow_flight)`. The brand sweep had rewritten github.com/ClickHouse/ClickHouse issue links to github.com/Datastore/Datastore — a dead host. Those point at upstream's own tracker and are restored, alongside the contributors list, sql.clickhouse.com and the ClickHouse Inc copyright: attribution is theirs, not ours to rename. base/glibc-compatibility/memcpy/memcpy.h is deleted, not kept: upstream dropped it for LLVM-libc's, and our only change to it was a comment rebrand.
215 lines
10 KiB
XML
215 lines
10 KiB
XML
<datastore>
|
|
<local_disk_check_period_ms>30000</local_disk_check_period_ms>
|
|
|
|
<storage_configuration>
|
|
<disks>
|
|
<s3_disk>
|
|
<type>s3</type>
|
|
<path>s3_disk/</path>
|
|
<endpoint>http://localhost:11111/test/s3/</endpoint>
|
|
<access_key_id>clickhouse</access_key_id>
|
|
<secret_access_key>clickhouse</secret_access_key>
|
|
<request_timeout_ms>20000</request_timeout_ms>
|
|
<persistent_removal_log>true</persistent_removal_log>
|
|
</s3_disk>
|
|
<s3_plain_disk>
|
|
<type>s3_plain</type>
|
|
<endpoint>http://localhost:11111/test/s3_plain/</endpoint>
|
|
<access_key_id>clickhouse</access_key_id>
|
|
<secret_access_key>clickhouse</secret_access_key>
|
|
</s3_plain_disk>
|
|
<!-- Non-fake disk transactions (as ClickHouse Cloud / Keeper metadata uses): file
|
|
writes are batched into the part transaction and only become visible on commit. -->
|
|
<s3_disk_no_fake_transaction>
|
|
<type>s3</type>
|
|
<path>s3_disk_no_fake_transaction/</path>
|
|
<endpoint>http://localhost:11111/test/s3_no_fake_transaction/</endpoint>
|
|
<access_key_id>clickhouse</access_key_id>
|
|
<secret_access_key>clickhouse</secret_access_key>
|
|
<use_fake_transaction>false</use_fake_transaction>
|
|
</s3_disk_no_fake_transaction>
|
|
<s3_cache>
|
|
<type>cache</type>
|
|
<disk>s3_disk</disk>
|
|
<path>s3_cache/</path>
|
|
<max_size>209715200</max_size>
|
|
<max_file_segment_size>5Mi</max_file_segment_size>
|
|
<cache_on_write_operations>1</cache_on_write_operations>
|
|
<cache_policy>LRU</cache_policy>
|
|
<slru_size_ratio>0.3</slru_size_ratio>
|
|
<reserve_granularity>4Mi</reserve_granularity>
|
|
<allow_dynamic_cache_resize>1</allow_dynamic_cache_resize>
|
|
<keep_free_space_size_ratio>0.15</keep_free_space_size_ratio>
|
|
<keep_free_space_elements_ratio>0.15</keep_free_space_elements_ratio>
|
|
<keep_free_space_eviction_threads>4</keep_free_space_eviction_threads>
|
|
<background_download_queue_size_limit>50</background_download_queue_size_limit>
|
|
<load_metadata_asynchronously>0</load_metadata_asynchronously>
|
|
</s3_cache>
|
|
<s3_cache_02933>
|
|
<type>cache</type>
|
|
<disk>s3_disk</disk>
|
|
<path>s3_cache_02933/</path>
|
|
<max_size>128Mi</max_size>
|
|
<cache_on_write_operations>1</cache_on_write_operations>
|
|
<background_download_threads>0</background_download_threads>
|
|
<background_download_queue_size_limit>0</background_download_queue_size_limit>
|
|
<load_metadata_asynchronously>0</load_metadata_asynchronously>
|
|
</s3_cache_02933>
|
|
<s3_plain_rewritable>
|
|
<type>object_storage</type>
|
|
<object_storage_type>s3</object_storage_type>
|
|
<metadata_type>plain_rewritable</metadata_type>
|
|
<endpoint>http://localhost:11111/test/s3_plain_rewritable/</endpoint>
|
|
<access_key_id>clickhouse</access_key_id>
|
|
<secret_access_key>clickhouse</secret_access_key>
|
|
</s3_plain_rewritable>
|
|
<s3_plain_rewritable_cache>
|
|
<type>cache</type>
|
|
<disk>s3_plain_rewritable</disk>
|
|
<path>s3_plain_rewritable_cache/</path>
|
|
<max_size>128Mi</max_size>
|
|
<cache_on_write_operations>1</cache_on_write_operations>
|
|
<background_download_threads>0</background_download_threads>
|
|
<background_download_queue_size_limit>0</background_download_queue_size_limit>
|
|
<load_metadata_asynchronously>0</load_metadata_asynchronously>
|
|
</s3_plain_rewritable_cache>
|
|
<dynamically_resize_filesystem_cache>
|
|
<type>cache</type>
|
|
<disk>s3_disk</disk>
|
|
<path>dynamically_resize_filesystem_cache/</path>
|
|
<max_size>2124</max_size>
|
|
<max_file_segment_size>100</max_file_segment_size>
|
|
<boundary_alignment>100</boundary_alignment>
|
|
<cache_on_write_operations>1</cache_on_write_operations>
|
|
<cache_policy>LRU</cache_policy>
|
|
<!-- This cache is intentionally tiny (used to test dynamic resize). Disable
|
|
reserve-ahead so the default reserve_granularity does not hold a whole
|
|
segment of space and lower the steady-state occupancy below the test's threshold. -->
|
|
<reserve_granularity>0</reserve_granularity>
|
|
<allow_dynamic_cache_resize>1</allow_dynamic_cache_resize>
|
|
</dynamically_resize_filesystem_cache>
|
|
<!-- multi layer cache -->
|
|
<s3_cache_multi>
|
|
<type>cache</type>
|
|
<disk>s3_cache</disk>
|
|
<path>s3_cache_multi/</path>
|
|
<max_size>22548578304</max_size>
|
|
</s3_cache_multi>
|
|
<s3_cache_multi_2>
|
|
<type>cache</type>
|
|
<disk>s3_cache_multi</disk>
|
|
<path>s3_cache_multi_2/</path>
|
|
<max_size>22548578304</max_size>
|
|
</s3_cache_multi_2>
|
|
<s3_no_cache>
|
|
<type>s3</type>
|
|
<endpoint>http://localhost:11111/test/special/</endpoint>
|
|
<access_key_id>clickhouse</access_key_id>
|
|
<secret_access_key>clickhouse</secret_access_key>
|
|
<s3_check_objects_after_upload>0</s3_check_objects_after_upload>
|
|
</s3_no_cache>
|
|
<s3_plain_rewritable_cache_multi>
|
|
<type>cache</type>
|
|
<disk>s3_plain_rewritable_cache</disk>
|
|
<path>s3_plain_rewritable_cache_multi/</path>
|
|
<max_size>22548578304</max_size>
|
|
</s3_plain_rewritable_cache_multi>
|
|
<encrypted_s3_plain_rewritable_cache>
|
|
<type>encrypted</type>
|
|
<disk>s3_plain_rewritable_cache</disk>
|
|
<key>1234567812345678</key>
|
|
</encrypted_s3_plain_rewritable_cache>
|
|
<s3_cache_encrypted>
|
|
<type>encrypted</type>
|
|
<disk>s3_cache</disk>
|
|
<key>1234567812345678</key>
|
|
</s3_cache_encrypted>
|
|
<!-- Encryption over raw object storage with NO filesystem cache: lets the ReaderExecutor
|
|
engage on an encrypted disk (a cache would force the legacy fallback). -->
|
|
<s3_no_cache_encrypted>
|
|
<type>encrypted</type>
|
|
<disk>s3_no_cache</disk>
|
|
<key>1234567812345678</key>
|
|
</s3_no_cache_encrypted>
|
|
<!-- Two stacked encryption layers over raw object storage: exercises the executor's
|
|
multi-layer decryption on a real DiskEncrypted-over-DiskEncrypted config. -->
|
|
<s3_no_cache_encrypted_double>
|
|
<type>encrypted</type>
|
|
<disk>s3_no_cache_encrypted</disk>
|
|
<key>0987654321zyxwvu</key>
|
|
</s3_no_cache_encrypted_double>
|
|
</disks>
|
|
<policies>
|
|
<local_remote>
|
|
<volumes>
|
|
<local><disk>default</disk></local>
|
|
<remote><disk>s3_disk</disk></remote>
|
|
</volumes>
|
|
</local_remote>
|
|
<s3_cache>
|
|
<volumes>
|
|
<main>
|
|
<disk>s3_cache</disk>
|
|
</main>
|
|
</volumes>
|
|
</s3_cache>
|
|
<s3_no_fake_transaction>
|
|
<volumes>
|
|
<main>
|
|
<disk>s3_disk_no_fake_transaction</disk>
|
|
</main>
|
|
</volumes>
|
|
</s3_no_fake_transaction>
|
|
<s3_no_cache>
|
|
<volumes>
|
|
<main>
|
|
<disk>s3_no_cache</disk>
|
|
</main>
|
|
</volumes>
|
|
</s3_no_cache>
|
|
<s3_cache_multi>
|
|
<volumes>
|
|
<main>
|
|
<disk>s3_cache_multi_2</disk>
|
|
</main>
|
|
</volumes>
|
|
</s3_cache_multi>
|
|
<s3_cache_encrypted>
|
|
<volumes>
|
|
<main>
|
|
<disk>s3_cache_encrypted</disk>
|
|
</main>
|
|
</volumes>
|
|
</s3_cache_encrypted>
|
|
<s3_no_cache_encrypted>
|
|
<volumes>
|
|
<main>
|
|
<disk>s3_no_cache_encrypted</disk>
|
|
</main>
|
|
</volumes>
|
|
</s3_no_cache_encrypted>
|
|
<s3_no_cache_encrypted_double>
|
|
<volumes>
|
|
<main>
|
|
<disk>s3_no_cache_encrypted_double</disk>
|
|
</main>
|
|
</volumes>
|
|
</s3_no_cache_encrypted_double>
|
|
</policies>
|
|
</storage_configuration>
|
|
<!-- Allow data-lake table engines (Iceberg, Delta, Hudi) to use these disks via
|
|
`SETTINGS disk = '...'`. Used by tests like 04208_iceberg_disk_with_cache_no_bad_cast.sql.
|
|
The list intentionally only contains disks that wrap an `S3ObjectStorage` (raw,
|
|
cached, or with a non-default metadata layer) so we exercise the unwrap-before-cast
|
|
path in `S3StorageParsedArguments::fromDisk` without affecting other tests. -->
|
|
<allowed_disks_for_table_engines>s3_disk,s3_cache,s3_plain_disk,s3_plain_rewritable</allowed_disks_for_table_engines>
|
|
<filesystem_caches>
|
|
<cache_for_readbigat>
|
|
<path>./cache_for_readbigat/</path>
|
|
<max_size>1Gi</max_size>
|
|
<max_file_segment_size>1Mi</max_file_segment_size>
|
|
<boundary_alignment>1Mi</boundary_alignment>
|
|
</cache_for_readbigat>
|
|
</filesystem_caches>
|
|
</datastore>
|