mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Report the real corruption when checking a projection part that failed to load
A projection part that fails to load before its columns are set (for example because of a corrupted serialization.json) has an empty column list. CHECK TABLE compared the projection's on-disk columns.txt against that empty list and reported a misleading "Columns doesn't match ... Expected: 0 columns" error, hiding the real corruption. Take the expected columns from the projection metadata when the part's own columns are unavailable, so the check proceeds and reports the actual broken file. Extracted from #93694. Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Fable 5
parent
dd02d56ca8
commit
ddc2007d54
@@ -353,6 +353,10 @@ static IMergeTreeDataPart::Checksums checkDataPart(
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep the metadata snapshot alive for the whole loop: `projections_description` references into it.
|
||||
auto metadata_snapshot = data_part->storage.getInMemoryMetadataPtr(nullptr, /*bypass_metadata_cache=*/true);
|
||||
const auto & projections_description = metadata_snapshot->projections;
|
||||
|
||||
std::string broken_projections_message;
|
||||
for (const auto & [name, projection] : data_part->getProjectionParts())
|
||||
{
|
||||
@@ -365,9 +369,18 @@ static IMergeTreeDataPart::Checksums checkDataPart(
|
||||
try
|
||||
{
|
||||
bool noop = false;
|
||||
|
||||
/// A projection part that failed to load before its columns were set (e.g. because of a
|
||||
/// corrupted serialization.json) has an empty column list. Checking against it would
|
||||
/// report a misleading "columns don't match" error and hide the real corruption, so
|
||||
/// take the expected columns from the projection metadata in that case.
|
||||
NamesAndTypesList projection_columns = projection->getColumns();
|
||||
if (projection_columns.empty() && projections_description.has(name))
|
||||
projection_columns = projections_description.get(name).metadata->getColumns().getAll();
|
||||
|
||||
projection_checksums = checkDataPart(
|
||||
projection, *data_part_storage.getProjection(projection_file),
|
||||
projection->getColumns(), projection->getType(),
|
||||
projection_columns, projection->getType(),
|
||||
projection->getFileNamesWithoutChecksums(),
|
||||
read_settings, require_checksums, is_cancelled, noop, /* throw_on_broken_projection */false);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
part check passed: 0
|
||||
reports the misleading columns mismatch: 0
|
||||
reports the real corruption in serialization.json: 1
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest, no-shared-merge-tree, no-object-storage
|
||||
|
||||
# A projection part that fails to load before its columns are set (here: corrupted
|
||||
# serialization.json) has an empty column list. CHECK TABLE used to compare the projection's
|
||||
# on-disk columns.txt against that empty list and report a misleading "Columns doesn't match ...
|
||||
# Expected: 0 columns" error, hiding the real corruption. It must report the actual problem.
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS t_broken_proj_check SYNC"
|
||||
${CLICKHOUSE_CLIENT} --query "
|
||||
CREATE TABLE t_broken_proj_check (id UInt64, v UInt64, PROJECTION p1 (SELECT v, count() GROUP BY v))
|
||||
ENGINE = MergeTree ORDER BY id"
|
||||
${CLICKHOUSE_CLIENT} --query "INSERT INTO t_broken_proj_check SELECT number, number % 10 FROM numbers(1000)"
|
||||
|
||||
DATA_PATH=$(${CLICKHOUSE_CLIENT} --query "SELECT path FROM system.parts WHERE database = currentDatabase() AND table = 't_broken_proj_check' AND active")
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "DETACH TABLE t_broken_proj_check"
|
||||
printf 'garbage' > "${DATA_PATH}p1.proj/serialization.json"
|
||||
# The attach legitimately logs the broken projection at error level; do not let the
|
||||
# server logs reach stderr under randomized send_logs_level.
|
||||
${CLICKHOUSE_CLIENT} --send_logs_level=fatal --query "ATTACH TABLE t_broken_proj_check"
|
||||
|
||||
RESULT=$(${CLICKHOUSE_CLIENT} --send_logs_level=fatal --query "CHECK TABLE t_broken_proj_check SETTINGS check_query_single_value_result = 0 FORMAT TSV")
|
||||
|
||||
echo -n "part check passed: "
|
||||
echo "$RESULT" | cut -f2
|
||||
echo -n "reports the misleading columns mismatch: "
|
||||
echo "$RESULT" | grep -c "Columns doesn.t match" || true
|
||||
echo -n "reports the real corruption in serialization.json: "
|
||||
echo "$RESULT" | grep -c "serialization.json" || true
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query "DROP TABLE t_broken_proj_check SYNC"
|
||||
Reference in New Issue
Block a user