datastore/tests/docker_scripts/upgrade_runner.sh

#!/bin/bash
# shellcheck disable=SC2094
# shellcheck disable=SC2086
# shellcheck disable=SC2024

# Avoid overlaps with previous runs
dmesg --clear

set -ex

# we mount tests folder from repo to /usr/share
ln -s /repo/ci/jobs/scripts/stress/stress.py /usr/bin/stress
ln -s /repo/tests/datastore-test /usr/bin/datastore-test
ln -s /repo/tests/ci/download_release_packages.py /usr/bin/download_release_packages
ln -s /repo/tests/ci/get_previous_release_tag.py /usr/bin/get_previous_release_tag

# Stress tests and upgrade check uses similar code that was placed
# in a separate bash library. See tests/ci/stress_tests.lib
# shellcheck source=../stateless/stress_tests.lib
source /repo/tests/docker_scripts/stress_tests.lib

cd /repo && python3 /repo/ci/jobs/scripts/datastore_proc.py start_azurite || { echo "Failed to start azurite"; exit 1; }
cd /repo && python3 /repo/ci/jobs/scripts/datastore_proc.py start_minio stateless || ( echo "Failed to start minio" && exit 1 ) # to have a proper environment

bash /repo/ci/jobs/scripts/functional_tests/setup_kafka.sh || { echo "Failed to start Kafka (Redpanda)"; exit 1; }

echo "Get previous release tag"
PACKAGES_DIR=/repo/ci/tmp
# shellcheck disable=SC2016
previous_release_tag=$(dpkg-deb --showformat='${Version}' --show $PACKAGES_DIR/datastore-client*.deb | get_previous_release_tag)
if [ $? -ne 0 ]; then
    echo "Failed to get previous release tag"
    exit 1
fi
echo $previous_release_tag

echo "Clone previous release repository"
git clone https://github.com/ClickHouse/ClickHouse.git --no-tags --progress --branch=$previous_release_tag --no-recurse-submodules --depth=1 previous_release_repository

echo "Download datastore-server from the previous release"
mkdir previous_release_package_folder

# --- download previous release packages: fail closed on a missing required one ---
# `download_release_packages` exits nonzero when a required previous-release
# package is missing or fails to download. Stop here at the download boundary
# with a clear, attributable status instead of letting the later `install_packages`
# die with an opaque `dpkg` glob error. (`set -e` does not fire inside an
# `&& ... || ...` list, so the nonzero status must be handled explicitly.)
if echo $previous_release_tag | download_release_packages; then
    echo -e "Download script exit code$OK" >> /test_output/test_results.tsv
else
    echo -e "Download script failed$FAIL" >> /test_output/test_results.tsv
    echo -e 'failure\tFailed to download previous release packages' > /test_output/check_status.tsv
    exit 1
fi
# --- end download previous release packages ---

# Check if we cloned previous release repository successfully
if ! [ "$(ls -A previous_release_repository/tests/queries)" ]
then
    echo -e 'failure\tFailed to clone previous release tests' > /test_output/check_status.tsv
    exit 1
elif ! [ "$(ls -A previous_release_package_folder/datastore-common-static_*.deb && ls -A previous_release_package_folder/datastore-server_*.deb)" ]
then
    echo -e 'failure\tFailed to download previous release packages' > /test_output/check_status.tsv
    exit 1
fi

echo -e "Successfully cloned previous release tests$OK" >> /test_output/test_results.tsv
echo -e "Successfully downloaded previous release packages$OK" >> /test_output/test_results.tsv

# Make upgrade check more funny by forcing Ordinary engine for system database
mkdir -p /var/lib/datastore/metadata
echo "ATTACH DATABASE system ENGINE=Ordinary" > /var/lib/datastore/metadata/system.sql

# Install previous release packages
install_packages previous_release_package_folder

# NOTE: we need to run datastore-local under script to get settings without any adjustments, like datastore-local does in case of stdout is not a tty
function save_settings_clean()
{
  local out=$1 && shift
  script -q -c "datastore-local --implicit-select 0 -q \"select * from system.settings into outfile '$out'\"" --log-out /dev/null
}

function save_mergetree_settings_clean()
{
  local out=$1 && shift
  script -q -c "datastore-local --implicit-select 0 -q \"select * from system.merge_tree_settings into outfile '$out'\"" --log-out /dev/null
}

# We save the (numeric) version of the old server to compare setting changes between the 2
# We do this since we are testing against the latest release, not taking into account release candidates, so we might
# be testing current master (24.6) against the latest stable release (24.4)
function save_major_version()
{
  local out=$1 && shift
  datastore-local -q "SELECT a[1]::UInt64 * 100 + a[2]::UInt64 as v FROM (Select splitByChar('.', version()) as a) into outfile '$out'"
}

save_settings_clean 'old_settings.native'
save_mergetree_settings_clean 'old_merge_tree_settings.native'
save_major_version 'old_version.native'
old_major_version=$(datastore-local -q "select a[1] || '.' || a[2] from (select splitByChar('.', version()) as a)")

configure_opts=(
    # Let's enable S3 storage by default
    --s3-storage
)
if [ $((RANDOM % 2)) -eq 0 ]; then
    configure_opts+=(--encrypted-storage)
fi

# Start server from previous release
configure "${configure_opts[@]}" --previous-release

# But we still need default disk because some tables loaded only into it
sudo sed -i "s|<main><disk>s3</disk></main>|<main><disk>s3</disk></main><default><disk>default</disk></default>|" /etc/datastore-server/config.d/s3_storage_policy_by_default.xml
sudo chown datastore /etc/datastore-server/config.d/s3_storage_policy_by_default.xml
sudo chgrp datastore /etc/datastore-server/config.d/s3_storage_policy_by_default.xml

start_server || (echo "Failed to start server" && exit 1)

datastore-client --receive_timeout 30 --query="SELECT 'Server version: ', version()"

mkdir tmp_stress_output

stress --test-cmd="/usr/bin/datastore-test --queries=\"previous_release_repository/tests/queries\""  --upgrade-check --output-folder tmp_stress_output --global-time-limit=1200 \
    && echo -e "Test script exit code$OK" >> /test_output/test_results.tsv \
    || echo -e "Test script failed$FAIL script exit code: $?" >> /test_output/test_results.tsv

rm -rf tmp_stress_output

# We experienced deadlocks in this command in very rare cases. Let's debug it:
timeout 10m datastore-client --query="SELECT 'Tables count:', count() FROM system.tables" ||
(
    echo "thread apply all backtrace (on select tables count)" >> /test_output/gdb.log
    timeout 30m gdb -batch -ex 'thread apply all backtrace' -p "$(cat /var/run/datastore-server/datastore-server.pid)" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log
    datastore stop --force
)

# Use bigger timeout for previous version and disable additional hang check
stop_server 300 false || (echo "Failed to stop server" && exit 1)
mv /var/log/datastore-server/datastore-server.log /var/log/datastore-server/datastore-server.stress.log

# Install and start new server
install_packages $PACKAGES_DIR
configure "${configure_opts[@]}"

# Check that all new/changed setting were added in settings changes history.
# Some settings can be different for builds with sanitizers, so we check
# settings changes only for non-sanitizer builds.
# The automatic value of 'max_threads' and similar settings is rendered as auto(N); older releases
# rendered it as the quoted 'auto(N)' (with the quotes baked into the value). Suppress only this pure
# rendering difference - a row where the old value is exactly the new value wrapped in single quotes -
# so it is not reported as a setting change. A genuine change of an auto-valued setting's default is
# still caught and must have a settings changes history entry.
IS_SANITIZED=$(datastore-local --query "SELECT value LIKE '%-fsanitize=%' FROM system.build_options WHERE name = 'CXX_FLAGS'")
if [ "${IS_SANITIZED}" -eq "0" ]
then
  save_settings_clean 'new_settings.native'
  save_mergetree_settings_clean 'new_merge_tree_settings.native'
  datastore-local -nmq "
  CREATE TABLE old_settings AS file('old_settings.native');
  CREATE TABLE old_merge_tree_settings AS file('old_merge_tree_settings.native');
  CREATE TABLE old_version AS file('old_version.native');
  CREATE TABLE new_settings AS file('new_settings.native');
  CREATE TABLE new_merge_tree_settings AS file('new_merge_tree_settings.native');

  SELECT
      name,
      new_settings.value AS new_value,
      old_settings.value AS old_value
  FROM new_settings
  LEFT JOIN old_settings ON new_settings.name = old_settings.name
  WHERE (old_value IS NULL OR new_value != old_value)
      AND NOT (old_value IS NOT NULL AND new_value LIKE 'auto(%' AND old_value = concat('''', new_value, ''''))
      AND (name NOT IN (
      SELECT arrayJoin(tupleElement(changes, 'name'))
      FROM
      (
          SELECT *, splitByChar('.', version) AS version_array FROM system.settings_changes WHERE type = 'Session'
      )
      WHERE (version_array[1]::UInt64 * 100 + version_array[2]::UInt64) > (SELECT v FROM old_version LIMIT 1)
  ))
  SETTINGS join_use_nulls = 1
  INTO OUTFILE 'changed_settings.txt'
  FORMAT PrettyCompactNoEscapes;

  SELECT
      name,
      new_merge_tree_settings.value AS new_value,
      old_merge_tree_settings.value AS old_value
  FROM new_merge_tree_settings
  LEFT JOIN old_merge_tree_settings ON new_merge_tree_settings.name = old_merge_tree_settings.name
  WHERE (old_value IS NULL OR new_value != old_value)
      AND NOT (old_value IS NOT NULL AND new_value LIKE 'auto(%' AND old_value = concat('''', new_value, ''''))
      AND (name NOT IN (
      SELECT arrayJoin(tupleElement(changes, 'name'))
      FROM
      (
          SELECT *, splitByChar('.', version) AS version_array FROM system.settings_changes WHERE type = 'MergeTree'
      )
      WHERE (version_array[1]::UInt64 * 100 + version_array[2]::UInt64) > (SELECT v FROM old_version LIMIT 1)
  ))
  SETTINGS join_use_nulls = 1
  INTO OUTFILE 'changed_merge_tree_settings.txt'
  FORMAT PrettyCompactNoEscapes;

  SELECT name
  FROM new_settings
  WHERE (name NOT IN (
      SELECT name
      FROM old_settings
  )) AND (name NOT IN (
      SELECT arrayJoin(tupleElement(changes, 'name'))
      FROM
      (
          SELECT *, splitByChar('.', version) AS version_array FROM system.settings_changes WHERE type = 'Session'
      )
      WHERE (version_array[1]::UInt64 * 100 + version_array[2]::UInt64) > (SELECT v FROM old_version LIMIT 1)
  ))
  INTO OUTFILE 'new_settings.txt'
  FORMAT PrettyCompactNoEscapes;

  SELECT name
  FROM new_merge_tree_settings
  WHERE (name NOT IN (
      SELECT name
      FROM old_merge_tree_settings
  )) AND (name NOT IN (
      SELECT arrayJoin(tupleElement(changes, 'name'))
      FROM
      (
          SELECT *, splitByChar('.', version) AS version_array FROM system.settings_changes WHERE type = 'MergeTree'
      )
      WHERE (version_array[1]::UInt64 * 100 + version_array[2]::UInt64) > (SELECT v FROM old_version LIMIT 1)
  ))
  INTO OUTFILE 'new_merge_tree_settings.txt'
  FORMAT PrettyCompactNoEscapes;
  "

  if [ -s changed_settings.txt ]
  then
      mv changed_settings.txt /test_output/
      echo -e "Changed settings are not reflected in the settings changes history (see changed_settings.txt)$FAIL$(head_escaped /test_output/changed_settings.txt)" >> /test_output/test_results.tsv
  else
      echo -e "There are no changed settings or they are reflected in settings changes history$OK" >> /test_output/test_results.tsv
  fi

  if [ -s changed_merge_tree_settings.txt ]
  then
      mv changed_merge_tree_settings.txt /test_output/
      echo -e "Changed MergeTree settings are not reflected in the settings changes history (see changed_merge_tree_settings.txt)$FAIL$(head_escaped /test_output/changed_merge_tree_settings.txt)" >> /test_output/test_results.tsv
  else
      echo -e "There are no changed MergeTree settings or they are reflected in settings changes history$OK" >> /test_output/test_results.tsv
  fi

  if [ -s new_settings.txt ]
  then
      mv new_settings.txt /test_output/
      echo -e "New settings are not reflected in settings changes history (see new_settings.txt)$FAIL$(head_escaped /test_output/new_settings.txt)" >> /test_output/test_results.tsv
  else
      echo -e "There are no new settings or they are reflected in settings changes history$OK" >> /test_output/test_results.tsv
  fi

  if [ -s new_merge_tree_settings.txt ]
  then
      mv new_merge_tree_settings.txt /test_output/
      echo -e "New MergeTree settings are not reflected in settings changes history (see new_merge_tree_settings.txt)$FAIL$(head_escaped /test_output/new_merge_tree_settings.txt)" >> /test_output/test_results.tsv
  else
      echo -e "There are no new MergeTree settings or they are reflected in settings changes history$OK" >> /test_output/test_results.tsv
  fi

fi

# Just in case previous version left some garbage in zk
sudo sed -i "s|>1<|>0<|g" /etc/datastore-server/config.d/lost_forever_check.xml
rm -f /etc/datastore-server/config.d/filesystem_caches_path.xml

# Set compatibility setting to previous version, so we won't fail due to known backward incompatible changes.
echo "<datastore>
    <profiles>
        <default>
            <compatibility>$old_major_version</compatibility>
            <!-- Allow loading projections with positional arguments (e.g., GROUP BY 1, 2) created by the old server. -->
            <enable_positional_arguments_for_projections>1</enable_positional_arguments_for_projections>
        </default>
    </profiles>
</datastore>" > /etc/datastore-server/users.d/compatibility.xml

cat /etc/datastore-server/users.d/compatibility.xml

# List of allowed reasons why the server cannot start up
# ADD ENTRIES HERE ONLY IF YOU ARE CERTAIN THEY DO NOT INTRODUCE BACKWARD-INCOMPATIBLE CHANGES
# 1. Lazy database engine has been removed in a backward-incompatible manner
check_allow_list() {
    local log="/var/log/datastore-server/datastore-server.log"
    if [ -f "$log" ] && rg -q "Unknown database engine: Lazy" "$log"; then
        # cleanup errors
        echo -e "Found allow-listed error in logs. Suppressing failure"$OK > /test_output/test_results.tsv
        # Finishing tests because following checks would fail
        exit 0
    fi
}

start_server || check_allow_list || (echo "Failed to start server" && exit 1)

datastore-client --receive_timeout 30 --query "SELECT 'Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \
    || (rg --text "<Error>.*Application" /var/log/datastore-server/datastore-server.log > /test_output/application_errors.txt \
    && echo -e "Server failed to start (see application_errors.txt and datastore-server.clean.log)$FAIL$(trim_server_logs application_errors.txt)" \
    >> /test_output/test_results.tsv)

# Remove file application_errors.txt if it's empty
[ -s /test_output/application_errors.txt ] || rm -f /test_output/application_errors.txt

datastore-client --receive_timeout 30 --query="SELECT 'Server version: ', version()"

# Let the server run for a while before checking log.
sleep 60

stop_server || (echo "Failed to stop server" && exit 1)
mv /var/log/datastore-server/datastore-server.log /var/log/datastore-server/datastore-server.upgrade.log
cp /var/log/datastore-server/datastore-server.upgrade.log /test_output/datastore-server.upgrade.log

# Error messages (we should ignore some errors)
# FIXME https://github.com/ClickHouse/ClickHouse/issues/38643 ("Unknown index: idx.")
# FIXME Not sure if it's expected, but some tests from stress test may not be finished yet when we restarting server.
#       Let's just ignore all errors from queries ("} <Error> TCPHandler: Code:", "} <Error> executeQuery: Code:")
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'")
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 - bad mutation does not indicate backward incompatibility:
#       stress tests may leave behind intentionally-broken mutations that retry after upgrade.
#       `CANNOT_PARSE_TEXT` errors come from:
#       - 00834_kill_mutation{,_replicated_zookeeper}: `DELETE WHERE toUInt32(s) = 1` on String data ('a', 'b')
#       - 01414_mutations_and_errors_zookeeper: `MODIFY COLUMN value UInt64` on String data ('Hello')
#       - 04338_on_fly_mutation_read_overwritten_lc_source: `MODIFY COLUMN v UInt64` on String data ('x')
#       - 01155_old_mutation_parts_to_do: `UPDATE m = m*toInt8(s) WHERE n=3` on String data ('fail')
#       `MutateFromLogEntryTask` is also excluded for the same reason, but only catches the first log line;
#       the wrapping `MergeTreeBackgroundExecutor` line also needs to be excluded.
# `Value passed to 'throwIf' function is non-zero` (`FUNCTION_THROW_IF_VALUE_IS_NON_ZERO`, Code: 395) is the same
#       class of expected test-induced mutation error. It comes from tests that run an intentionally-failing
#       `ALTER TABLE ... UPDATE <col> = <col> <op> throwIf(1)` async mutation:
#       - 04341_broken_mutation_part_log_flush: plain `MergeTree`, so the error is emitted by
#         `MutatePlainMergeTreeTask`, which is NOT covered by the `MutateFromLogEntryTask` exclusion above.
#       - 02597_column_{update,delete,update_tricky_expression}_and_replication: `ReplicatedMergeTree`, where
#         `MutateFromLogEntryTask` catches only the first line and the wrapping `MergeTreeBackgroundExecutor`
#         line leaks.
#       After the upgrade restart the broken mutation is retried in the background and logged to
#       `clickhouse-server.upgrade.log`. `throwIf` is a user-level function that only ever raises when a query
#       explicitly calls it with a truthy argument, so this default message can only ever originate from such a
#       test query, never from a background or internal assertion - matching it is therefore safe to suppress
#       globally, exactly like the `Code: 236 ... Cancelled mutating parts` message that the same cancelled test
#       mutations emit above. Matching the message rather than the task type also covers the wrapping
#       `MergeTreeBackgroundExecutor` line of the replicated case in a single entry.
# `NO_SUCH_INTERSERVER_IO_ENDPOINT` is expected during upgrades because replicated tables try to fetch parts
# from replicas that are being restarted and whose interserver endpoints are temporarily unavailable.
# `Unknown tokenizer: 'unicode_word'` appears because the `unicode_word` tokenizer was renamed to `asciiCJK`
#       (with `unicodeWord` as a transitional alias). Tables from old versions using `unicode_word` trigger this
#       on attach. Narrowed to the exact legacy name so genuinely unsupported tokenizer names are not masked.
# `Azure::Storage::StorageException.*Not found address of host` is a transient Azure blob DNS resolution failure
#       for `openbucketforpublicci.blob.core.windows.net`. Filtered via regex in the secondary pipe below to match
#       both the Azure SDK exception type AND the DNS error together, so non-Azure DNS errors are not masked.
# `SystemLogQueue` + `Queue had been full` overflow happens under heavy stress test load and is not a
#       compatibility bug. Filtered via regex in the secondary pipe below to require both the component name
#       AND the specific overflow phrase together (the log format is `SystemLogQueue (system.<table>): Queue
#       had been full ...`), so other SystemLogQueue errors are not masked.
# `TraceCollector` + `CANNOT_READ_FROM_FILE_DESCRIPTOR` is a transient pipe close error during server shutdown,
#       unrelated to upgrade compatibility. Filtered via regex in the secondary pipe below to require both
#       the component name AND the specific error code together, so non-pipe TraceCollector errors are not masked.
# `This engine is deprecated and is not supported in transactions` appears for Ordinary engine tables from old versions.
# `Prevent converting Nullable type to non-Nullable type inside mutation` is from stricter validation in new versions
#       applied to old mutations that were created before the validation existed.
# `e.what() = failed to parse response body` is a transient Azure blob storage batch-parsing error from
#       `Azure::Storage::Blobs`. Narrowed with the `e.what() = ` prefix to only match caught C++ exceptions of this
#       type (stable Datastore exception formatting), so arbitrary log lines containing the phrase are not masked.
# `while loading statistics` + `ILLEGAL_STATISTICS` appears when the statistics file format version changes between
#       releases. The new binary cannot deserialize old statistics files and throws ILLEGAL_STATISTICS (Code: 708).
#       Filtered via regex in the secondary pipe below to require both the loading context AND the error code together.
# `rdk:FAIL` + `Connect to` + `Connection refused` is a librdkafka broker connection error when the Kafka
#       broker is unavailable during upgrade (no broker is running in the upgrade test environment). Filtered
#       via regex in the secondary pipe below to require the `rdk:FAIL` tag AND the specific connection-refused
#       message together, so real Kafka regressions (auth, protocol, config) that also emit `rdk:FAIL` are
#       not masked.
# `StorageKafka2` + `Exception during get topic partitions from Kafka: Local: Broker transport failure` is
#       the wrapper-exception variant of the same class of error: the `KafkaConsumer2` background poll loop
#       (`KafkaConsumer2::getAllTopicPartitionOffsets` -> `cppkafka::HandleException`) keeps polling while the
#       Redpanda broker is in transition during the upgrade restart sequence and logs `<Error>` for each retry.
#       Filtered via regex in the secondary pipe below to require both the `StorageKafka2` engine context AND
#       the `Broker transport failure` symptom together, so real `StorageKafka2` regressions (auth errors,
#       timeouts, protocol errors, other broker errors) still surface.
# `No stream (column1_renamedcolumn1.bin) file checksum for column column1_renamed` is the unique signature of
#       issue #102259 (`getFileNameForRenamedColumnStream` uses `substr(0, N)` instead of `substr(N)`, producing
#       `<renamed><original>.bin` instead of `<renamed>.bin`). The fix is in PR #102689; until it lands, the
#       upgraded server detaches the renamed-column parts of `02538_alter_rename_sequence`'s `wrong_metadata_wide`
#       table. Matched via the exact corrupted filename + column name, which is unique to that test and that bug.
# `No stream (ba1.bin) file checksum for column b` is the same bug observed on
#       `02555_davengers_rename_chain`'s `wrong_metadata` table. The test chains `a -> a1` and then
#       `a1 -> b`, so the corrupted file name is `<new=b><old=a1>.bin = ba1.bin` for column `b`. The
#       `<column><stream>` combination is unique to this chained-rename test.
# `wrong_metadata` + `Detaching broken part` + `backward incompatibility` is the follow-up cleanup line for
#       the same issue: a "Detaching broken part" notice that does not contain the corrupted filename.
#       Filtered via regex in the secondary pipe below to require all three substrings together, so unrelated
#       broken-part detach messages are not masked. The regex matches both `wrong_metadata` (from
#       `02555_davengers_rename_chain`) and `wrong_metadata_wide` (from `02538_alter_rename_sequence`).
# `RaftInstance: session` + `failed to read rpc header from socket` + `due to error` is a benign NuRaft
#       shutdown-time message emitted by `rpc_session::start` in `contrib/NuRaft/src/asio_service.cxx` when
#       the peer side of an accepted RPC connection (in single-node Keeper this is a loopback `::1` client)
#       closes its socket before the acceptor cancels its pending header read. The already-allow-listed sibling
#       `RaftInstance: failed to accept a rpc connection due to error 125` is the inverse race (acceptor wins);
#       this regex covers the other variant (peer wins, read returns EOF or another transient socket error).
#       Filtered via regex in the secondary pipe below to require all three substrings together, so unrelated
#       RaftInstance errors are not masked.
# `Failed to flush system log system.metric_log` + `DEADLOCK_AVOIDED` is a transient lock-timeout emitted by
#       `SystemLog<MetricLogElement>::flushImpl` when the background `MetricLog` flush loop races with the
#       ongoing upgrade-test shutdown sequence. Another worker (DROP/RENAME/DETACH on `system.metric_log`,
#       or a parallel mutation/merge) holds the table-level write lock, the flush blocks for the 60s timeout,
#       and then aborts with code 473 (`DEADLOCK_AVOIDED`). Losing a few `MetricLogElement` samples during
#       shutdown is harmless; the upgrade-check stage is not asserting on metric continuity. Filtered via
#       regex in the secondary pipe below to require BOTH the `SystemLog` flush wrapper for `metric_log` AND
#       the `DEADLOCK_AVOIDED` error code together, so unrelated lock-timeout errors and unrelated
#       `metric_log` errors are not masked.
# `PostgreSQLConnectionPool: Connection error` and `DatabasePostgreSQL::removeOutdatedTables` + `Connection to`
#       + `failed` are benign background-reconnection errors from a `DatabasePostgreSQL` engine left behind by
#       `04210_show_remote_databases_in_system_tables` when the stress phase interrupts that test between its
#       `CREATE DATABASE ... ENGINE = PostgreSQL('192.0.2.1:5432', ...)` and the final `DROP DATABASE` (the
#       documentation IP `192.0.2.1`, RFC 5737, is intentionally unreachable). `DatabasePostgreSQL::startup`
#       always activates the `PostgreSQLCleanerTask`, so after the upgrade restart the leftover database's cleaner
#       task (`removeOutdatedTables`) tries to connect and the connection pool logs `<Error>` for each retry.
#       Filtered via regex in the secondary pipe below to require the PostgreSQL connection-pool / cleaner-task
#       context AND a connection failure to the known 04210 fixture host `192.0.2.1:5432` together, so a real
#       cleaner-task connect failure on a different (persisted) host, or a non-connection PostgreSQL regression
#       (auth, protocol, query errors), still fails this job.
#       The same leftover `DatabasePostgreSQL` engine reaches the same unreachable host through two more code
#       paths that also log the benign connection failure, so they are filtered the same way:
#       `DatabasePostgreSQL::getTablesIterator` (a `system.tables` scan reads the leftover engine and probes
#       the pool; it deliberately swallows the error and logs it via `tryLogCurrentException`), and
#       `AsyncLoader::worker` (the post-upgrade startup asynchronously loads the leftover engine and logs the
#       same `POSTGRESQL_CONNECTION_FAILURE` (Code: 614) exception). Both matchers require the PostgreSQL
#       code-path context AND a connection failure to the known 04210 fixture host `192.0.2.1:5432` together
#       (the `AsyncLoader` one additionally pins the PostgreSQL-specific `Code: 614`). `PoolWithFailover::get`
#       builds every `pqxx::broken_connection` into the same `Code: 614` / `Connection to <host_port> failed`
#       text, so scoping to the fixture host (not any `Connection to .* failed`) keeps genuine connect-time
#       PostgreSQL regressions on a persisted `DatabasePostgreSQL` (a different host, or a non-614 code)
#       still failing this job.
# The MySQL matchers below filter the same class of benign connection failure from a `DatabaseMySQL` engine
#       that `04210_show_remote_databases_in_system_tables` also creates
#       (`ENGINE = MySQL('192.0.2.1:3306', ...)`, the same unreachable RFC 5737 host). On the post-upgrade
#       restart the engine probes the server while loading the persisted object and logs `<Error>` for the
#       expected connection failure. Filtered to require the MySQL component AND the connection-failure
#       symptom together, so real MySQL regressions (auth, protocol, query errors) are not masked.
echo "Check for Error messages in server log:"
rg -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
           -e "Code: 236. DB::Exception: Cancelled mutating parts" \
           -e "REPLICA_IS_ALREADY_ACTIVE" \
           -e "REPLICA_ALREADY_EXISTS" \
           -e "ALL_REPLICAS_LOST" \
           -e "DDLWorker: Cannot parse DDL task query" \
           -e "RaftInstance: failed to accept a rpc connection due to error 125" \
           -e "UNKNOWN_DATABASE" \
           -e "NETWORK_ERROR" \
           -e "UNKNOWN_TABLE" \
           -e "ZooKeeperClient" \
           -e "KEEPER_EXCEPTION" \
           -e "DirectoryMonitor" \
           -e "DistributedInsertQueue" \
           -e "TABLE_IS_READ_ONLY" \
           -e "Code: 1000, e.code() = 111, Connection refused" \
           -e "[rdk:FAIL]" \
           -e "[rdk:ERROR]" \
           -e "Error during draining" \
           -e "Timeout during draining" \
           -e "UNFINISHED" \
           -e "NETLINK_ERROR" \
           -e "Renaming unexpected part" \
           -e "PART_IS_TEMPORARILY_LOCKED" \
           -e "and a merge is impossible: we didn't find" \
           -e "found in queue and some source parts for it was lost" \
           -e "is lost forever." \
           -e "Unknown index: idx." \
           -e "Cannot parse string 'Hello' as UInt64" \
           -e "Cannot parse string 'x' as UInt64" \
           -e "Cannot parse string 'Hello' as UInt32" \
           -e "Cannot parse string \'Hello\' as UInt32" \
           -e "Cannot parse string \\'Hello\\' as UInt32" \
           -e "Cannot parse string \'a\' as UInt32" \
           -e "Cannot parse string \'b\' as UInt32" \
           -e "Cannot parse string 'a' as UInt32" \
           -e "Cannot parse string 'b' as UInt32" \
           -e "Cannot parse string 'fail' as Int8" \
           -e "} <Error> TCPHandler: Code:" \
           -e "} <Error> executeQuery: Code:" \
           -e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \
           -e "The set of parts restored in place of" \
           -e "(ReplicatedMergeTreeAttachThread): Initialization failed. Error" \
           -e "Code: 269. DB::Exception: Destination table is myself" \
           -e "Coordination::Exception: Connection loss" \
           -e "MutateFromLogEntryTask" \
           -e "No connection to ZooKeeper, cannot get shared table ID" \
           -e "Session expired" \
           -e "TOO_MANY_PARTS" \
           -e "Authentication failed" \
           -e "Cannot flush" \
           -e "Container already exists" \
           -e "doesn't have metadata version on disk" \
           -e "Unknown codec family: ZSTD_QAT" \
           -e "Unknown codec family: DEFLATE_QPL" \
           -e "Bad get: has String, requested UInt64. (BAD_GET" \
           -e "Disk does not support stat. (NOT_IMPLEMENTED" \
           -e "QUALIFY clause is not supported in the old analyzer" \
           -e "Cannot attach table \`test_7\`" \
           -e "Cannot open file /var/lib/datastore/access/" \
           -e "NO_SUCH_INTERSERVER_IO_ENDPOINT" \
           -e "Mapping for table with UUID=1f474183-1403-4282-9309-21f6e3518dab already exists" \
           -e "Cannot parse projection test_projection" \
           -e "Key expressions cannot contain subqueries" \
           -e "Expression must be deterministic but it contains non-deterministic part" \
           -e "Unknown tokenizer: 'unicode_word'" \
           -e "This engine is deprecated and is not supported in transactions" \
           -e "Prevent converting Nullable type to non-Nullable type inside mutation" \
           -e "e.what() = failed to parse response body" \
           -e "Tuple element name 'null' is reserved" \
           -e "No stream (column1_renamedcolumn1.bin) file checksum for column column1_renamed" \
           -e "No stream (ba1.bin) file checksum for column b" \
           -e "Exception during get topic partitions from Kafka: Local: Broker transport failure" \
    /test_output/datastore-server.upgrade.log \
    | grep -av -e "_repl_01111_.*Mapping for table with UUID" \
    | grep -av -e "Azure::Storage::StorageException.*Not found address of host" \
    | grep -av -e "SystemLogQueue.*Queue had been full" \
    | grep -av -e "TraceCollector.*CANNOT_READ_FROM_FILE_DESCRIPTOR" \
    | grep -av -e "while loading statistics.*ILLEGAL_STATISTICS" \
    | grep -av -e "rdk:FAIL.*Connect to.*failed: Connection refused" \
    | grep -av -e "StorageKafka2.*Exception during get topic partitions from Kafka: Local: Broker transport failure" \
    | grep -av -e "wrong_metadata.*Detaching broken part.*backward incompatibility" \
    | grep -av -e "RaftInstance: session.*failed to read rpc header from socket.*due to error" \
    | grep -av -e "SystemLog.*Failed to flush system log system\.metric_log.*DEADLOCK_AVOIDED" \
    | grep -av -e "Value passed to 'throwIf' function is non-zero" \
    | grep -av -e "PostgreSQLConnectionPool: Connection error.*192\.0\.2\.1., port 5432 failed" \
    | grep -av -e "DatabasePostgreSQL::removeOutdatedTables.*Connection to .192\.0\.2\.1:5432. failed" \
    | grep -av -e "DatabasePostgreSQL::getTablesIterator.*Connection to .192\.0\.2\.1:5432. failed" \
    | grep -av -e "AsyncLoader::worker.*Code: 614.*Connection to .192\.0\.2\.1:5432. failed" \
    | grep -av -e "mysqlxx::Pool.*Failed to connect to MySQL" \
    | grep -av -e "Application: Connection to mysql failed" \
    | grep -av -e "DatabaseMySQL.*Connections to mysql failed" \
    | grep -Fa "<Error>" > /test_output/upgrade_error_messages.txt || true

if [ -s /test_output/upgrade_error_messages.txt ]; then
    echo -e "Error message in datastore-server.log (see upgrade_error_messages.txt)$FAIL$(head_escaped /test_output/upgrade_error_messages.txt)" >> /test_output/test_results.tsv
else
    echo -e "No Error messages after server upgrade$OK" >> /test_output/test_results.tsv
fi

# Remove file upgrade_error_messages.txt if it's empty
[ -s /test_output/upgrade_error_messages.txt ] || rm -f /test_output/upgrade_error_messages.txt

# Grep logs for sanitizer asserts, crashes and other critical errors
check_logs_for_critical_errors

tar -chf /test_output/coordination.tar /var/lib/datastore/coordination ||:

collect_query_and_trace_logs

mv /var/log/datastore-server/stderr.log /test_output/