datastore/ci/jobs/scripts/fuzzer/run-fuzzer.sh

#!/bin/bash
# shellcheck disable=SC2086,SC2001,SC2046,SC2030,SC2031,SC2010,SC2015

set -x

dmesg --clear ||:

set -e
set -u
set -o pipefail

stage=${stage:-}

repo_dir=/repo

CONFIG_DIR="/etc/datastore-server"

export PATH="$repo_dir/ci/tmp/:$PATH"
export PYTHONPATH=$repo_dir:$repo_dir/ci

cd /workspace

# Direct sanitizer reports to files instead of the server's stderr to avoid 
# losing the report when the server aborts. The runtime appends ".<pid>"
# to `log_path`; reports are merged back in by collect_sanitizer_reports.
# Existing options from the environment/image are preserved.
SANITIZER_LOG_BASE="/workspace/sanitizer.log"
for _san in ASAN TSAN MSAN UBSAN LSAN; do
    _var="${_san}_OPTIONS"
    export "$_var"="${!_var:+${!_var} }log_path=${SANITIZER_LOG_BASE}"
done
unset _san _var

function collect_sanitizer_reports
{
    # Merge sanitizer reports captured via log_path into stderr.log (for the
    # failure parser) and server.log (for context and the OOM grep). Run from an
    # EXIT trap so early `set -e` aborts are covered too; `|| true` keeps the
    # exit code intact.
    local report
    for report in "${SANITIZER_LOG_BASE}".*; do
        [ -e "$report" ] || continue
        echo "Found sanitizer report: $report"
        {
            echo "=== sanitizer report from ${report} ==="
            cat "$report"
            echo
        } | tee -a stderr.log >> server.log || true
    done
}
trap collect_sanitizer_reports EXIT

function configure
{
    chmod +x $repo_dir/ci/tmp/datastore
    # datastore may be compressed - run once to decompress
    $repo_dir/ci/tmp/datastore --query "SELECT 1" ||:
    ln -sf $repo_dir/ci/tmp/datastore $repo_dir/ci/tmp/datastore-server
    ln -sf $repo_dir/ci/tmp/datastore $repo_dir/ci/tmp/datastore-client
    ln -sf $repo_dir/ci/tmp/datastore $repo_dir/ci/tmp/datastore-local
    rm -rf $CONFIG_DIR ||:
    mkdir -p $CONFIG_DIR ||:
    cp -av --dereference "$repo_dir"/programs/server/config* $CONFIG_DIR
    cp -av --dereference "$repo_dir"/programs/server/user* $CONFIG_DIR
    # TODO figure out which ones are needed
    cp -av --dereference "$repo_dir"/tests/config/config.d/listen.xml $CONFIG_DIR/config.d
    cp -av --dereference "$repo_dir"/tests/config/users.d/ci_logs_sender.yaml $CONFIG_DIR/users.d
    cp -av --dereference "$repo_dir"/ci/jobs/scripts/fuzzer/query-fuzzer-tweaks-users.xml $CONFIG_DIR/users.d
    cp -av --dereference "$repo_dir"/ci/jobs/scripts/fuzzer/limit-recursion-settings.xml $CONFIG_DIR/users.d
    cp -av --dereference "$repo_dir"/ci/jobs/scripts/fuzzer/fuzz-server-settings.xml $CONFIG_DIR/config.d

    cat > $CONFIG_DIR/config.d/max_server_memory_usage_to_ram_ratio.xml <<EOL
<datastore>
    <max_server_memory_usage_to_ram_ratio>0.75</max_server_memory_usage_to_ram_ratio>
</datastore>
EOL


    (cd $repo_dir && python3 $repo_dir/ci/jobs/scripts/datastore_proc.py logs_export_config) || echo "Failed to create log export config"
}

function filter_exists_and_template
{
    local path
    for path in "$@"; do
        if [ -e "$path" ]; then
            # SC2001 shellcheck suggests:
            # echo ${path//.sql.j2/.gen.sql}
            # but it doesn't allow to use regex
            echo "$path" | sed 's/\.sql\.j2$/.gen.sql/'
        else
            echo "'$path' does not exist" >&2
        fi
    done
}

function stop_server
{
    datastore-client --query "select elapsed, query from system.processes" ||:
    datastore stop

    # Debug.
    date
    sleep 10
    jobs
    pstree -aspgT
}

function fuzz
{
    $repo_dir/ci/jobs/scripts/fuzzer/generate-test-j2.py --path $repo_dir/tests/queries/0_stateless

    # Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests.
    # Don't overwrite the NEW_TESTS_OPT so that it can be set from the environment.
    NEW_TESTS="$(sed -n 's!\(^tests/queries/0_stateless/.*\.sql\(\.j2\)\?\)$!ch/\1!p' /workspace/ci-changed-files.txt | sort -R)"
    # ci-changed-files.txt contains also files that has been deleted/renamed, filter them out.
    NEW_TESTS="$(filter_exists_and_template $NEW_TESTS)"
    if [[ -n "$NEW_TESTS" ]]
    then
        NEW_TESTS_OPT="${NEW_TESTS_OPT:---interleave-queries-file ${NEW_TESTS}}"
    else
        NEW_TESTS_OPT="${NEW_TESTS_OPT:-}"
    fi

    mkdir -p /var/run/datastore-server

    # server.log -> All server logs, including sanitizer
    # stderr.log -> Process logs (sanitizer) only
    ( datastore-server \
          --config-file $CONFIG_DIR/config.xml \
          --pid-file /var/run/datastore-server/datastore-server.pid \
          --  --path $CONFIG_DIR \
              --logger.console=0 \
              --logger.log=server.log 2>&1 | tee -a stderr.log >> server.log 2>&1
      exit "${PIPESTATUS[0]}" ) &
    server_bg_pid=$!
    # A debug or sanitizer server can take over a minute from fork to listening;
    # give it the same 120s budget as `wait_ready` in `datastore_proc.py`.
    # A dead server is detected early, so the deadline only affects hung servers.
    for _ in {1..120}
    do
        if datastore-client --receive_timeout=5 --query "select 1" || ! kill -0 $server_bg_pid
        then
            break
        fi
        sleep 1
    done
    server_pid=$(cat /var/run/datastore-server/datastore-server.pid)

    kill -0 $server_pid

    IS_ASAN=$(datastore-client --query "SELECT count() FROM system.build_options WHERE name = 'CXX_FLAGS' AND position('sanitize=address' IN value)")
    if [[ "$IS_ASAN" = "1" ]];
    then
        echo "ASAN build detected. Not using gdb since it disables LeakSanitizer detections"
    else
        # Set follow-fork-mode to parent, because we attach to datastore-server, not to watchdog
        # and datastore-server can do fork-exec, for example, to run some bridge.
        # Do not set nostop noprint for all signals, because some it may cause gdb to hang,
        # explicitly ignore non-fatal signals that are used by server.
        # Number of SIGRTMIN can be determined only in runtime.
        RTMIN=$(kill -l SIGRTMIN)
        echo "
    set follow-fork-mode parent
    handle SIGHUP nostop noprint pass
    handle SIGINT nostop noprint pass
    handle SIGQUIT nostop noprint pass
    handle SIGPIPE nostop noprint pass
    handle SIGTERM nostop noprint pass
    handle SIGUSR1 nostop noprint pass
    handle SIGUSR2 nostop noprint pass
    handle SIG$RTMIN nostop noprint pass
    info signals
    continue
    backtrace full
    thread apply all backtrace full
    info registers
    disassemble /s
    up
    disassemble /s
    up
    disassemble /s
    p \"done\"
    detach
    quit
    " > script.gdb

        gdb -batch -command script.gdb -p $server_pid &
        sleep 5
        # gdb will send SIGSTOP, spend some time loading debug info, and then send SIGCONT, wait for it (up to send_timeout, 300s)
        time datastore-client --query "SELECT 'Connected to datastore-server after attaching gdb'" ||:

        # Check connectivity after we attach gdb, because it might cause the server
        # to freeze, and the fuzzer will fail. In debug build, it can take a lot of time.
        for _ in {1..180}
        do
            if datastore-client --receive_timeout=5 --query "select 1"
            then
                break
            fi
            sleep 1
        done
        kill -0 $server_pid # This checks that it is our server that is started and not some other one
    fi

    echo 'Server started and responded.'

    (cd $repo_dir && python3 $repo_dir/ci/jobs/scripts/datastore_proc.py logs_export_start) || echo "Failed to start log exports"

    # Setup arguments for the fuzzer
    FUZZER_OUTPUT_SQL_FILE=''

    if [[ "$FUZZER_TO_RUN" = "AST Fuzzer" ]];
    then
        if [[ -n "${TARGETED_QUERIES_FILE:-}" ]] && [[ -f "${TARGETED_QUERIES_FILE}" ]];
        then
            QUERIES_FILE="$(cat "${TARGETED_QUERIES_FILE}")"
            echo "Using targeted AST fuzzer corpus from ${TARGETED_QUERIES_FILE}"
        else
            QUERIES_FILE=$(find /repo/tests/queries/0_stateless -type f -name "*.sql" | sort -R)
        fi
        if [[ -n "${FUZZER_COMPATIBILITY:-}" ]];
        then
            COMPAT_ARG="--compatibility=${FUZZER_COMPATIBILITY}"
            echo "Using AST fuzzer compatibility setting: ${FUZZER_COMPATIBILITY}"
        else
            COMPAT_ARG=""
        fi
        FUZZER_ARGS="--query-fuzzer-runs=1000 --create-query-fuzzer-runs=50 $COMPAT_ARG --queries-file $QUERIES_FILE $NEW_TESTS_OPT"
    elif [ "$FUZZER_TO_RUN" = "BuzzHouse" ]
    then
        FUZZER_ARGS="--buzz-house-config=fuzz.json"
    else
        >&2 echo "Fuzzer \"$FUZZER_TO_RUN\" unknown, provide either \"AST Fuzzer\" or \"BuzzHouse\""
        exit 1
    fi

    # Convert FUZZ_TIME_LIMIT (e.g. "30m", "60m", "1800s" or plain seconds) into seconds
    # so that the remaining budget can be recomputed between fuzzer passes.
    fuzz_time_limit="${FUZZ_TIME_LIMIT:-30m}"
    case "$fuzz_time_limit" in
        *h) fuzz_budget_seconds=$(( ${fuzz_time_limit%h} * 3600 ));;
        *m) fuzz_budget_seconds=$(( ${fuzz_time_limit%m} * 60 ));;
        *s) fuzz_budget_seconds=$(( ${fuzz_time_limit%s} ));;
        *)  fuzz_budget_seconds=$(( fuzz_time_limit ));;
    esac

    fuzz_started_at=$SECONDS
    fuzzer_exit_code=0
    fuzz_pass=0
    : > fuzzer.log
    while :; do
        fuzz_pass=$((fuzz_pass+1))
        remaining_seconds=$(( fuzz_budget_seconds - (SECONDS - fuzz_started_at) ))
        echo "=== Fuzzer pass $fuzz_pass, remaining time budget: ${remaining_seconds}s ==="

        # Allow the fuzzer to run for some time, giving it a grace period of 5m to finish once the time
        # out triggers. After that, it'll send a SIGKILL to the fuzzer to make sure it finishes within
        # a reasonable time.
        # Bound the parser/AST recursion on the client command line, matching the server-side caps in
        # limit-recursion-settings.xml. The client parses every corpus query locally, and a corpus
        # `SET compatibility=...` reverts max_parser_backtracks to its pre-24.3 default of 0 (unbounded);
        # a command-line value survives that revert, unlike a profile value.
        timeout --verbose --signal TERM --kill-after=5m --preserve-status "${remaining_seconds}s" datastore-client \
            --max_memory_usage_in_client=1000000000 \
            --receive_timeout=10 \
            --receive_data_timeout_ms=10000 \
            --stacktrace \
            --max_parser_backtracks=1000000 \
            --max_parser_depth=1000 \
            $FUZZER_ARGS \
            >> fuzzer.log \
            2>&1 &
        fuzzer_pid=$!
        echo "Fuzzer pid is $fuzzer_pid"

        # We need to give timeout some time to execute the underlying command with that many arguments
        elapsed=0
        maximum=50
        while [[ $elapsed -lt $maximum ]]; do
            if ps -o pid= --ppid "$fuzzer_pid"; then
                echo "Found underlying PID!"
                break;
            else
                echo "Not found. Trying again..."
            fi
            sleep 0.1
            elapsed=$((elapsed+1))
        done

        # The fuzzer_pid belongs to the timeout process.
        actual_fuzzer_pid=$(ps -o pid= --ppid "$fuzzer_pid")

        if [[ "$IS_ASAN" = "1" ]];
        then
            echo "ASAN build detected. Not using gdb since it disables LeakSanitizer detections"
        else
            echo "Attaching gdb to the fuzzer itself"
            gdb -batch -command script.gdb -p $actual_fuzzer_pid &
        fi

        # Wait for the fuzzer to complete.
        # Note that the 'wait || ...' thing is required so that the script doesn't
        # exit because of 'set -e' when 'wait' returns nonzero code.
        fuzzer_exit_code=0
        wait "$fuzzer_pid" || fuzzer_exit_code=$?
        echo "Fuzzer exit code is $fuzzer_exit_code"

        # A non-zero exit code is either the time limit (TERM/KILL sent by `timeout`) or a
        # client/server failure — in both cases stop and let the code below classify it.
        if [[ "$fuzzer_exit_code" != "0" ]]; then
            break
        fi
        # BuzzHouse enforces its own internal time budget and exits 0 when it is reached.
        if [[ "$FUZZER_TO_RUN" != "AST Fuzzer" ]]; then
            break
        fi
        # Exit code 0 means the client walked the whole corpus and exited voluntarily. For the
        # targeted AST fuzzer the corpus is small, so a pass can finish within a couple of
        # minutes of a 30-minute budget. Each pass starts from a fresh random seed, so rerunning
        # the corpus explores new mutations instead of throwing away the remaining budget.
        remaining_seconds=$(( fuzz_budget_seconds - (SECONDS - fuzz_started_at) ))
        if [[ "$remaining_seconds" -lt 60 ]]; then
            echo "Fuzzer finished the corpus and only ${remaining_seconds}s of the budget remain, not restarting"
            break
        fi
        # Make sure the server is still accepting queries before restarting: a fuzzer that
        # exited with code 0 against a dead server would otherwise restart-loop until the budget
        # runs out and hide the failure.
        if ! datastore-client --receive_timeout=5 --query "SELECT 'fuzzer restart liveness check'"; then
            echo "Server is not responding, not restarting the fuzzer"
            break
        fi
        echo "Fuzzer finished the corpus with success, restarting it to use the remaining ${remaining_seconds}s of the budget"
    done

    # If the server dies, most often the fuzzer returns Code 210: Connetion
    # refused, and sometimes also code 32: attempt to read after eof. For
    # simplicity, check again whether the server is accepting connections using
    # datastore-client. We don't check for the existence of the server process, because
    # the process is still present while the server is terminating and not
    # accepting the connections anymore.

    # Default: the loop leaves this unset if it exhausts all retries via the
    # "alive but busy" branches (TOO_MANY_SIMULTANEOUS_QUERIES /
    # MEMORY_LIMIT_EXCEEDED / probe timeout); a dead server sets server_died=1
    # and breaks. A receive/socket timeout means the server is alive but slow to
    # answer (common right after a 30m ASAN fuzz run), not dead -- a dead server
    # returns "Connection refused"/EOF instead -- so count repeated timeouts and
    # only declare a hang once they persist, otherwise a single transient timeout
    # turns a clean (exit 143) run into a bogus "server died" FAIL.
    # BEGIN: server-liveness probe loop (exercised verbatim by
    # ci/tests/test_fuzzer_liveness_loop.py)
    server_died=0
    timeouts=0
    timeouts_max=12

    for _ in {1..100}
    do
        if datastore-client --receive_timeout=5 --query "SELECT 1" 2> err
        then
            server_died=0
            break
        else
            # There are legitimate queries leading to this error, example:
            # SELECT * FROM remote('127.0.0.{1..255}', system, one)
            if grep -F 'TOO_MANY_SIMULTANEOUS_QUERIES' err
            then
                # Give it some time to cool down. The SHOW PROCESSLIST is only a
                # diagnostic and runs under `set -e`; if the same overload rejects
                # it, do not abort the script (that would skip the status.tsv
                # write below and surface as a missing-status job ERROR).
                datastore-client --query "SHOW PROCESSLIST" ||:
                timeouts=0
                sleep 1
            elif grep -F 'MEMORY_LIMIT_EXCEEDED' err
            then
                # Server is alive but at memory limit, give it time to reclaim
                timeouts=0
                sleep 1
            elif grep -F 'Timeout exceeded while' err
            then
                # Alive but slow to answer: retry, and only treat it as a real
                # hang once the timeouts persist (a dead server hits the branch
                # below with "Connection refused"/EOF, not a timeout).
                timeouts=$((timeouts + 1))
                if [[ "$timeouts" -ge "$timeouts_max" ]]
                then
                    echo "Server live check: probe timed out $timeouts times, treating server as hung"
                    cat err
                    server_died=1
                    break
                fi
                sleep 1
            else
                echo "Server live check returns $?"
                cat err
                server_died=1
                break
            fi
        fi
    done
    # END: server-liveness probe loop

    # Stop the server in background so we can wait for the subshell to
    # finish in the foreground. We wait on server_bg_pid (the subshell running
    # the server pipeline) rather than server_pid (from the PID file), because
    # the PID file contains the forked server process which is not a direct
    # child of this shell, so wait would fail with "not a child of this shell".
    # The subshell exits with datastore-server's exit code via PIPESTATUS.
    stop_server &
    server_exit_code=0
    wait $server_bg_pid || server_exit_code=$?
    echo "Server exit code is $server_exit_code"

    echo -e "$server_died\t$server_exit_code\t$fuzzer_exit_code" > status.tsv

    if test -f core.*; then
        zstd --threads=0 core.*
        mv core.*.zst core.zst
    fi
}

case "$stage" in
"")
    ;&  # Did you know? This is "fallthrough" in bash. https://stackoverflow.com/questions/12010686/case-statement-fallthrough
"configure")
    time configure
    ;&
"fuzz")
    time fuzz
    ;&
esac

exit $server_exit_code