mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Groene AIandClaude Opus 4.8
33a3f81f13
Address review on #107117. prepare-time-trace.sh collects binary sizes with `find ... | xargs wc -c`. On a build with no readable objects (cross-arch / non-Linux) GNU xargs runs `wc -c` once with no arguments and writes a bare `0` row into binary_sizes.txt. That row does not match the binary_sizes FORMAT Regexp, so insert_build_size_data still failed and the post-hook exited 1 - the exact case this PR set out to fix. The earlier "skip absent or empty artifact" guard did not help because the file was non-empty. Fix at the producer: use `xargs -r` on the size and symbol collection so an empty input set leaves the artifact empty instead of emitting a junk row. With that, an empty artifact reliably means "no such profile data for this build", and the hook can: no-op for builds that produce no data, upload for builds that do, and fail loudly on any genuine processing or upload error (it never swallows one). Regression tests now run the real producer against an empty build dir and assert binary_sizes.txt is empty, model the present-but-empty producer output for the cross-arch case instead of an absent file, and cover that an upload error propagates. Co-Authored-By: Claude Opus 4.8 <[email protected]>
131 lines
4.3 KiB
Bash
Executable File
131 lines
4.3 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# This scripts transforms the output of clang's -ftime-trace JSON files into a format to upload to ClickHouse
|
|
|
|
# Example:
|
|
# mkdir time_trace
|
|
# utils/prepare-time-trace/prepare-time-trace.sh build time_trace
|
|
|
|
# See also https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview
|
|
|
|
true<<///
|
|
CREATE TABLE build_time_trace
|
|
(
|
|
-- Extra columns:
|
|
pull_request_number UInt32,
|
|
commit_sha String,
|
|
check_start_time DateTime,
|
|
check_name LowCardinality(String),
|
|
instance_type LowCardinality(String),
|
|
instance_id String,
|
|
|
|
-- Normal columns:
|
|
file String,
|
|
library LowCardinality(String),
|
|
date Date DEFAULT toDate(time),
|
|
time DateTime64(6),
|
|
pid UInt32,
|
|
tid UInt32,
|
|
ph Enum8('B', 'E', 'X', 'i', 'I', 'C', 'b', 'n', 'e', 'S', 'T', 'p', 'F', 's', 't', 'f', 'P', 'N', 'O', 'D', 'M'),
|
|
ts UInt64,
|
|
dur UInt64,
|
|
cat LowCardinality(String),
|
|
name LowCardinality(String),
|
|
detail String,
|
|
count UInt64,
|
|
avgMs UInt64,
|
|
args_name LowCardinality(String),
|
|
is_total Bool DEFAULT name LIKE 'Total %'
|
|
)
|
|
ENGINE = MergeTree
|
|
ORDER BY (date, file, name, args_name);
|
|
///
|
|
|
|
INPUT_DIR=$1
|
|
OUTPUT_DIR=$2
|
|
|
|
find "$INPUT_DIR" -name '*.json' -or -name '*.time-trace' | grep -P '\.(c|cpp|cc|cxx)\.json|\.time-trace$' | xargs -P "$(nproc)" -I{} bash -c "
|
|
|
|
ORIGINAL_FILENAME=\$(echo '{}' | sed -r -e 's!\.(json|time-trace)\$!!; s!/CMakeFiles/[^/]+\.dir!!')
|
|
LIBRARY_NAME=\$(echo '{}' | sed -r -e 's!^.*/CMakeFiles/([^/]+)\.dir/.*\$!\1!')
|
|
START_TIME=\$(jq '.beginningOfTime' '{}')
|
|
|
|
jq -c '.traceEvents[] | [\"'\"\$ORIGINAL_FILENAME\"'\", \"'\"\$LIBRARY_NAME\"'\", '\$START_TIME', .pid, .tid, .ph, .ts, .dur, .cat, .name, .args.detail, .args.count, .args[\"avg ms\"], .args.name]' '{}' > \"${OUTPUT_DIR}/\$\$\"
|
|
"
|
|
|
|
# Now you can upload it as follows:
|
|
|
|
#cat "$OUTPUT_DIR"/* | clickhouse-client --progress --query "INSERT INTO build_time_trace (extra_column_names, file, library, time, pid, tid, ph, ts, dur, cat, name, detail, count, avgMs, args_name) FORMAT JSONCompactEachRow"
|
|
|
|
# Additionally, collect information about the sizes of translation units
|
|
|
|
true<<///
|
|
CREATE TABLE binary_sizes
|
|
(
|
|
-- Extra columns:
|
|
pull_request_number UInt32,
|
|
commit_sha String,
|
|
check_start_time DateTime,
|
|
check_name LowCardinality(String),
|
|
instance_type LowCardinality(String),
|
|
instance_id String,
|
|
|
|
-- Normal columns:
|
|
file LowCardinality(String),
|
|
library LowCardinality(String) DEFAULT extract(file, 'CMakeFiles/([^/]+)\.dir/'),
|
|
size UInt64,
|
|
date Date DEFAULT toDate(time),
|
|
time DateTime64(6) DEFAULT now64()
|
|
)
|
|
ENGINE = MergeTree
|
|
ORDER BY (date, file, pull_request_number, commit_sha, check_name);
|
|
///
|
|
|
|
# xargs -r: with no matching objects (cross-arch/non-Linux builds) emit no row at
|
|
# all, instead of GNU xargs running 'wc -c' once with no args and writing '0'.
|
|
find "$INPUT_DIR" -type f -executable -or -name '*.o' -or -name '*.a' | grep -v cargo | xargs -r wc -c | grep -v 'total' > "${OUTPUT_DIR}/binary_sizes.txt"
|
|
|
|
# Additionally, collect information about the symbols inside translation units
|
|
true<<///
|
|
CREATE TABLE binary_symbols
|
|
(
|
|
-- Extra columns:
|
|
pull_request_number UInt32,
|
|
commit_sha String,
|
|
check_start_time DateTime,
|
|
check_name LowCardinality(String),
|
|
instance_type LowCardinality(String),
|
|
instance_id String,
|
|
|
|
-- Normal columns:
|
|
file LowCardinality(String),
|
|
library LowCardinality(String) DEFAULT extract(file, 'CMakeFiles/([^/]+)\.dir/'),
|
|
address UInt64,
|
|
size UInt64,
|
|
type FixedString(1),
|
|
symbol LowCardinality(String),
|
|
date Date DEFAULT toDate(time),
|
|
time DateTime64(6) DEFAULT now64()
|
|
)
|
|
ENGINE = MergeTree
|
|
ORDER BY (date, file, symbol, pull_request_number, commit_sha, check_name);
|
|
///
|
|
|
|
# nm does not work with LTO
|
|
if ! grep -q -- '-flto' "$INPUT_DIR/compile_commands.json"
|
|
then
|
|
# Find the best alternative of nm
|
|
for name in llvm-nm-{30..18} llvm-nm nm
|
|
do
|
|
NM=$(command -v ${name})
|
|
[[ -n "${NM}" ]] && break
|
|
done
|
|
|
|
find "$INPUT_DIR" -type f -name '*.o' | grep -v cargo | xargs -P $(nproc) -I {} bash -c "
|
|
${NM} --demangle --defined-only --print-size '{}' | grep -v -P '[0-9a-zA-Z] r ' | sed 's@^@{} @' > '{}.symbols'
|
|
"
|
|
|
|
# xargs -r: with no '*.o.symbols' files emit nothing rather than running cat once.
|
|
find "$INPUT_DIR" -type f -name '*.o.symbols' | xargs -r cat > "${OUTPUT_DIR}/binary_symbols.txt"
|
|
fi
|