Merge remote-tracking branch 'origin/master' into add-executable-udf-drivers

README.md

@@ -42,13 +42,10 @@ Watch all release presentations and videos at [ClickHouse Theater](https://prese
 ## Upcoming Events
 
 Keep an eye out for upcoming meetups and events around the world.
-Somewhere else you want us to be?
-Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com.
+Want to speak? Apply [here](https://forms.gle/3h4XCEENJZ3eaVGy7)
 You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
 
 Upcoming meetups
-* [Toronto Meetup](https://luma.com/d5sytpvo) - June 2nd, 2026
-* [ClickHouse Café @ Snowflake Summit](https://luma.com/clickh-0h6k) - June 3rd, 2026
 * [Stockholm Meetup](https://www.meetup.com/clickhouse-stockholm-user-group/events/314862596/) - June 9th, 2026
 * [LA Happy Hour](https://luma.com/clickh-tshu) - June 10th, 2026
 * [ClickHouse + Hex AI hackathon](https://luma.com/clickh-2ujv)- June 11th, 2026
@@ -57,10 +54,17 @@ Upcoming meetups
 * [ClickHouse Cafe @ Data & AI Summit](https://luma.com/clickh-vrjd) - June 16th, 2026
 * [NY Happy Hour](https://luma.com/odgqf98e) - June 17th, 2026
 * [Seattle Iceberg Meetup](https://luma.com/vwt2i2rs) - June 25th, 2026
+* [KL Meetup](https://luma.com/clickh-8cfv) - June 26th, 2026 
 * [AI Demo Night SF](https://luma.com/clickh-2crf) - July 1st, 2026
+* [Data at scale Amsterdam](https://luma.com/clickh-ha56) - July 7th, 2026
+* [AI Builders Night NY](https://luma.com/clickh-lz8k) - July 8th, 2026
 * [Happy Hour Open Source de Montréal](https://luma.com/clickh-o8up) - July 9th, 2026
+* [AI Builders Night SF](https://luma.com/clickh-gz0r)- July 14th, 2026
+* [Bangkok OSS & Data Evening: Queries, Code & Community](https://luma.com/gpzn0n8v) - July 23, 2026 
+
 
 Recent meetups
+* [Toronto Meetup](https://luma.com/d5sytpvo) - June 2nd, 2026
 * [AI Demo Night NY](https://luma.com/zi2360cm) - May 21st, 2026
 * [Chicago Meetup](https://luma.com/c5evgnbc) - May 19th, 2026
 * [Agentic AI Unplugged: Bengaluru Edition](https://luma.com/AgenticAI-BLR2026) - May 15th, 2026

ci/defs/job_configs.py

@@ -489,6 +489,7 @@ class JobConfigs:
             ],
         ),
         timeout=900,
+        post_hooks=["python3 ./ci/jobs/scripts/job_hooks/docker_clean_up_hook.py"],
     ).parametrize(
         Job.ParamSet(
             parameter="amd_release",
@@ -1378,6 +1379,7 @@ class JobConfigs:
             include_paths=[
                 "./docs",
                 "./ci/jobs/docs_job_mintlify.py",
+                "./ci/jobs/scripts/docs",
             ],
             # Exclude everything currently in ./docs so that this job runs only
             # on files that are NOT part of the legacy docs tree (i.e. the new

ci/jobs/ast_fuzzer_job.py

@@ -307,9 +307,24 @@ def run_fuzz_job(check_name: str):
             sanitizer_oom = Shell.get_output(
                 f"rg --text 'Sanitizer:? (out-of-memory|out of memory|failed to allocate)|Child process was terminated by signal 9' {server_log}"
             )
-            if sanitizer_oom:
+            # Sanitizer shadow memory is invisible to the server's memory tracker,
+            # so the kernel OOM killer may SIGKILL the server before any limit
+            # fires. It may also kill the watchdog, losing the "terminated by
+            # signal 9" message in the server log. A SIGKILLed server (exit 137)
+            # with no sanitizer report is an OOM, not a bug.
+            has_sanitizer_report = any(WORKSPACE_PATH.glob("sanitizer.log.*"))
+            kernel_oom_kill = (
+                server_died and server_exit_code == 137 and not has_sanitizer_report
+            )
+            if sanitizer_oom or kernel_oom_kill:
                 print("Sanitizer OOM")
-                info.append("WARNING: Sanitizer OOM - test considered passed")
+                if sanitizer_oom:
+                    info.append("WARNING: Sanitizer OOM - test considered passed")
+                else:
+                    info.append(
+                        "WARNING: Server was killed by the kernel OOM killer "
+                        "(sanitizer build) - test considered passed"
+                    )
                 status = Result.Status.OK
                 is_failed = False
         else:

ci/jobs/buzzhouse_job.py

@@ -166,7 +166,7 @@ def main():
 
     allow_hardcoded_inserts = random.choice([True, False])
     min_nested_rows = random.randint(0, 5)
-    max_nested_rows = min_nested_rows + (5 if allow_hardcoded_inserts else 100)
+    max_nested_rows = min_nested_rows + (5 if allow_hardcoded_inserts else 30)
     min_insert_rows = random.randint(1, 100)
     max_insert_rows = min_insert_rows + (10 if allow_hardcoded_inserts else 3000)
     min_string_length = random.randint(0, 100)
@@ -270,7 +270,6 @@ def main():
         "hot_table_settings": [
             "allow_coalescing_columns_in_partition_or_order_key",
             # "allow_experimental_replacing_merge_with_cleanup",
-            "allow_experimental_reverse_key",
             "allow_floating_point_partition_key",
             "allow_nullable_key",
             "allow_summing_columns_in_partition_or_order_key",

ci/jobs/docs_job_mintlify.py

@@ -1,34 +1,42 @@
-import os
-
+from ci.jobs.scripts.docs.check_readonly_copies import check_readonly_copies
+from ci.jobs.scripts.docs.mintlify_docs_check import DEFAULT_CHECKS
+from ci.praktika.info import Info
 from ci.praktika.result import Result
-from ci.praktika.utils import Shell, Utils
+from ci.praktika.utils import Utils
+
+
+def _readonly_copies_guard():
+    # One-way sync: fail edits to docs folders whose canonical source lives in
+    # another repo (declared in ci/jobs/scripts/docs/readonly_copies.json). This
+    # is aggregator-only -- the consuming repos are the source of truth, so this
+    # is deliberately not part of the shared DEFAULT_CHECKS.
+    changed_files = Info().get_changed_files()
+    if changed_files is None:
+        # Fail close: without the changed-file list we cannot prove the PR does
+        # not touch read-only copies, so do not report success.
+        print("Error: the changed-file list is unavailable, cannot run the check.")
+        return False
+    return check_readonly_copies(changed_files)
+
 
 if __name__ == "__main__":
 
-    results = []
-    stop_watch = Utils.Stopwatch()
-    temp_dir = f"{Utils.cwd()}/ci/tmp/"
-
     docs_dir = f"{Utils.cwd()}/docs"
 
-    results.append(
-        Result.from_commands_run(
-            name="Verify Mintlify docs.json file is valid",
-            command=[
-                "mint validate",
-            ],
-            workdir=docs_dir,
-        )
-    )
+    # The mint check definitions are shared with the standalone driver
+    # (ci/jobs/scripts/docs/mintlify_docs_check.py). This job already runs inside
+    # the docs-builder image with the docs present natively, so it runs them
+    # directly; add new checks to DEFAULT_CHECKS, not here.
+    results = [
+        Result.from_commands_run(name=name, command=command, workdir=docs_dir)
+        for name, command in DEFAULT_CHECKS
+    ]
 
     results.append(
         Result.from_commands_run(
-            name="Check for broken links",
-            command=[
-                "mint broken-links",
-            ],
-            workdir=docs_dir,
+            name="No direct edits to read-only copied docs",
+            command=_readonly_copies_guard,
         )
     )
 
-    Result.create_from(results=results).complete_job()
+    Result.create_from(results=results).complete_job()

ci/jobs/jepsen_check.py

@@ -178,58 +178,81 @@ def main():
         logging.info("Not jepsen test label in labels list, skipping")
         sys.exit(0)
 
+    image_name = KEEPER_IMAGE_NAME if args.program == "keeper" else SERVER_IMAGE_NAME
+    # Pull by the immutable praktika digest tag instead of mutable ":latest".
+    # Jepsen runs docker directly, so use the architecture-specific tag built by praktika.
+    docker_tag = info.docker_tag(image_name)
+    if docker_tag:
+        if Utils.is_arm():
+            docker_tag += "_arm"
+        elif Utils.is_amd():
+            docker_tag += "_amd"
+        else:
+            raise RuntimeError("Unsupported CPU architecture")
+        docker_image = f"{image_name}:{docker_tag}"
+    else:
+        if not info.is_local_run:
+            raise RuntimeError(
+                f"No praktika digest tag for image [{image_name}] "
+                f"in workflow [{info.workflow_name}]"
+            )
+        logging.warning(
+            "No praktika digest tag for %s in local run; falling back to mutable ':latest' "
+            "(may be served stale through the pull-through cache)",
+            image_name,
+        )
+        docker_image = image_name
+
     temp_path.mkdir(parents=True, exist_ok=True)
 
     result_path = temp_path / "result_path"
     result_path.mkdir(parents=True, exist_ok=True)
 
-    instances = prepare_autoscaling_group_and_get_hostnames(
-        KEEPER_DESIRED_INSTANCE_COUNT
-        if args.program == "keeper"
-        else SERVER_DESIRED_INSTANCE_COUNT
-    )
-    nodes_path = save_nodes_to_file(
-        instances[:KEEPER_DESIRED_INSTANCE_COUNT], temp_path
-    )
-
-    # always use latest
-    docker_image = KEEPER_IMAGE_NAME if args.program == "keeper" else SERVER_IMAGE_NAME
-
-    # build amd_binary assumed to be always ready on the master as it's part of the merge queue workflow
-    urls = read_build_urls(temp_path, "amd_binary")
-    build_url = None
-    for url in urls:
-        if url.endswith("clickhouse"):
-            build_url = url
-    assert build_url, "No build url found in the report"
-
-    extra_args = ""
-    if args.program == "server":
-        extra_args = f"-e KEEPER_NODE={instances[-1]}"
-
-    ssh_key_secret = Secret.Config(
-        name="jepsen_ssh_key", type=Secret.Type.AWS_SSM_PARAMETER
-    )
-    with SSHKey(key_value=ssh_key_secret.get_value() + "\n"):
-        ssh_auth_sock = os.environ["SSH_AUTH_SOCK"]
-        auth_sock_dir = os.path.dirname(ssh_auth_sock)
-        cmd = get_run_command(
-            ssh_auth_sock,
-            auth_sock_dir,
-            info,
-            nodes_path,
-            Utils.cwd(),
-            build_url,
-            result_path,
-            extra_args,
-            docker_image,
+    try:
+        instances = prepare_autoscaling_group_and_get_hostnames(
+            KEEPER_DESIRED_INSTANCE_COUNT
+            if args.program == "keeper"
+            else SERVER_DESIRED_INSTANCE_COUNT
+        )
+        nodes_path = save_nodes_to_file(
+            instances[:KEEPER_DESIRED_INSTANCE_COUNT], temp_path
         )
-        logging.info("Going to run jepsen: %s", cmd)
 
-        if Shell.run(cmd) != 0:
-            logging.error("Jepsen run command failed")
+        # build amd_binary assumed to be always ready on the master as it's part of the merge queue workflow
+        urls = read_build_urls(temp_path, "amd_binary")
+        build_url = None
+        for url in urls:
+            if url.endswith("clickhouse"):
+                build_url = url
+        assert build_url, "No build url found in the report"
 
-    clear_autoscaling_group()
+        extra_args = ""
+        if args.program == "server":
+            extra_args = f"-e KEEPER_NODE={instances[-1]}"
+
+        ssh_key_secret = Secret.Config(
+            name="jepsen_ssh_key", type=Secret.Type.AWS_SSM_PARAMETER
+        )
+        with SSHKey(key_value=ssh_key_secret.get_value() + "\n"):
+            ssh_auth_sock = os.environ["SSH_AUTH_SOCK"]
+            auth_sock_dir = os.path.dirname(ssh_auth_sock)
+            cmd = get_run_command(
+                ssh_auth_sock,
+                auth_sock_dir,
+                info,
+                nodes_path,
+                Utils.cwd(),
+                build_url,
+                result_path,
+                extra_args,
+                docker_image,
+            )
+            logging.info("Going to run jepsen: %s", cmd)
+
+            if Shell.run(cmd) != 0:
+                logging.error("Jepsen run command failed")
+    finally:
+        clear_autoscaling_group()
 
     status = Result.Status.OK
     description = "No invalid analysis found ヽ(‘ー`)ノ"

ci/jobs/scripts/docs/check_readonly_copies.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""
+Guard for one-way-synced docs folders.
+
+Some docs folders in this repo are *copies* whose canonical source lives in
+another repository (e.g. ``docs/integrations/language-clients/python`` is owned
+by ``clickhouse-connect``). Editing the copy here is almost always a mistake --
+the next sync overwrites it. This check fails such edits and points the
+contributor at the canonical source.
+
+The protected folders and their owning repos are declared in
+``readonly_copies.json`` next to this file. The check matches a list of changed
+files (repo-relative paths, as they appear in the PR diff) against those
+folders.
+
+The input must cover *both* sides of renames: a rename out of a protected
+folder is an edit to it (a deletion), and only the source path falls under the
+protected prefix. ``Info().get_changed_files()``, which the Praktika job feeds
+in, includes rename sources; standalone, pipe ``git diff --name-only
+--no-renames`` (with rename detection disabled a rename is listed as a
+deletion plus an addition, so both paths appear).
+"""
+
+import argparse
+import json
+import os
+import sys
+
+CONFIG_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "readonly_copies.json"
+)
+
+
+def load_sources(config_path):
+    with open(config_path, encoding="utf-8") as f:
+        return json.load(f)["sources"]
+
+
+def _normalize(path):
+    return path.strip().removeprefix("./")
+
+
+def find_violations(changed_files, sources):
+    """Return [(source, [offending files])] for every source that was touched."""
+    normalized = [_normalize(f) for f in changed_files]
+    violations = []
+    for src in sources:
+        prefix = src["path"].rstrip("/")
+        hit = [f for f in normalized if f == prefix or f.startswith(prefix + "/")]
+        if hit:
+            violations.append((src, sorted(hit)))
+    return violations
+
+
+def format_message(violations):
+    lines = []
+    for src, files in violations:
+        owner = src.get("repo") or src.get("url", "")
+        subdir = src.get("subdir")
+        where = f" (in its `{subdir}` folder)" if subdir else ""
+        lines.append(f"'{src['path']}' is a one-way copy from {owner}{where}.")
+        if src.get("url"):
+            lines.append(f"  Canonical source: {src['url']}")
+        lines.append(
+            "  Edit it there -- changes made directly here are overwritten by the "
+            "next sync."
+        )
+        lines.append("  Offending files:")
+        lines.extend(f"    {f}" for f in files)
+        lines.append("")
+    return "\n".join(lines).rstrip()
+
+
+def check_readonly_copies(changed_files, config_path=CONFIG_PATH):
+    """Return True if no read-only copied docs were edited, else False."""
+    sources = load_sources(config_path)
+    violations = find_violations(changed_files, sources)
+    if not violations:
+        print("OK: no edits to read-only copied docs.")
+        return True
+    print("Edits to read-only copied docs detected:\n")
+    print(format_message(violations))
+    return False
+
+
+def _read_changed_files(args):
+    if args.changed_file:
+        return args.changed_file
+    return [line.strip() for line in sys.stdin.read().splitlines() if line.strip()]
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--config", default=CONFIG_PATH, help="Path to readonly_copies.json.")
+    p.add_argument(
+        "--changed-file",
+        action="append",
+        default=[],
+        help="A changed file path (repeatable). If omitted, newline-separated "
+        "paths are read from stdin.",
+    )
+    args = p.parse_args(argv)
+    return 0 if check_readonly_copies(_read_changed_files(args), args.config) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ci/jobs/scripts/docs/mintlify_docs_check.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""
+Run the Mintlify docs checks for a slice of a larger docs site.
+
+Run this inside the docs image (``clickhouse/docs-builder``), which provides
+``mint`` along with ``git`` and ``python3``; nothing is installed locally. A
+consuming repo vendors nothing -- its CI runs the image as the job container,
+fetches this one file from the aggregator, and runs it.
+
+Steps:
+  1. Shallow + sparse clone the aggregator docs repo (the docs root and the
+     docs CI scripts at ``ci/jobs/scripts/docs``).
+  2. Replace one or more folders in it with local folders (e.g. drop the
+     locally edited Python client docs over
+     ``integrations/language-clients/python``). The destination is wiped first,
+     so deletions and renames in the local source are reflected -- the docs sync
+     is one-way, so the checks see exactly what the next sync would produce.
+  3. Run the checks from the docs root against the resulting docs.
+
+The check definitions live in ``DEFAULT_CHECKS`` so they are declared once and
+shared: this driver runs them directly, as does the Praktika job
+(``ci/jobs/docs_job_mintlify.py``), which imports the same list. Add a new check
+here and both pick it up.
+
+A check can be any shell command, including ``python3 <script>``. Checks run
+from the docs root, so a Python check script committed at
+``ci/jobs/scripts/docs`` is referenced relative to the docs root --
+``python3 ../ci/jobs/scripts/docs/foo.py`` -- which resolves because the sparse
+clone includes that directory. A consuming repo therefore gets those scripts
+for free from the clone; it never needs to vendor them.
+"""
+
+import argparse
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+
+# The checks to run (name, shell command), executed from the docs root. Kept
+# here, independent of any CI framework, as the single source of truth shared
+# with the Praktika job.
+#
+# A command may invoke a Python check script committed at ci/jobs/scripts/docs,
+# referenced relative to the docs root, e.g.:
+#     ("Frontmatter lint", "python3 ../ci/jobs/scripts/docs/frontmatter_lint.py ."),
+DEFAULT_CHECKS = [
+    ("Validate docs.json", "mint validate"),
+    ("Check for broken links", "mint broken-links"),
+]
+
+
+def run(cmd, **kw):
+    print("+ " + " ".join(shlex.quote(c) for c in cmd), flush=True)
+    return subprocess.run(cmd, **kw)
+
+
+def clone_aggregator(repo, ref, sparse, dest):
+    # Fetch only what the checks need: the tip of `ref` with no history
+    # (--depth 1), only the trees/blobs under the sparse paths (blob:none filter
+    # + sparse-checkout), and nothing else. `git clone` without --depth would
+    # download the aggregator's entire commit history -- enormous for a repo like
+    # ClickHouse -- so build the repo with `git init` + a single shallow fetch
+    # instead, which never materialises history or out-of-slice files.
+    run(["git", "init", "-q", dest], check=True)
+    run(["git", "-C", dest, "remote", "add", "origin", repo], check=True)
+    run(["git", "-C", dest, "config", "remote.origin.promisor", "true"], check=True)
+    run(["git", "-C", dest, "config", "remote.origin.partialclonefilter", "blob:none"], check=True)
+    run(["git", "-C", dest, "sparse-checkout", "init", "--cone"], check=True)
+    run(["git", "-C", dest, "sparse-checkout", "set", *sparse], check=True)
+    run(["git", "-C", dest, "fetch", "--depth", "1", "--filter=blob:none", "origin", ref], check=True)
+    run(["git", "-C", dest, "checkout", "FETCH_HEAD"], check=True)
+
+
+def replace(src, dest):
+    src = os.path.abspath(src)
+    if not os.path.isdir(src):
+        raise FileNotFoundError(f"Replace source is not a directory: {src}")
+    print(f"Replace {dest} with {src}", flush=True)
+    # Wipe dest first: the docs sync is one-way (the consuming repo is the source
+    # of truth), so the checks must see exactly what the next sync produces. A
+    # merge would leave stale files from the cloned aggregator behind, hiding
+    # deletions and renames in the source repo.
+    if os.path.exists(dest):
+        shutil.rmtree(dest)
+    os.makedirs(os.path.dirname(dest) or ".", exist_ok=True)
+    shutil.copytree(src, dest)
+
+
+def resolve_replace_dest(docs_root, dest):
+    # `replace` wipes its destination with rmtree, so a DEST that escapes the
+    # cloned docs tree (absolute, `..`, or a symlink pointing out) would delete
+    # an unrelated directory on the host. Resolve it against the docs root and
+    # refuse anything that lands outside, including the docs root itself.
+    if not dest or os.path.isabs(dest):
+        raise ValueError(
+            f"--replace destination must be a relative path inside the docs "
+            f"root: '{dest}'"
+        )
+    root = os.path.realpath(docs_root)
+    resolved = os.path.realpath(os.path.join(root, dest))
+    if not resolved.startswith(root + os.sep):
+        raise ValueError(
+            f"--replace destination '{dest}' resolves to '{resolved}', "
+            f"outside the docs root '{root}'"
+        )
+    return resolved
+
+
+def run_check(docs_root, name, command):
+    # Checks run from the docs root. A command may reach a sibling CI script via
+    # the docs root (e.g. ../ci/jobs/scripts/docs/foo.py); that path resolves
+    # because the sparse clone includes ci/jobs/scripts/docs. `mint` and friends
+    # come from the docs image this script is expected to run inside.
+    print(f"\n=== {name} ===", flush=True)
+    return run(["sh", "-lc", command], cwd=docs_root).returncode == 0
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--repo", required=True,
+                   help="Aggregator docs repo to shallow/sparse clone.")
+    p.add_argument("--ref", default="HEAD", help="Branch/tag/sha (default: HEAD).")
+    p.add_argument("--sparse", action="append", default=None,
+                   help="Sparse-checkout path (repeatable; default: the docs root "
+                        "plus ci/jobs/scripts/docs).")
+    p.add_argument("--docs-root", default="docs",
+                   help="Dir containing docs.json within the clone (default: docs).")
+    p.add_argument("--replace", action="append", default=[], metavar="SRC:DEST",
+                   help="Replace DEST (relative to docs root) with the local folder "
+                        "SRC, wiping DEST first; repeatable.")
+    p.add_argument("--workdir", help="Where to clone (default: a temp dir).")
+    args = p.parse_args(argv)
+
+    base = args.workdir or tempfile.mkdtemp(prefix="docs-check-")
+    clone_dir = os.path.abspath(os.path.join(base, "aggregator"))
+    sparse = args.sparse or [args.docs_root, "ci/jobs/scripts/docs"]
+    clone_aggregator(args.repo, args.ref, sparse, clone_dir)
+
+    docs_root = os.path.join(clone_dir, args.docs_root)
+    if not os.path.isfile(os.path.join(docs_root, "docs.json")):
+        raise FileNotFoundError(f"No docs.json in docs root: {docs_root}")
+
+    for spec in args.replace:
+        parts = spec.split(":")
+        if len(parts) != 2:
+            raise ValueError(f"Invalid --replace '{spec}'. Expected SRC:DEST.")
+        replace(parts[0], resolve_replace_dest(docs_root, parts[1]))
+
+    results = [(name, run_check(docs_root, name, command))
+               for name, command in DEFAULT_CHECKS]
+
+    print("\n=== Summary ===", flush=True)
+    for name, ok in results:
+        print(f"[{'PASS' if ok else 'FAIL'}] {name}", flush=True)
+    return 0 if all(ok for _, ok in results) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ci/jobs/scripts/docs/readonly_copies.json

@@ -0,0 +1,11 @@
+{
+  "_comment": "Docs folders in this repo that are one-way copies of a canonical source living in another repository. Editing them here is overwritten by the next sync; the readonly-copies guard fails such edits and points contributors at the source. 'path' is repo-relative (as it appears in the PR diff).",
+  "sources": [
+    {
+      "path": "docs/integrations/language-clients/python",
+      "repo": "ClickHouse/clickhouse-connect",
+      "url": "https://github.com/ClickHouse/clickhouse-connect",
+      "subdir": "docs"
+    }
+  ]
+}

ci/jobs/scripts/job_hooks/docker_clean_up_hook.py

@@ -2,9 +2,11 @@ from ci.praktika.utils import Shell
 
 
 def check():
-    print("Remove all images with names starting with clickhouse/clickhouse- (forced)")
+    print(
+        "Remove all images with names starting with clickhouse/clickhouse- or clickhouse/install- (forced)"
+    )
     Shell.check(
-        "docker images --format '{{.Repository}}:{{.Tag}}' | grep '^clickhouse/clickhouse-' | xargs -r docker rmi -f",
+        "docker images --format '{{.Repository}}:{{.Tag}}' | grep -E '^clickhouse/(clickhouse|install)-' | xargs -r docker rmi -f",
         verbose=True,
     )
     print("Clean up non-latest images per each Repository")

ci/jobs/scripts/log_parser.py

@@ -9,6 +9,11 @@ from ci.praktika.utils import Shell
 class FuzzerLogParser:
     UNKNOWN_ERROR = "Unknown error"
     MAX_INLINE_REPRODUCE_COMMANDS = 20
+    SANITIZER_ERROR_PATTERN = (
+        r"(SUMMARY|ERROR|WARNING): [a-zA-Z]+Sanitizer:.*|"
+        r".*[a-zA-Z]+Sanitizer: CHECK failed:.*"
+    )
+    RUNTIME_ERROR_PATTERN = r".*runtime error: .*|.*is located.*"
     SQL_COMMANDS = [
         "SELECT",
         "INSERT",
@@ -63,7 +68,7 @@ class FuzzerLogParser:
             (
                 "Sanitizer",
                 "is_sanitizer_error",
-                r"(SUMMARY|ERROR|WARNING): [a-zA-Z]+Sanitizer:.*",
+                self.SANITIZER_ERROR_PATTERN,
             ),
             ("Logical error", "is_logical_error", r"Logical error.*"),
             (
@@ -74,7 +79,7 @@ class FuzzerLogParser:
             (
                 "Runtime error",
                 "is_sanitizer_error",
-                r".*runtime error: .*|.*is located.*",
+                self.RUNTIME_ERROR_PATTERN,
             ),
             ("SegFault", "is_segfault", r"Segmentation fault.*"),
             (
@@ -147,12 +152,19 @@ class FuzzerLogParser:
                     substring = substring.strip().rstrip(".").strip("'\"")
                     format_message = substring
                 break
+        is_check_failed = bool(
+            error_lines and re.search(r"\w+Sanitizer: CHECK failed:", error_lines[0])
+        )
         # keep all lines before next log line
         for i, line in enumerate(error_lines):
             if "] {" in line and "} <" in line or line.startswith("    #"):
                 # it's a new log line or sanitizer frame - break
                 error_lines = error_lines[:i]
                 break
+            elif is_check_failed and not line.strip():
+                # CHECK failed reports end at the first blank line
+                error_lines = error_lines[:i]
+                break
         error_output = "\n".join(error_lines)
         failed_query = ""
         reproduce_commands = []
@@ -294,7 +306,8 @@ class FuzzerLogParser:
         def _extract_sanitizer_trace(log_file):
             ansi_escape = re.compile(r"\x1b\[[0-9;]*m")
             sanitizer_start = re.compile(
-                r"(==\d+==\s*)?(ERROR|WARNING): \w+Sanitizer:|: runtime error: "
+                r"(==\d+==\s*)?(ERROR|WARNING): \w+Sanitizer:|"
+                r"\b\w+Sanitizer: CHECK failed:|: runtime error: "
             )
             summary_pattern = re.compile(r"SUMMARY: \w+Sanitizer:")
             # ClickHouse log line: "2024.01.15 12:34:56.789 [ 123 ] {id} <Level>"
@@ -308,6 +321,7 @@ class FuzzerLogParser:
             result_lines = []
             in_report = False
             is_runtime_error = False
+            is_check_failed_report = False
             consecutive_blank = 0
 
             for line in all_lines:
@@ -318,12 +332,17 @@ class FuzzerLogParser:
                     if sanitizer_start.search(clean_line):
                         in_report = True
                         is_runtime_error = ": runtime error: " in clean_line
+                        is_check_failed_report = (
+                            "Sanitizer: CHECK failed:" in clean_line
+                        )
                         result_lines.append(stripped)
                         consecutive_blank = 0
                 else:
                     if summary_pattern.search(stripped):
                         result_lines.append(stripped)
                         break
+                    elif is_check_failed_report and not stripped:
+                        break
                     elif not stripped:
                         consecutive_blank += 1
                         if consecutive_blank >= 2:

ci/jobs/scripts/perf/compare.sh

@@ -1178,7 +1178,7 @@ create table ci_checks engine File(TSVWithNamesAndTypes, 'ci-checks.tsv')
         fromUnixTimestamp($CHPC_CHECK_START_TIMESTAMP) check_start_time,
         test_name :: LowCardinality(String) AS test_name ,
         test_status :: LowCardinality(String) AS test_status,
-        test_duration_ms :: UInt64 AS test_duration_ms,
+        test_duration_ms :: Float64 AS test_duration_ms,
         report_url,
         $PR_TO_TEST = 0
             ? 'https://github.com/ClickHouse/ClickHouse/commit/$SHA_TO_TEST'

ci/praktika/gh.py

@@ -83,20 +83,32 @@ class GH:
         assert repo_name
         print(repo_name)
 
+        # Include both sides of renames: .filename is the destination path and
+        # .previous_filename (REST API only, absent for non-renames) is the
+        # source path. Without the source path a rename out of a watched
+        # directory is invisible to consumers such as job filtering and the
+        # read-only docs guard. Rename sources do not exist in the checkout at
+        # HEAD, same as deleted files, which were always included.
+        jq_both_sides = ".filename, (.previous_filename // empty)"
+
         for attempt in range(3):
             # store changed files
             if info.pr_number > 0:
                 exit_code, changed_files_str, err = Shell.get_res_stdout_stderr(
-                    f"gh pr view {info.pr_number} --repo {repo_name} --json files --jq '.files[].path'",
+                    f"gh api repos/{repo_name}/pulls/{info.pr_number}/files --paginate --jq '.[] | {jq_both_sides}'",
                 )
                 assert exit_code == 0, "Failed to retrieve changed files list"
             else:
                 exit_code, changed_files_str, err = Shell.get_res_stdout_stderr(
-                    f"gh api repos/{repo_name}/commits/{sha} | jq -r '.files[].filename'",
+                    f"gh api repos/{repo_name}/commits/{sha} --jq '.files[] | {jq_both_sides}'",
                 )
 
             if exit_code == 0:
-                res = changed_files_str.split("\n") if changed_files_str else []
+                res = (
+                    list(dict.fromkeys(changed_files_str.split("\n")))
+                    if changed_files_str
+                    else []
+                )
                 break
             else:
                 print(

ci/praktika/json.html

@@ -691,16 +691,20 @@
     }
 
     function formatDuration(durationInSeconds, detailed = false) {
-        // Check if the duration is empty, null, or not a number
-        if (!durationInSeconds || isNaN(durationInSeconds)) {
+        // Treat null/undefined/non-numeric as "unknown"; a literal 0 is a
+        // real measurement and flows through the precision ladder below
+        // (otherwise sub-millisecond perf-comparison rows showed up as the
+        // misleading placeholder "00s").
+        if (durationInSeconds === null || durationInSeconds === undefined) {
+            return '00s';
+        }
+        const duration = parseFloat(durationInSeconds);
+        if (isNaN(duration)) {
             return '00s';
         }
 
-        // Ensure duration is a floating-point number
-        const duration = parseFloat(durationInSeconds);
-
-        if (detailed) {
-            // Format in the detailed format with hours, minutes, and seconds
+        if (detailed && duration >= 1) {
+            // H/M/S format for job-level totals (e.g. "1h 23m 45s").
             const hours = Math.floor(duration / 3600);
             const minutes = Math.floor((duration % 3600) / 60);
             const seconds = Math.floor(duration % 60);
@@ -710,20 +714,31 @@
             const formattedSeconds = `${String(seconds).padStart(2, '0')}s`;
 
             return `${formattedHours}${formattedMinutes}${formattedSeconds}`.trim();
-        } else {
-            if (duration < 1) {
-                // Sub-second durations: show milliseconds
-                return `${Math.floor(duration * 1000)}ms`;
-            }
-            // Format in the default format with seconds and milliseconds
-            const seconds = Math.floor(duration);
-            const milliseconds = Math.floor((duration % 1) * 1000);
-
-            const formattedSeconds = String(seconds);
-            const formattedMilliseconds = String(milliseconds).padStart(2, 0).slice(-2);
-
-            return `${formattedSeconds}.${formattedMilliseconds}`;
         }
+
+        // Sub-second precision ladder: drop to a smaller unit when the
+        // next-coarser one would round to zero. Covers true zero too.
+        if (duration < 0.00001) {
+            // < 0.01ms — integer nanoseconds.
+            return `${Math.round(duration * 1e9)}ns`;
+        }
+        if (duration < 0.001) {
+            // < 1ms but >= 0.01ms — two decimals in ms.
+            return `${(duration * 1000).toFixed(2)}ms`;
+        }
+        if (duration < 1) {
+            // Sub-second: integer milliseconds.
+            return `${Math.floor(duration * 1000)}ms`;
+        }
+
+        // >= 1s and not detailed: seconds with two-digit milliseconds.
+        const seconds = Math.floor(duration);
+        const milliseconds = Math.floor((duration % 1) * 1000);
+
+        const formattedSeconds = String(seconds);
+        const formattedMilliseconds = String(milliseconds).padStart(2, 0).slice(-2);
+
+        return `${formattedSeconds}.${formattedMilliseconds}`;
     }
 
     // Function to determine status class based on value

ci/praktika/result.py

@@ -781,12 +781,24 @@ class Result(MetaClasses.Serializable):
             # Covers sanitizer reports ("SUMMARY:") and ClickHouse logical errors.
             _ERROR_PREFIXES = ("SUMMARY:", "Logical error:", "Code: ", "Signal description:")
             crash_info = ""
+            # Some gtests (e.g. the function property fuzzer, gtest_functions_stress) log the
+            # offending call as a runnable SQL query right before crashing, via
+            # logCurrentOperation: "(while executing: SELECT f(...);)". Surface it next to the
+            # error so the report shows a copy-pasteable repro instead of just the message.
+            repro_info = ""
+            _REPRO_MAX_LEN = 8192
             if result.files:
                 log_content = Shell.get_output(f"cat {result.files[0]}", verbose=False)
                 for line in log_content.splitlines():
-                    if any(line.startswith(p) for p in _ERROR_PREFIXES):
+                    if not crash_info and any(line.startswith(p) for p in _ERROR_PREFIXES):
                         crash_info = line
+                    if not repro_info and line.startswith("(while ") and "SELECT " in line:
+                        repro_info = line
+                        if len(repro_info) > _REPRO_MAX_LEN:
+                            repro_info = repro_info[:_REPRO_MAX_LEN] + " ... (truncated, see log)"
+                    if crash_info and repro_info:
                         break
+            crash_info = "\n".join(p for p in (crash_info, repro_info) if p)
             result.info = crash_info or info
             # Synthesize a failed sub-result so the job summary is not empty.
             crashed_test = gtest_filter.rstrip(".*") or "unknown"

ci/tests/test_log_parser.py

@@ -0,0 +1,82 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../.."))
+
+from ci.jobs.scripts.log_parser import FuzzerLogParser
+
+_ASAN_CHECK_FAILED_STDERR = """\
+==2138==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
+AddressSanitizer: CHECK failed: sanitizer_allocator_secondary.h:200 "((nearest_chunk)) < ((h->map_beg + h->map_size))" (0x7b44c2461000, 0x0) (tid=3005)
+    #0 0x55b9822b6021 in __asan::CheckUnwind() asan_rtl.cpp
+    #1 0x55b9822cedcb in __sanitizer::CheckFailed(char const*, int, char const*, unsigned long long, unsigned long long) sanitizer_termination.cpp
+    #2 0x55b9b8db9bc7 in DB::ExceptionKeepingTransform::work() src/Processors/Transforms/ExceptionKeepingTransform.cpp:189:42
+    #3 0x55b9b85a724d in DB::executeJob(DB::ExecutingGraph::Node*, DB::ReadProgressCallback*) src/Processors/Executors/ExecutionThreadContext.cpp:54:28
+    #4 0x55b9b85a724d in DB::ExecutionThreadContext::executeTask() src/Processors/Executors/ExecutionThreadContext.cpp:103:9
+    #5 0x55b9b85728d8 in DB::PipelineExecutor::executeStepImpl(unsigned long, DB::IAcquiredSlot*, std::__1::atomic<bool>*) src/Processors/Executors/PipelineExecutor.cpp:363:26
+    #6 0x55b9b8571ae1 in DB::PipelineExecutor::executeStep(std::__1::atomic<bool>*) src/Processors/Executors/PipelineExecutor.cpp:191:5
+    #7 0x55b9b85cb318 in DB::PushingPipelineExecutor::finish() src/Processors/Executors/PushingPipelineExecutor.cpp:131:47
+    #8 0x7f4ae1fdb8cf  misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
+
+dpkg: error processing package clickhouse-server (--install):
+"""
+
+
+def test_parse_failure_prefers_asan_check_failed_over_server_assertion(tmp_path):
+    server_log = tmp_path / "clickhouse-server.err.log"
+    stderr_log = tmp_path / "stderr.log"
+
+    server_log.write_text(
+        "2026.06.09 00:00:00.000000 [ 1 ] {} <Fatal> Application: "
+        "Assertion 'px != 0' failed.\n",
+        encoding="utf-8",
+    )
+    stderr_log.write_text(
+        "AddressSanitizer: CHECK failed: sanitizer_allocator_secondary.h:200 "
+        "\"((nearest_chunk)) < ((h->map_beg + h->map_size))\" "
+        "(0x7b44c2461000, 0x0) (tid=3005)\n"
+        "    <empty stack>\n"
+        "\n"
+        "dpkg: error processing package clickhouse-server (--install):\n",
+        encoding="utf-8",
+    )
+
+    parser = FuzzerLogParser(
+        server_log=str(server_log),
+        stderr_log=str(stderr_log),
+        fuzzer_log="",
+    )
+
+    result_name, info, files = parser.parse_failure()
+
+    assert result_name == "AddressSanitizer (STID: None)"
+    assert "AddressSanitizer: CHECK failed:" in info
+    assert "Assertion 'px != 0' failed" not in info
+    assert "dpkg" not in info
+    assert files == []
+
+
+def test_parse_failure_asan_check_failed_with_stack_trace(tmp_path):
+    server_log = tmp_path / "clickhouse-server.err.log"
+    stderr_log = tmp_path / "stderr.log"
+
+    server_log.write_text(
+        "2026.06.09 00:00:00.000000 [ 1 ] {} <Fatal> Application: "
+        "Assertion 'px != 0' failed.\n",
+        encoding="utf-8",
+    )
+    stderr_log.write_text(_ASAN_CHECK_FAILED_STDERR, encoding="utf-8")
+
+    parser = FuzzerLogParser(
+        server_log=str(server_log),
+        stderr_log=str(stderr_log),
+        fuzzer_log="",
+    )
+
+    result_name, info, files = parser.parse_failure()
+
+    assert result_name == "AddressSanitizer (STID: 1288-3bd5)"
+    assert "AddressSanitizer: CHECK failed:" in info
+    assert "Assertion 'px != 0' failed" not in info
+    assert "dpkg" not in info
+    assert files == []

cmake/warnings.cmake

@@ -49,6 +49,11 @@ no_warning(missing-noreturn) # too aggressive with no clear benefit, see https:/
 if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 23)
     no_warning(lifetime-safety-intra-tu-suggestions)
     no_warning(lifetime-safety-cross-tu-suggestions)
+    # Too many false positives: it treats mutation of a container captured by
+    # reference in a lambda (e.g. `push_back`, `clear`, `+=`) as invalidating the
+    # captured reference to the container variable itself, while in reality only
+    # references to the container's elements would be invalidated.
+    no_warning(lifetime-safety-invalidation)
 endif ()
 if (ARCH_E2K)
     # disable "use of GNU statement expression extension from macro expansion" warning

contrib/azure-cmake/CMakeLists.txt

@@ -20,6 +20,8 @@ file(GLOB AZURE_SDK_SRC
     "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/*.cpp"
     "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/src/private/*.cpp"
     "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-common/src/*.cpp"
+    "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-files-datalake/src/*.cpp"
+    "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-files-datalake/src/private/*.cpp"
 )
 
 set(AZURE_SDK_INCLUDES
@@ -27,6 +29,7 @@ set(AZURE_SDK_INCLUDES
     "${AZURE_SDK_LIBRARY_DIR}/identity/azure-identity/inc/"
     "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-common/inc/"
     "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/inc/"
+    "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-files-datalake/inc/"
 )
 
 if (CLICKHOUSE_CLOUD)

docs/changelogs/v26.3.13.31-lts.md

@@ -0,0 +1,52 @@
+---
+sidebar_position: 1
+sidebar_label: 2026
+---
+
+# 2026 Changelog
+
+### ClickHouse release v26.3.13.31-lts (27ae4e9fe0e) FIXME as compared to v26.3.12.3-lts (d23c7536b98)
+
+#### Backward Incompatible Change
+* Backported in [#106045](https://github.com/ClickHouse/ClickHouse/issues/106045): The setting `show_data_lake_catalogs_in_system_tables` has been renamed to `show_remote_databases_in_system_tables` and broadened: when its value is 0 (the default), `MySQL` and `PostgreSQL` databases are also hidden from `system.tables`, `system.columns`, and `system.completions`, in addition to data lake catalogs. The old setting name is kept as an alias. [#104416](https://github.com/ClickHouse/ClickHouse/pull/104416) ([Pablo Marcos](https://github.com/pamarcos)).
+
+#### New Feature
+* Backported in [#104562](https://github.com/ClickHouse/ClickHouse/issues/104562): Added `SYSTEM PAUSE VIEW [db.]name` and `SYSTEM PAUSE VIEWS` queries for refreshable materialized views. Unlike `SYSTEM STOP VIEW`, `SYSTEM PAUSE VIEW` does not interrupt the currently running refresh — the in-flight refresh is allowed to complete and only subsequent refreshes are prevented. Undone by `SYSTEM START VIEW` or `SYSTEM START VIEWS`, which now uniformly clear both the stopped and paused states. [#103252](https://github.com/ClickHouse/ClickHouse/pull/103252) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+
+#### Improvement
+* Backported in [#105589](https://github.com/ClickHouse/ClickHouse/issues/105589): Replicated refreshable materialized view in APPEND mode will no longer refresh twice if connection to zookeeper was briefly lost during refresh. [#104051](https://github.com/ClickHouse/ClickHouse/pull/104051) ([Michael Kolupaev](https://github.com/al13n321)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#106003](https://github.com/ClickHouse/ClickHouse/issues/106003): Fix `NOT_FOUND_COLUMN_IN_BLOCK` error when selecting from a VIEW over a table with a normal projection. [#101218](https://github.com/ClickHouse/ClickHouse/pull/101218) ([Amos Bird](https://github.com/amosbird)).
+* Backported in [#105786](https://github.com/ClickHouse/ClickHouse/issues/105786): Fix undefined behavior in `MergeTreeDataPartWriterCompact::cancel` when a stream allocation fails. [#101292](https://github.com/ClickHouse/ClickHouse/pull/101292) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#103373](https://github.com/ClickHouse/ClickHouse/issues/103373): Fixes a case where timezone was not included during the assignment in the ALTER statement. Closes [#101328](https://github.com/ClickHouse/ClickHouse/issues/101328). Related https://github.com/ClickHouse/ClickHouse/pull/100647. [#101403](https://github.com/ClickHouse/ClickHouse/pull/101403) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Backported in [#105074](https://github.com/ClickHouse/ClickHouse/issues/105074): Fix `materialize_skip_indexes_on_merge=false` not suppressing text (full-text) indexes during merge. Previously, only non-text skip indexes (minmax, set, bloom_filter) were suppressed; text indexes continued to be built, wasting CPU and I/O. [#101932](https://github.com/ClickHouse/ClickHouse/pull/101932) ([Groene AI](https://github.com/groeneai)).
+* Backported in [#105510](https://github.com/ClickHouse/ClickHouse/issues/105510): Throw `INCORRECT_DATA` instead of `LOGICAL_ERROR` when client-supplied data in the Native format is truncated or malformed. [#102975](https://github.com/ClickHouse/ClickHouse/pull/102975) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#105673](https://github.com/ClickHouse/ClickHouse/issues/105673): Fix `cast_string_to_date_time_mode` being ignored for `CAST` to `Nullable(DateTime)`. Closes [#101840](https://github.com/ClickHouse/ClickHouse/issues/101840). [#103035](https://github.com/ClickHouse/ClickHouse/pull/103035) ([Pavel Kruglov](https://github.com/Avogar)).
+* Backported in [#105802](https://github.com/ClickHouse/ClickHouse/issues/105802): Fix data corruption when writing Parquet (and other trailer-bearing formats such as ORC and Arrow) to HDFS via `INSERT INTO FUNCTION hdfs(...)`. Since 26.1, `WriteBufferFromHDFS` did not flush its working buffer on `finalize()`, so the last up to `DBMS_DEFAULT_BUFFER_SIZE` bytes of every file were silently lost, including the Parquet `PAR1` footer. Reading such files returned `Not a Parquet file (wrong magic bytes at the end of file)`. [#103268](https://github.com/ClickHouse/ClickHouse/pull/103268) ([Groene AI](https://github.com/groeneai)).
+* Backported in [#105668](https://github.com/ClickHouse/ClickHouse/issues/105668): Long-running `INSERT ... SELECT` and other reads from a `DataLakeCatalog` database with `catalog_type = 'glue'` fail with `Code: 117. DB::Exception: IOError: Code: 499. DB::Exception: Unable to parse ExceptionName: ExpiredToken Message: The provided token has expired. (S3_ERROR) ... While executing ParquetBlockInputFormat: While executing ReadFromObjectStorage. (INCORRECT_DATA)` once the STS session credentials captured at table-metadata load time cross their expiry. [#105381](https://github.com/ClickHouse/ClickHouse/pull/105381) ([Pratima Patel](https://github.com/pratimapatel2008)).
+* Backported in [#105977](https://github.com/ClickHouse/ClickHouse/issues/105977): Fix `BAD_ARGUMENTS` exception "It's a bug! Only integer types are supported by `__bitWrapperFunc`" thrown when a `SELECT` with a `TYPE set(N)` skip index has a WHERE atom whose result type is `Float`, `BFloat16`, or any other non-integer type (for example `WHERE c0 + 0.1` or `WHERE log(c0)`). The skip index now falls back to the regular filter path in that case. [#105384](https://github.com/ClickHouse/ClickHouse/pull/105384) ([Groene AI](https://github.com/groeneai)).
+* Backported in [#105880](https://github.com/ClickHouse/ClickHouse/issues/105880): Fix `dictGetOrDefault` throwing `CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN` when the default argument is a `Nullable` expression and `short_circuit_function_evaluation` is enabled with a mix of found and not-found dictionary keys. [#105461](https://github.com/ClickHouse/ClickHouse/pull/105461) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#105872](https://github.com/ClickHouse/ClickHouse/issues/105872): Fix a server segfault in `uniqStateOrNull` / `uniqStateOrDefault` / `uniqOrNullState` (and similar combinator chains over `uniq`) when used with `GROUP BY ... WITH ROLLUP`, `WITH CUBE`, or `WITH TOTALS` and a Nullable argument. [#105470](https://github.com/ClickHouse/ClickHouse/pull/105470) ([Groene AI](https://github.com/groeneai)).
+* Backported in [#105969](https://github.com/ClickHouse/ClickHouse/issues/105969): Fix SEGFAULT in `singleValueOrNull` deserialization for JSON type. Closes [#103630](https://github.com/ClickHouse/ClickHouse/issues/103630). [#105535](https://github.com/ClickHouse/ClickHouse/pull/105535) ([Pavel Kruglov](https://github.com/Avogar)).
+* Backported in [#105745](https://github.com/ClickHouse/ClickHouse/issues/105745): Fix crash in replxx due to hitting FD_SETSIZE limitation of select() (replace with poll()). Note with enabled SSH/web terminal it may crash the server. [#105559](https://github.com/ClickHouse/ClickHouse/pull/105559) ([Azat Khuzhin](https://github.com/azat)).
+* Backported in [#105965](https://github.com/ClickHouse/ClickHouse/issues/105965): Hive partition value extraction now honors the `cast_string_to_date_time_mode` setting and accepts ISO 8601 timestamps with timezone suffixes (e.g. `+0000`, `+00:00`, `Z`) in partition keys by default. [#105584](https://github.com/ClickHouse/ClickHouse/pull/105584) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#105810](https://github.com/ClickHouse/ClickHouse/issues/105810): Fixed a silent under-count in `SELECT` queries when `use_query_condition_cache = 1` (default). A query of the shape `PREWHERE pk_prefix = X WHERE non_pk IN (...)` against a column with a bloom-filter skip index poisoned the `QueryConditionCache` for the `pk_prefix = X` predicate, so a subsequent benign `SELECT count() ... WHERE pk_prefix = X` returned an incorrect, under-counted result. Affected all 26.x releases. [#104781](https://github.com/ClickHouse/ClickHouse/issues/104781). [#105686](https://github.com/ClickHouse/ClickHouse/pull/105686) ([Groene AI](https://github.com/groeneai)).
+* Backported in [#105834](https://github.com/ClickHouse/ClickHouse/issues/105834): Fix a crash in the `mongodb` table function, MongoDB storage and MongoDB dictionary source when the collection name is empty or contains NUL bytes. [#105776](https://github.com/ClickHouse/ClickHouse/pull/105776) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#106061](https://github.com/ClickHouse/ClickHouse/issues/106061): Fix `Cannot find column` error for distributed queries with `IN Array(...)` filter for the new analyzer. [#105894](https://github.com/ClickHouse/ClickHouse/pull/105894) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#106498](https://github.com/ClickHouse/ClickHouse/issues/106498): Fixed a possible crash due to a too-large string literal sent within the query. [#105996](https://github.com/ClickHouse/ClickHouse/pull/105996) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#106130](https://github.com/ClickHouse/ClickHouse/issues/106130): Fix `system.dictionaries` returning 0 rows with partial `SHOW DICTIONARIES` revoke. [#106105](https://github.com/ClickHouse/ClickHouse/pull/106105) ([Pavel Kruglov](https://github.com/Avogar)).
+* Backported in [#106170](https://github.com/ClickHouse/ClickHouse/issues/106170): Fixed a server crash when querying DeltaLake tables with `allow_experimental_delta_kernel_rs` enabled and a credential or option that contained invalid bytes (the Rust FFI panicked across the `extern "C"` boundary). [#106109](https://github.com/ClickHouse/ClickHouse/pull/106109) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### NO CL ENTRY
+
+* NO CL ENTRY:  'Revert "Backport [#105584](https://github.com/ClickHouse/ClickHouse/issues/105584) to 26.3: Honor cast_string_to_date_time_mode for hive partition values"'. [#106613](https://github.com/ClickHouse/ClickHouse/pull/106613) ([Nikita Taranov](https://github.com/nickitat)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#105932](https://github.com/ClickHouse/ClickHouse/issues/105932): Update mongo-c-driver to 2.3.0. [#104300](https://github.com/ClickHouse/ClickHouse/pull/104300) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#106142](https://github.com/ClickHouse/ClickHouse/issues/106142): Fix azure init params. [#105302](https://github.com/ClickHouse/ClickHouse/pull/105302) ([Konstantin Vedernikov](https://github.com/scanhex12)).
+* Backported in [#105656](https://github.com/ClickHouse/ClickHouse/issues/105656): Fix test_mysql_dotnet_client by anchoring to /testapp. [#105636](https://github.com/ClickHouse/ClickHouse/pull/105636) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#105695](https://github.com/ClickHouse/ClickHouse/issues/105695): This PR modifies the settings history so that compatibility='X.Y' will never revert apply_row_policy_after_final to false. Previously, compatibility levels before 26.2 would revert the setting to false (the buggy pre-fix behavior). After this change, the setting remains true regardless of compatibility level, ensuring users always get the correct row policy application semantics with FINAL. [#105637](https://github.com/ClickHouse/ClickHouse/pull/105637) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fix 04229_defer_partition_pruning_after_final test on 26.3. [#105681](https://github.com/ClickHouse/ClickHouse/pull/105681) ([Nikita Fomichev](https://github.com/fm4v)).
+

docs/changelogs/v26.4.4.38-stable.md

@@ -0,0 +1,55 @@
+---
+sidebar_position: 1
+sidebar_label: 2026
+---
+
+# 2026 Changelog
+
+### ClickHouse release v26.4.4.38-stable (fe0b1d6e9c1) FIXME as compared to v26.4.3.37-stable (a5dd9c96830)
+
+#### Backward Incompatible Change
+* Backported in [#106046](https://github.com/ClickHouse/ClickHouse/issues/106046): The setting `show_data_lake_catalogs_in_system_tables` has been renamed to `show_remote_databases_in_system_tables` and broadened: when its value is 0 (the default), `MySQL` and `PostgreSQL` databases are also hidden from `system.tables`, `system.columns`, and `system.completions`, in addition to data lake catalogs. The old setting name is kept as an alias. [#104416](https://github.com/ClickHouse/ClickHouse/pull/104416) ([Pablo Marcos](https://github.com/pamarcos)).
+
+#### Improvement
+* Backported in [#106583](https://github.com/ClickHouse/ClickHouse/issues/106583): Fixed `PREWHERE` with `IN` subquery on primary key columns not using primary key index for granule pruning, causing full table scans instead of reading only relevant granules. [#102570](https://github.com/ClickHouse/ClickHouse/pull/102570) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Backported in [#105242](https://github.com/ClickHouse/ClickHouse/issues/105242): Replace the `HTTPConnectionPool*TCP{Rcv,Snd}BufBytes_{p50,p75,p90,p95}` async metrics with a bucket-based histogram `http_pool_tcp_buf_bytes` (labels `group`, `direction`) in `system.histogram_metrics`. Per-group total async metrics are preserved. [#103704](https://github.com/ClickHouse/ClickHouse/pull/103704) ([Sema Checherinda](https://github.com/CheSema)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#105366](https://github.com/ClickHouse/ClickHouse/issues/105366): Fix incorrect results for RIGHT ANY JOIN when the right table has multiple rows per key and the output block is split due to size limits. Close [#99431](https://github.com/ClickHouse/ClickHouse/issues/99431). [#102064](https://github.com/ClickHouse/ClickHouse/pull/102064) ([Vladimir Cherkasov](https://github.com/vdimir)).
+* Backported in [#105358](https://github.com/ClickHouse/ClickHouse/issues/105358): Check for stack overflow in Avro reader during nested types deserialization. [#102417](https://github.com/ClickHouse/ClickHouse/pull/102417) ([Pavel Kruglov](https://github.com/Avogar)).
+* Backported in [#105512](https://github.com/ClickHouse/ClickHouse/issues/105512): Throw `INCORRECT_DATA` instead of `LOGICAL_ERROR` when client-supplied data in the Native format is truncated or malformed. [#102975](https://github.com/ClickHouse/ClickHouse/pull/102975) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#105409](https://github.com/ClickHouse/ClickHouse/issues/105409): Fix `StorageObjectStorageQueue` (`S3Queue`, `AzureQueue`) blocking shutdown until partially-processed files were fully read from object storage. The source now aborts the read immediately on shutdown; deduplication ensures rows already streamed into the destination table before shutdown are not duplicated when the file is retried on next start. [#103126](https://github.com/ClickHouse/ClickHouse/pull/103126) ([Tuan Pham Anh](https://github.com/tuanpach)).
+* Backported in [#105804](https://github.com/ClickHouse/ClickHouse/issues/105804): Fix data corruption when writing Parquet (and other trailer-bearing formats such as ORC and Arrow) to HDFS via `INSERT INTO FUNCTION hdfs(...)`. Since 26.1, `WriteBufferFromHDFS` did not flush its working buffer on `finalize()`, so the last up to `DBMS_DEFAULT_BUFFER_SIZE` bytes of every file were silently lost, including the Parquet `PAR1` footer. Reading such files returned `Not a Parquet file (wrong magic bytes at the end of file)`. [#103268](https://github.com/ClickHouse/ClickHouse/pull/103268) ([Groene AI](https://github.com/groeneai)).
+* Backported in [#106284](https://github.com/ClickHouse/ClickHouse/issues/106284): Fix processing of url encoded path in deltaLakeAzure resolves. [#103525](https://github.com/ClickHouse/ClickHouse/pull/103525) ([Smita Kulkarni](https://github.com/SmitaRKulkarni)).
+* Backported in [#105426](https://github.com/ClickHouse/ClickHouse/issues/105426): Disable **`additional_table_filters`** with **parallel replicas** and analyzer and w/o query plan serialization. **Previously it may lead to incorrect results**. [#104296](https://github.com/ClickHouse/ClickHouse/pull/104296) ([Azat Khuzhin](https://github.com/azat)).
+* Backported in [#106242](https://github.com/ClickHouse/ClickHouse/issues/106242): Fix a broken patch part after `ALTER TABLE ... DROP PARTITION ID 'patch-...'` followed by `DETACH`/`ATTACH TABLE`. Previously the empty covering part was written without `partition.dat` and `source_parts.dat`, leading to a `broken-on-start_patch-...` entry in `system.detached_parts` after the next attach or server restart. Closes [#93132](https://github.com/ClickHouse/ClickHouse/issues/93132). Closes [#102103](https://github.com/ClickHouse/ClickHouse/issues/102103). [#104353](https://github.com/ClickHouse/ClickHouse/pull/104353) ([Groene AI](https://github.com/groeneai)).
+* Backported in [#105753](https://github.com/ClickHouse/ClickHouse/issues/105753): `LEFT ANTI JOIN` now correctly returns the right-side join key column with default values for unmatched rows, instead of duplicating the left key value. Previously, `SELECT a.x AS l, b.x AS r FROM a LEFT ANTI JOIN b ON a.x = b.x` could return `r = a.x` for the unmatched rows when it should return the default (`0` for `Int32`, `''` for `String`, `NULL` for `Nullable`). Only the join key column was affected; non-key right columns were already defaulted correctly. The same bug appeared in `RIGHT ANTI JOIN` whenever the optimizer swapped sides. Closes [#99959](https://github.com/ClickHouse/ClickHouse/issues/99959). [#105278](https://github.com/ClickHouse/ClickHouse/pull/105278) ([Groene AI](https://github.com/groeneai)).
+* Backported in [#105665](https://github.com/ClickHouse/ClickHouse/issues/105665): Long-running `INSERT ... SELECT` and other reads from a `DataLakeCatalog` database with `catalog_type = 'glue'` fail with `Code: 117. DB::Exception: IOError: Code: 499. DB::Exception: Unable to parse ExceptionName: ExpiredToken Message: The provided token has expired. (S3_ERROR) ... While executing ParquetBlockInputFormat: While executing ReadFromObjectStorage. (INCORRECT_DATA)` once the STS session credentials captured at table-metadata load time cross their expiry. [#105381](https://github.com/ClickHouse/ClickHouse/pull/105381) ([Pratima Patel](https://github.com/pratimapatel2008)).
+* Backported in [#105979](https://github.com/ClickHouse/ClickHouse/issues/105979): Fix `BAD_ARGUMENTS` exception "It's a bug! Only integer types are supported by `__bitWrapperFunc`" thrown when a `SELECT` with a `TYPE set(N)` skip index has a WHERE atom whose result type is `Float`, `BFloat16`, or any other non-integer type (for example `WHERE c0 + 0.1` or `WHERE log(c0)`). The skip index now falls back to the regular filter path in that case. [#105384](https://github.com/ClickHouse/ClickHouse/pull/105384) ([Groene AI](https://github.com/groeneai)).
+* Backported in [#105621](https://github.com/ClickHouse/ClickHouse/issues/105621): Fixed a heap-buffer-overflow when reading Arrow or ArrowStream files with corrupted intermediate offsets in a binary or string column, and a null-pointer dereference when reading geo-tagged Arrow columns. [#105449](https://github.com/ClickHouse/ClickHouse/pull/105449) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#105882](https://github.com/ClickHouse/ClickHouse/issues/105882): Fix `dictGetOrDefault` throwing `CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN` when the default argument is a `Nullable` expression and `short_circuit_function_evaluation` is enabled with a mix of found and not-found dictionary keys. [#105461](https://github.com/ClickHouse/ClickHouse/pull/105461) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#105874](https://github.com/ClickHouse/ClickHouse/issues/105874): Fix a server segfault in `uniqStateOrNull` / `uniqStateOrDefault` / `uniqOrNullState` (and similar combinator chains over `uniq`) when used with `GROUP BY ... WITH ROLLUP`, `WITH CUBE`, or `WITH TOTALS` and a Nullable argument. [#105470](https://github.com/ClickHouse/ClickHouse/pull/105470) ([Groene AI](https://github.com/groeneai)).
+* Backported in [#105971](https://github.com/ClickHouse/ClickHouse/issues/105971): Fix SEGFAULT in `singleValueOrNull` deserialization for JSON type. Closes [#103630](https://github.com/ClickHouse/ClickHouse/issues/103630). [#105535](https://github.com/ClickHouse/ClickHouse/pull/105535) ([Pavel Kruglov](https://github.com/Avogar)).
+* Backported in [#105746](https://github.com/ClickHouse/ClickHouse/issues/105746): Fix crash in replxx due to hitting FD_SETSIZE limitation of select() (replace with poll()). Note with enabled SSH/web terminal it may crash the server. [#105559](https://github.com/ClickHouse/ClickHouse/pull/105559) ([Azat Khuzhin](https://github.com/azat)).
+* Backported in [#106557](https://github.com/ClickHouse/ClickHouse/issues/106557): Fix a client crash on `CREATE TABLE ... TO INNER UUID '...' ...` when the statement has no `ENGINE` clause. The parser now reports a proper `BAD_ARGUMENTS` error instead of dereferencing a null pointer. [#105579](https://github.com/ClickHouse/ClickHouse/pull/105579) ([Groene AI](https://github.com/groeneai)).
+* Backported in [#105729](https://github.com/ClickHouse/ClickHouse/issues/105729): Hive partition value extraction now honors the `cast_string_to_date_time_mode` setting and accepts ISO 8601 timestamps with timezone suffixes (e.g. `+0000`, `+00:00`, `Z`) in partition keys by default. [#105584](https://github.com/ClickHouse/ClickHouse/pull/105584) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#105812](https://github.com/ClickHouse/ClickHouse/issues/105812): Fixed a silent under-count in `SELECT` queries when `use_query_condition_cache = 1` (default). A query of the shape `PREWHERE pk_prefix = X WHERE non_pk IN (...)` against a column with a bloom-filter skip index poisoned the `QueryConditionCache` for the `pk_prefix = X` predicate, so a subsequent benign `SELECT count() ... WHERE pk_prefix = X` returned an incorrect, under-counted result. Affected all 26.x releases. [#104781](https://github.com/ClickHouse/ClickHouse/issues/104781). [#105686](https://github.com/ClickHouse/ClickHouse/pull/105686) ([Groene AI](https://github.com/groeneai)).
+* Backported in [#105836](https://github.com/ClickHouse/ClickHouse/issues/105836): Fix a crash in the `mongodb` table function, MongoDB storage and MongoDB dictionary source when the collection name is empty or contains NUL bytes. [#105776](https://github.com/ClickHouse/ClickHouse/pull/105776) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#106063](https://github.com/ClickHouse/ClickHouse/issues/106063): Fix `Cannot find column` error for distributed queries with `IN Array(...)` filter for the new analyzer. [#105894](https://github.com/ClickHouse/ClickHouse/pull/105894) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#106549](https://github.com/ClickHouse/ClickHouse/issues/106549): Fixed a server crash that could occur when a query reading from PostgreSQL — via the postgresql table function, the PostgreSQL table engine, or a dictionary with a PostgreSQL source — was cancelled (for example with KILL QUERY) and cancelling the remote PostgreSQL query failed. [#105949](https://github.com/ClickHouse/ClickHouse/pull/105949) ([Rory Shanks](https://github.com/rorylshanks)).
+* Backported in [#106499](https://github.com/ClickHouse/ClickHouse/issues/106499): Fixed a possible crash due to a too-large string literal sent within the query. [#105996](https://github.com/ClickHouse/ClickHouse/pull/105996) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#106132](https://github.com/ClickHouse/ClickHouse/issues/106132): Fix `system.dictionaries` returning 0 rows with partial `SHOW DICTIONARIES` revoke. [#106105](https://github.com/ClickHouse/ClickHouse/pull/106105) ([Pavel Kruglov](https://github.com/Avogar)).
+* Backported in [#106171](https://github.com/ClickHouse/ClickHouse/issues/106171): Fixed a server crash when querying DeltaLake tables with `allow_experimental_delta_kernel_rs` enabled and a credential or option that contained invalid bytes (the Rust FFI panicked across the `extern "C"` boundary). [#106109](https://github.com/ClickHouse/ClickHouse/pull/106109) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#106455](https://github.com/ClickHouse/ClickHouse/issues/106455): Continuation of https://github.com/ClickHouse/ClickHouse/pull/96754 but for delta lake. [#106115](https://github.com/ClickHouse/ClickHouse/pull/106115) ([Konstantin Vedernikov](https://github.com/scanhex12)).
+* Backported in [#106536](https://github.com/ClickHouse/ClickHouse/issues/106536): Fix sporadic incorrect results for `ORDER BY ... DESC` queries when reading wide parts in reverse order with `read_in_order_use_virtual_row_per_block = 1` and a small `max_block_size`. [#106429](https://github.com/ClickHouse/ClickHouse/pull/106429) ([Vladimir Cherkasov](https://github.com/vdimir)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#105934](https://github.com/ClickHouse/ClickHouse/issues/105934): Update mongo-c-driver to 2.3.0. [#104300](https://github.com/ClickHouse/ClickHouse/pull/104300) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#105576](https://github.com/ClickHouse/ClickHouse/issues/105576): Use an explicit flag in Context for secondary DDL/ON CLUSTER queries instead of `SECONDARY_QUERY`. [#104881](https://github.com/ClickHouse/ClickHouse/pull/104881) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#106144](https://github.com/ClickHouse/ClickHouse/issues/106144): Fix azure init params. [#105302](https://github.com/ClickHouse/ClickHouse/pull/105302) ([Konstantin Vedernikov](https://github.com/scanhex12)).
+* Backported in [#105599](https://github.com/ClickHouse/ClickHouse/issues/105599): Skip test_numbers_check on release branches. [#105492](https://github.com/ClickHouse/ClickHouse/pull/105492) ([Nikita Fomichev](https://github.com/fm4v)).
+* Backported in [#105658](https://github.com/ClickHouse/ClickHouse/issues/105658): Fix test_mysql_dotnet_client by anchoring to /testapp. [#105636](https://github.com/ClickHouse/ClickHouse/pull/105636) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#105697](https://github.com/ClickHouse/ClickHouse/issues/105697): This PR modifies the settings history so that compatibility='X.Y' will never revert apply_row_policy_after_final to false. Previously, compatibility levels before 26.2 would revert the setting to false (the buggy pre-fix behavior). After this change, the setting remains true regardless of compatibility level, ensuring users always get the correct row policy application semantics with FINAL. [#105637](https://github.com/ClickHouse/ClickHouse/pull/105637) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* On release branches `cast_string_to_date_time_mode` defaults to `basic` (master defaults to `best_effort`), so the timezone-suffix partition queries must pin `best_effort` explicitly. Mirrors the adaptation already applied for 25.12 and 26.2. [#106493](https://github.com/ClickHouse/ClickHouse/pull/106493) ([Jimmy Aguilar Mena](https://github.com/Ergus)).
+

docs/en/development/backports.md

@@ -62,6 +62,8 @@ An active release branch is any branch whose corresponding release PR (carrying
 
 A release branch can be in a **rolling-out** state (its release PR carries the `rolling-out` label) during the period when a new release is being deployed. General backports are paused for rolling-out branches to avoid complicating the rollout. Version-specific labels (e.g. `v25.3-must-backport`) override this and force backporting even during a rollout.
 
+A version-specific label sets the *oldest* release the PR must reach: it is backported to that release **and to every newer active release branch**, not only to the named one. For example, `v25.3-must-backport` on a PR merged into the development branch backports to `25.3` and to every active release after it (`25.4`, `25.5`, …). If multiple version-specific labels are present, the lowest version wins, since it already covers the newer ones.
+
 ## Implementation {#implementation}
 
 ### Overview {#overview}
@@ -82,7 +84,7 @@ Labels on the original PR control whether and where backporting happens.
 | `pr-must-backport` | Backport to all active release branches (skipping branches marked `rolling-out`) |
 | `pr-must-backport-force` | Backport to all active release branches, ignoring `rolling-out` restrictions |
 | `pr-critical-bugfix` | Triggers `pr-must-backport` automatically (via `AUTO_BACKPORT` in `pr_labels_and_category.py`) |
-| `v{VER}-must-backport` (e.g. `v25.3-must-backport`) | Backport only to that specific release branch; overrides `rolling-out` skip for that branch |
+| `v{VER}-must-backport` (e.g. `v25.3-must-backport`) | Backport to that release branch **and every newer active release branch** — the version marks the *oldest* release the PR must reach. With several such labels, the lowest version wins. Overrides the `rolling-out` skip for those branches |
 | `pr-backports-created` | Set by the bot when all required backport PRs have been created; cleared if a cherry-pick PR is reopened |
 | `pr-cherrypick` | Applied to cherry-pick PRs created by the bot |
 | `pr-backport` | Applied to backport PRs created by the bot |
@@ -114,7 +116,7 @@ For each original PR number `N` and release branch `release/X.Y`:
 
 #### 3. Rolling-out branch handling {#rolling-out-branch-handling}
 
-When a release PR carries the `rolling-out` label, general backport labels (`pr-must-backport`, `pr-critical-bugfix`) skip that branch. The bot closes any previously created cherry-pick or backport PRs for that branch with an explanatory comment. A version-specific label (e.g. `v25.3-must-backport`) always overrides this. `pr-must-backport-force` bypasses the `rolling-out` check for all branches.
+When a release PR carries the `rolling-out` label, general backport labels (`pr-must-backport`, `pr-critical-bugfix`) skip that branch. The bot closes any previously created cherry-pick or backport PRs for that branch with an explanatory comment. A version-specific label (e.g. `v25.3-must-backport`) always overrides this — for the named release and for every newer active release branch it expands to. `pr-must-backport-force` bypasses the `rolling-out` check for all branches.
 
 #### 4. Cherry-pick stage (`ReleaseBranch.create_cherrypick`) {#cherry-pick-stage}
 

docs/en/engines/table-engines/integrations/s3queue.md

@@ -338,11 +338,11 @@ For 'Ordered' mode. Available since `24.6`. If there are several replicas of S3Q
 
 By default S3Queue table has always used ephemeral processing nodes, which could lead to duplicates in data in case zookeeper session expires before S3Queue commits processed files in zookeeper, but after it has started processing. This setting forces the server to eliminate possibility of duplicates in case of expired keeper session.
 
-### `persistent_processing_nodes_ttl_seconds` {#persistent_processing_nodes_ttl_seconds}
+### `persistent_processing_node_ttl_seconds` {#persistent_processing_node_ttl_seconds}
 
-In case of non-graceful server termination, it is possible that if `use_persistent_processing_nodes` is enabled, we can have not removed processing nodes. This setting defines a period of time when these processing nodes can safely be cleaned up.
+In case of non-graceful server termination, it is possible that if `use_persistent_processing_nodes` is enabled, we can have not removed processing nodes. This setting defines a period of time when these processing nodes can safely be cleaned up. The same TTL is also used for the bucket lock in `Ordered` mode, which can be held for a longer time than a single processing node, so the value should account for that as well.
 
-Default value: `3600` (1 hour).
+Default value: `21600` (6 hours).
 
 ## S3-related settings {#s3-settings}
 

docs/en/interfaces/formats/Arrow/Arrow.md

@@ -15,8 +15,9 @@ doc_type: 'reference'
 
 ## Description {#description}
 
-[Apache Arrow](https://arrow.apache.org/) comes with two built-in columnar storage formats. ClickHouse supports read and write operations for these formats.
-`Arrow` is Apache Arrow's "file mode" format. It is designed for in-memory random access.
+[Apache Arrow](https://arrow.apache.org/) comes with two built-in columnar storage formats.
+ClickHouse supports read and write operations for these formats.
+`Arrow` is Apache Arrow's "file mode" format, designed for in-memory random access.
 
 ## Data types matching {#data-types-matching}
 
@@ -65,20 +66,66 @@ The data types of ClickHouse table columns do not have to match the correspondin
 
 ## Example usage {#example-usage}
 
-### Inserting data {#inserting-data}
-
-You can insert Arrow data from a file into ClickHouse table using the following command:
-
-```bash
-$ cat filename.arrow | clickhouse-client --query="INSERT INTO some_table FORMAT Arrow"
-```
+In the example below we use the `forex` dataset available in the
+[ClickHouse SQL playground](https://sql.clickhouse.com).
 
 ### Selecting data {#selecting-data}
 
-You can select data from a ClickHouse table and save it into some file in the Arrow format using the following command:
+We select one day of `EUR/USD` exchange rates from the playground and save it
+into a local `forex_eurusd.arrow` file. We query the playground over the HTTP
+interface, where the host is `sql-clickhouse.clickhouse.com` and the user is
+`demo` (which has no password):
 
 ```bash
-$ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Arrow" > {filename.arrow}
+curl "https://sql-clickhouse.clickhouse.com:8443/?user=demo&database=forex" \
+    --data-binary "
+        SELECT
+            concat(base, '.', quote) AS base_quote,
+            datetime AS last_update,
+            CAST(bid, 'Float32') AS bid,
+            CAST(ask, 'Float32') AS ask,
+            ask - bid AS spread
+        FROM forex
+        WHERE base = 'EUR' AND quote = 'USD'
+            AND datetime >= '2020-01-01' AND datetime < '2020-01-02'
+        ORDER BY datetime ASC
+        FORMAT Arrow
+        SETTINGS output_format_arrow_compression_method='zstd'" > forex_eurusd.arrow
+```
+
+### Reading the file back {#reading-data}
+
+We can now read the local Arrow file back with
+[`clickhouse-local`](/operations/utilities/clickhouse-local) using the
+[`file`](/sql-reference/table-functions/file) table function. The file is
+self-describing, so the `Arrow` format infers the schema automatically:
+
+```bash
+clickhouse-local --query "
+    SELECT *
+    FROM file('forex_eurusd.arrow', Arrow)
+    ORDER BY last_update ASC
+    LIMIT 5
+    FORMAT PrettyCompact"
+```
+
+```response title="Response"
+   ┌─base_quote─┬─────────────last_update─┬─────bid─┬─────ask─┬────────────────spread─┐
+1. │ EUR.USD    │ 2020-01-01 17:00:00.065 │  1.1212 │ 1.12172 │ 0.0005199909210205078 │
+2. │ EUR.USD    │ 2020-01-01 17:00:10.447 │  1.1212 │ 1.12192 │ 0.0007200241088867188 │
+3. │ EUR.USD    │ 2020-01-01 17:00:10.498 │ 1.12117 │ 1.12161 │ 0.0004400014877319336 │
+4. │ EUR.USD    │ 2020-01-01 17:00:12.579 │  1.1212 │ 1.12161 │ 0.0004100799560546875 │
+5. │ EUR.USD    │ 2020-01-01 17:00:12.630 │  1.1212 │ 1.12172 │ 0.0005199909210205078 │
+   └────────────┴─────────────────────────┴─────────┴─────────┴───────────────────────┘
+```
+
+### Inserting data {#inserting-data}
+
+To load an Arrow file into a ClickHouse table, pipe it into `clickhouse-client`
+with `FORMAT Arrow`:
+
+```bash
+cat forex_eurusd.arrow | clickhouse-client --query="INSERT INTO some_table FORMAT Arrow"
 ```
 
 ## Format settings {#format-settings}

docs/en/interfaces/formats/Arrow/ArrowStream.md

@@ -19,4 +19,123 @@ doc_type: 'reference'
 
 ## Example usage {#example-usage}
 
+In the example below we use the `forex` dataset which is available in the
+[ClickHouse SQL playground](https://sql.clickhouse.com). You can connect to it
+remotely with `clickhouse-client` using the host `sql-clickhouse.clickhouse.com`
+and the user `demo` (which has no password). The `forex` table lives in the
+`forex` database, so we select it as the default database:
+
+```bash
+clickhouse-client --secure --host sql-clickhouse.clickhouse.com --user demo --database forex
+```
+
+The `forex` table stores currency exchange rates. We can inspect its size and
+how well it compresses on disk by querying [`system.columns`](/operations/system-tables/columns):
+
+```sql title="Query"
+SELECT
+    table,
+    formatReadableSize(sum(data_compressed_bytes)) AS compressed_size,
+    formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed_size,
+    sum(data_compressed_bytes) / sum(data_uncompressed_bytes) AS compression_ratio
+FROM system.columns
+WHERE (database = 'forex') AND (table = 'forex')
+GROUP BY table
+ORDER BY table ASC
+```
+
+```response title="Response"
+   ┌─table─┬─compressed_size─┬─uncompressed_size─┬───compression_ratio─┐
+1. │ forex │ 63.69 GiB       │ 280.48 GiB        │ 0.22708227109363446 │
+   └───────┴─────────────────┴───────────────────┴─────────────────────┘
+```
+
+Unlike the [`Arrow`](/interfaces/formats/Arrow) "file mode" format, which
+requires the whole result before it can be read, `ArrowStream` is delivered as a
+sequence of record batches that a consumer can read incrementally as they
+arrive. This makes it well suited to streaming a query result straight into a
+visualization or analytics tool without first materializing the entire dataset.
+
+To stream the result, send the query over ClickHouse's HTTP interface with a
+`POST` request and read the response as an Arrow stream. We disable compression
+of the Arrow output via the
+[`output_format_arrow_compression_method`](/operations/settings/formats#output_format_arrow_compression_method)
+setting so that consumers can decode batches directly as they are received.
+
+The `ArrowStream` output is raw binary, so rather than printing it to the
+terminal we pipe it into a consumer. The stream is self-describing (it carries
+its own schema), so here we pipe it straight into
+[`clickhouse-local`](/operations/utilities/clickhouse-local), which reads the
+incoming batches with `--input-format ArrowStream` and queries them as a table.
+The `forex` table is large, so we bound the remote query with a `WHERE`
+predicate and a `LIMIT` to keep this example small:
+
+```bash
+curl "https://sql-clickhouse.clickhouse.com:8443/?user=demo&database=forex" \
+    --data-binary "
+        SELECT
+            concat(base, '.', quote) AS base_quote,
+            datetime AS last_update,
+            CAST(bid, 'Float32') AS bid,
+            CAST(ask, 'Float32') AS ask,
+            ask - bid AS spread
+        FROM forex
+        WHERE base = 'USD' AND quote = 'CHF'
+        ORDER BY datetime ASC
+        LIMIT 5
+        FORMAT ArrowStream
+        SETTINGS output_format_arrow_compression_method='none'" \
+  | clickhouse-local --input-format ArrowStream \
+      --query "SELECT * FROM table ORDER BY last_update ASC FORMAT PrettyCompact"
+```
+
+```response title="Response"
+   ┌─base_quote─┬─────────────last_update─┬────bid─┬────ask─┬────────────────spread─┐
+1. │ USD.CHF    │ 2000-05-30 17:23:44.000 │  1.688 │ 1.6885 │ 0.0005000829696655273 │
+2. │ USD.CHF    │ 2000-05-30 17:23:46.000 │ 1.6885 │  1.689 │ 0.0004999637603759766 │
+3. │ USD.CHF    │ 2000-05-30 17:23:48.000 │ 1.6886 │ 1.6891 │ 0.0005000829696655273 │
+4. │ USD.CHF    │ 2000-05-30 17:23:49.000 │ 1.6888 │ 1.6893 │ 0.0004999637603759766 │
+5. │ USD.CHF    │ 2000-05-30 17:24:45.000 │  1.689 │ 1.6895 │ 0.0004999637603759766 │
+   └────────────┴─────────────────────────┴────────┴────────┴───────────────────────┘
+```
+
+The same stream can be consumed incrementally by any Arrow-aware client, which
+reads it batch-by-batch rather than buffering the result in full. For example,
+using the [Apache Arrow JavaScript library](https://arrow.apache.org/docs/js/), a
+`RecordBatchReader` yields each record batch as soon as it is streamed from the
+server:
+
+```js
+const reader = await RecordBatchReader.from(response);
+await reader.open();
+for await (const recordBatch of reader) {
+    const batchTable = new Table(recordBatch);
+    const ipcStream = tableToIPC(batchTable, 'stream');
+    const bytes = new Uint8Array(ipcStream);
+    table.update(bytes);
+}
+```
+
+For a full walkthrough of streaming `ArrowStream` data from ClickHouse into a
+real-time visualization with [Perspective](https://perspective.finos.org/), see
+the blog post
+[Streaming real-time visualizations with ClickHouse, Apache Arrow and Perspective](https://clickhouse.com/blog/streaming-real-time-visualizations-clickhouse-apache-arrow-perpsective).
+
 ## Format settings {#format-settings}
+
+`ArrowStream` shares the same format settings as the [`Arrow`](/interfaces/formats/Arrow) format.
+
+| Setting                                                                      | Description                                                                                                                                | Default     |
+|------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|-------------|
+| `input_format_arrow_allow_missing_columns`                                   | Allow missing columns while reading Arrow input formats                                                                                    | `1`         |
+| `input_format_arrow_case_insensitive_column_matching`                        | Ignore case when matching Arrow columns with CH columns.                                                                                   | `0`         |
+| `input_format_arrow_import_nested`                                           | Obsolete setting, does nothing.                                                                                                            | `0`         |
+| `input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference` | Skip columns with unsupported types while schema inference for format Arrow                                                                | `0`         |
+| `output_format_arrow_compression_method`                                     | Compression method for Arrow output format. Supported codecs: lz4_frame, zstd, none (uncompressed)                                         | `lz4_frame` |
+| `output_format_arrow_date_as_uint16`                                         | Write Date values as plain 16-bit numbers (read back as UInt16), instead of converting to a 32-bit Arrow DATE32 type (read back as Date32). | `0`         |
+| `output_format_arrow_fixed_string_as_fixed_byte_array`                       | Use Arrow FIXED_SIZE_BINARY type instead of Binary for FixedString columns.                                                                | `1`         |
+| `output_format_arrow_low_cardinality_as_dictionary`                          | Enable output LowCardinality type as Dictionary Arrow type                                                                                 | `0`         |
+| `output_format_arrow_string_as_string`                                       | Use Arrow String type instead of Binary for String columns                                                                                 | `1`         |
+| `output_format_arrow_unsupported_types_as_binary`                            | Output types having no conversion as raw binary data. If false - such types would raise UNKNOWN_TYPE exception.                             | `1`         |
+| `output_format_arrow_use_64_bit_indexes_for_dictionary`                      | Always use 64 bit integers for dictionary indexes in Arrow format                                                                          | `0`         |
+| `output_format_arrow_use_signed_indexes_for_dictionary`                      | Use signed integers for dictionary indexes in Arrow format                                                                                 | `1`         |

docs/en/interfaces/formats/HiveText.md

@@ -1,13 +1,126 @@
 ---
+alias: []
 description: 'Documentation for the HiveText format'
+input_format: true
 keywords: ['HiveText']
+output_format: false
 slug: /interfaces/formats/HiveText
 title: 'HiveText'
 doc_type: 'reference'
 ---
 
+| Input | Output | Alias |
+|-------|--------|-------|
+| ✔     | ✗      |       |
+
 ## Description {#description}
 
+`HiveText` reads the text serialization format used by [Apache Hive](https://hive.apache.org/)
+tables (the format produced by Hive's `LazySimpleSerDe`). It is a delimited text
+format, similar to [`CSV`](/interfaces/formats/CSV), in which fields are
+separated by the Hive default `\x01` (Ctrl-A) delimiter. The field delimiter is
+configurable via [`input_format_hive_text_fields_delimiter`](#format-settings).
+
+`HiveText` is an input-only format. The data has no header row: values are
+mapped positionally onto the columns of the destination table, so the column
+names and types are taken from the table (or from an explicitly provided
+structure) rather than inferred from the data. While reading, ClickHouse parses
+dates and times in best-effort mode (see [`date_time_input_format`](/operations/settings/formats#date_time_input_format)),
+fills omitted trailing fields with column defaults, and skips fields it does not
+recognize.
+
+Within a field, values are parsed using the same escaping rules as `CSV` rather
+than Hive's nested delimiters. In particular, a column of type
+[`Array`](/sql-reference/data-types/array) is read from the bracketed
+representation (for example, `"['a','b','c']"`), not from values separated by
+the Hive collection delimiter `\x02`.
+
+:::note Nested delimiter settings have no effect
+The [`input_format_hive_text_collection_items_delimiter`](#format-settings) and
+[`input_format_hive_text_map_keys_delimiter`](#format-settings) settings are
+accepted for compatibility but are currently not used during parsing.
+:::
+
+By default, rows are allowed to have a variable number of fields (see
+[`input_format_hive_text_allow_variable_number_of_columns`](#format-settings)):
+rows with fewer fields than the table have the missing columns filled with
+default values, and rows with extra trailing fields have the extras skipped.
+
 ## Example usage {#example-usage}
 
-## Format settings {#format-settings}
+The examples below override the default field delimiter with a comma (`,`) using
+[`input_format_hive_text_fields_delimiter`](#format-settings) so that the input
+files are easy to read.
+
+### Reading a HiveText file {#reading-data}
+
+Given a file `hive_data.txt` with comma-separated fields:
+
+```text title="hive_data.txt"
+1,3
+3,5,9
+```
+
+We create a table that defines the column names and types, and insert the file
+into it with `FORMAT HiveText`:
+
+```sql title="Query"
+CREATE TABLE test_tbl (a UInt16, b UInt32, c UInt32) ENGINE = MergeTree ORDER BY a;
+
+INSERT INTO test_tbl FROM INFILE 'hive_data.txt'
+SETTINGS input_format_hive_text_fields_delimiter = ','
+FORMAT HiveText;
+
+SELECT * FROM test_tbl;
+```
+
+```response title="Response"
+┌─a─┬─b─┬─c─┐
+│ 1 │ 3 │ 0 │
+│ 3 │ 5 │ 9 │
+└───┴───┴───┘
+```
+
+Note that the first row, `1,3`, has only two fields, so the missing column `c`
+is filled with its default value `0`.
+
+### Variable number of columns {#variable-number-of-columns}
+
+With the default `input_format_hive_text_allow_variable_number_of_columns = 1`,
+rows that have more fields than the table simply have the extra trailing fields
+skipped:
+
+```text title="hive_extras.txt"
+1,2,3,4,5
+6,7,8
+```
+
+```sql title="Query"
+CREATE TABLE test_extras (a UInt16, b UInt32, c UInt32) ENGINE = MergeTree ORDER BY a;
+
+INSERT INTO test_extras FROM INFILE 'hive_extras.txt'
+SETTINGS input_format_hive_text_fields_delimiter = ','
+FORMAT HiveText;
+
+SELECT * FROM test_extras ORDER BY a;
+```
+
+```response title="Response"
+┌─a─┬─b─┬─c─┐
+│ 1 │ 2 │ 3 │
+│ 6 │ 7 │ 8 │
+└───┴───┴───┘
+```
+
+Setting `input_format_hive_text_allow_variable_number_of_columns = 0` instead
+enforces a strict field count, and a row with fewer fields than the table raises
+a parsing exception.
+
+## Format settings {#format-settings}
+
+| Setting                                                | Description                                                                                                                           | Default |
+|--------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|---------|
+| `input_format_hive_text_fields_delimiter`              | Delimiter between fields in Hive Text File                                                                                             | `\x01`  |
+| `input_format_hive_text_collection_items_delimiter`    | Delimiter between collection (array or map) items in Hive Text File. Accepted but currently not used during parsing.                   | `\x02`  |
+| `input_format_hive_text_map_keys_delimiter`            | Delimiter between a pair of map key/values in Hive Text File. Accepted but currently not used during parsing.                          | `\x03`  |
+| `input_format_hive_text_allow_variable_number_of_columns` | Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields as default values        | `1`     |

docs/en/interfaces/formats/Values.md

@@ -35,6 +35,66 @@ This is the format that is used in `INSERT INTO t VALUES ...`, but you can also
 
 ## Example usage {#example-usage}
 
+### Inserting data {#inserting-data}
+
+The `Values` format is what `INSERT` uses, so any `INSERT ... VALUES` statement
+is already using it. The `FORMAT Values` clause can be stated explicitly, and the
+rows can be supplied from a stream or a file. Each row is a bracketed,
+comma-separated tuple, with the tuples themselves separated by commas:
+
+```sql title="Query"
+CREATE TABLE t (id UInt32, name String, values Array(UInt32)) ENGINE = Memory;
+
+INSERT INTO t FORMAT Values (1, 'a', [10, 20]), (2, 'b', [30]);
+
+SELECT * FROM t ORDER BY id;
+```
+
+```response title="Response"
+┌─id─┬─name─┬─values──┐
+│  1 │ a    │ [10,20] │
+│  2 │ b    │ [30]    │
+└────┴──────┴─────────┘
+```
+
+### Using expressions on input {#using-expressions}
+
+Unlike most input formats, `Values` can evaluate SQL expressions in each field
+rather than only accepting literals. This is controlled by
+[`input_format_values_interpret_expressions`](#format-settings) (enabled by
+default): when a field cannot be read by the fast streaming parser, ClickHouse
+falls back to the SQL parser and interprets the field as an expression.
+
+```sql title="Query"
+CREATE TABLE prices (item String, total UInt32) ENGINE = Memory;
+
+INSERT INTO prices FORMAT Values ('apple', 3 * 4), ('pear', length('hello') + 10);
+
+SELECT * FROM prices ORDER BY total;
+```
+
+```response title="Response"
+┌─item──┬─total─┐
+│ apple │    12 │
+│ pear  │    15 │
+└───────┴───────┘
+```
+
+### Selecting data {#selecting-data}
+
+The `Values` format can also be used to format query results. Numbers are
+written without quotes, arrays in `[]`, and strings and dates in single quotes;
+single quotes and backslashes inside strings are escaped with a backslash, and
+[`NULL`](/sql-reference/syntax.md) is written as `NULL`:
+
+```sql title="Query"
+SELECT 1 AS a, 'O''Reilly' AS b, NULL::Nullable(String) AS c FORMAT Values;
+```
+
+```response title="Response"
+(1,'O\'Reilly',NULL)
+```
+
 ## Format settings {#format-settings}
 
 | Setting                                                                                                                                                     | Description                                                                                                                                                                                   | Default |

docs/en/operations/system-tables/keeper_cluster.md

@@ -0,0 +1,44 @@
+---
+description: 'System table which exists only when this node runs an in-process ClickHouse Keeper. Contains one row per Raft cluster member as seen by this Keeper.'
+keywords: ['system table', 'keeper_cluster']
+slug: /operations/system-tables/keeper_cluster
+title: 'system.keeper_cluster'
+doc_type: 'reference'
+---
+
+import SystemTableCloud from '@site/docs/_snippets/_system_table_cloud.md';
+
+<SystemTableCloud/>
+
+## Description {#description}
+
+This table does not exist if this node is not configured to run an in-process ClickHouse Keeper. It contains one row per Raft cluster member, fusing static cluster topology (from the Raft configuration) with the local node's own log position.
+
+Every node fills exactly one `last_log_index` value — the row matching its own `server_id`. Peer log positions are not surfaced here because they are tracked only on the leader and that view is not symmetric across the cluster.
+
+## Columns {#columns}
+
+<!--AUTOGENERATED_START-->
+-   `server_id` ([Int32](../../sql-reference/data-types/int-uint.md)) — Raft server id of this cluster member.
+-   `host` ([String](../../sql-reference/data-types/string.md)) — Host parsed from the endpoint (prefix before the last colon).
+-   `endpoint` ([String](../../sql-reference/data-types/string.md)) — Raw Raft endpoint as configured (`host:port`).
+-   `is_observer` ([Bool](../../sql-reference/data-types/boolean.md)) — `true` if this member is a non-voting observer, configured via `<can_become_leader>false</can_become_leader>`.
+-   `priority` ([Int32](../../sql-reference/data-types/int-uint.md)) — Raft priority of this member; higher values are preferred during leader election.
+-   `is_leader` ([Bool](../../sql-reference/data-types/boolean.md)) — `true` if this member is the current Raft leader.
+-   `is_self` ([Bool](../../sql-reference/data-types/boolean.md)) — `true` if this row describes the local Keeper node.
+-   `last_log_index` ([Nullable(UInt64)](../../sql-reference/data-types/int-uint.md)) — Highest log index in the local Raft log store on this node. Populated only on the row matching the current node (`is_self = true`); `NULL` for other rows.
+<!--AUTOGENERATED_END-->
+
+Example:
+
+```sql
+SELECT * FROM system.keeper_cluster ORDER BY server_id;
+```
+
+```text
+┌─server_id─┬─host──┬─endpoint───┬─is_observer─┬─priority─┬─is_leader─┬─is_self─┬─last_log_index─┐
+│         1 │ node1 │ node1:9234 │ false       │        3 │ true      │ true    │             42 │
+│         2 │ node2 │ node2:9234 │ false       │        2 │ false     │ false   │           ᴺᵁᴸᴸ │
+│         3 │ node3 │ node3:9234 │ true        │        1 │ false     │ false   │           ᴺᵁᴸᴸ │
+└───────────┴───────┴────────────┴─────────────┴──────────┴───────────┴─────────┴────────────────┘
+```

docs/en/sql-reference/operators/index.md

@@ -198,6 +198,9 @@ Extract parts from a given date. For example, you can retrieve a month from a gi
 
 The `part` parameter specifies which part of the date to retrieve. The following values are available:
 
+- `NANOSECOND` — The nanosecond. Possible values: 0–999999999.
+- `MICROSECOND` — The microsecond. Possible values: 0–999999.
+- `MILLISECOND` — The millisecond. Possible values: 0–999.
 - `SECOND` — The second. Possible values: 0–59.
 - `MINUTE` — The minute. Possible values: 0–59.
 - `HOUR` — The hour. Possible values: 0–23.
@@ -214,10 +217,12 @@ The `part` parameter specifies which part of the date to retrieve. The following
 - `CENTURY` — The century. For example, the year 2024 is in the 21st century.
 - `DECADE` — The decade (year divided by 10). For example, the year 2024 has decade 202.
 - `MILLENNIUM` — The millennium. For example, the year 2024 is in the 3rd millennium.
+- `TIMEZONE_HOUR` — The signed hour part of the UTC offset of the operand's timezone. For example, `+5:30` returns `5`, `-3:30` returns `-3`.
+- `TIMEZONE_MINUTE` — The signed minute part of the UTC offset of the operand's timezone. For example, `+5:30` returns `30`, `-3:30` returns `-30`.
 
 The `part` parameter is case-insensitive.
 
-The `date` parameter specifies the date or the time to process. The [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md), and [DateTime64](../../sql-reference/data-types/datetime64.md) types are supported.
+The `date` parameter specifies the value to process. The [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md), [DateTime64](../../sql-reference/data-types/datetime64.md), and [Interval](../../sql-reference/data-types/special-data-types/interval.md) types are supported. When `date` is an `Interval`, the requested `part` must match the interval's stored kind (e.g. `EXTRACT(DAY FROM INTERVAL 5 DAY)` is allowed; `EXTRACT(HOUR FROM INTERVAL 5 DAY)` is rejected, because ClickHouse intervals are single-kind). The result for an `Interval` operand is `Int64`.
 
 Examples:
 
@@ -228,6 +233,10 @@ SELECT EXTRACT(YEAR FROM toDate('2017-06-15'));
 SELECT EXTRACT(EPOCH FROM toDateTime('2024-01-15 12:30:45', 'UTC'));
 SELECT EXTRACT(DOW FROM toDate('2024-01-15'));
 SELECT EXTRACT(CENTURY FROM toDate('2024-01-01'));
+SELECT EXTRACT(TIMEZONE_HOUR   FROM toDateTime('2024-01-15 12:00:00', 'Asia/Kolkata'));    -- 5
+SELECT EXTRACT(TIMEZONE_MINUTE FROM toDateTime('2024-01-15 12:00:00', 'Asia/Kolkata'));    -- 30
+SELECT EXTRACT(DAY   FROM INTERVAL 40 DAY);                                                -- 40
+SELECT EXTRACT(MONTH FROM INTERVAL 7 MONTH);                                               -- 7
 ```
 
 In the following example we create a table and insert into it a value with the `DateTime` type.

programs/client/FuzzLoop.cpp

@@ -495,11 +495,18 @@ bool Client::processWithASTFuzzer(std::string_view full_query)
 
 bool Client::processBuzzHouseQuery(const String & full_query)
 {
+    static constexpr size_t max_query_bytes = 1 << 20;
     bool server_up = true;
 
     have_error = false;
     error_code = 0;
-    if (!processQueryText(full_query))
+    if (full_query.size() > max_query_bytes)
+    {
+        have_error = true;
+        error_code = ErrorCodes::CANNOT_PARSE_TEXT;
+        LOG_WARNING(fuzz_config->log, "Skipping oversized query ({} bytes, limit {})", full_query.size(), max_query_bytes);
+    }
+    else if (!processQueryText(full_query))
     {
         have_error = true;
         error_code = ErrorCodes::CANNOT_PARSE_TEXT;

programs/keeper/keeper_main.cpp

@@ -170,7 +170,7 @@ static __attribute__((constructor(202))) void init_ssl()
 ///
 /// extern bool inside_main;
 /// class C { C() { assert(inside_main); } };
-static bool inside_main = false;
+bool inside_main = false;
 
 int main(int argc_, char ** argv_)
 {

programs/server/Server.cpp

@@ -47,6 +47,7 @@
 #include <Common/getExecutablePath.h>
 #include <Common/ProfileEvents.h>
 #include <Common/Scheduler/IResourceManager.h>
+#include <Common/ThreadGroupSwitcher.h>
 #include <Common/ThreadProfileEvents.h>
 #include <Common/ThreadStatus.h>
 #include <Common/getMappedArea.h>
@@ -117,8 +118,11 @@
 #include <Compression/CompressionCodecEncrypted.h>
 #include <Parsers/ASTAlterQuery.h>
 #include <Server/CloudPlacementInfo.h>
+#include <Server/DistributedQuery/ExchangeConnections.h>
+#include <Server/DistributedQuery/ExchangeServer.h>
 #include <Server/HTTP/HTTPServer.h>
 #include <Server/HTTP/HTTPServerConnectionFactory.h>
+#include <Server/StatelessWorker/StatelessWorkerEndpoint.h>
 #include <Server/MySQLHandlerFactory.h>
 #include <Server/PostgreSQLHandlerFactory.h>
 #include <Server/ProtocolServerAdapter.h>
@@ -1074,7 +1078,8 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, const
                 startup_context->setCurrentQueryId("");
 
                 {
-                    auto query_scope = QueryScope::create(startup_context);
+                    auto thread_group = ThreadGroup::createForQuery(startup_context);
+                    ThreadGroupSwitcher switcher(thread_group, /*allow_existing_group=*/ true);
                     executeQuery(condition_read_buffer, condition_write_buffer, startup_context, callback, QueryFlags{ .internal = true }, std::nullopt, {});
                 }
 
@@ -1107,7 +1112,8 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, const
             startup_context->setQueryKind(ClientInfo::QueryKind::INITIAL_QUERY);
             startup_context->setCurrentQueryId("");
 
-            auto query_scope = QueryScope::create(startup_context);
+            auto thread_group = ThreadGroup::createForQuery(startup_context);
+            ThreadGroupSwitcher switcher(thread_group, /*allow_existing_group=*/ true);
 
             executeQuery(read_buffer, write_buffer, startup_context, callback, QueryFlags{ .internal = true }, std::nullopt, {});
         }
@@ -2043,6 +2049,82 @@ try
     LOG_DEBUG(log, "Initializing interserver credentials.");
     global_context->updateInterserverCredentials(config());
 
+    std::shared_ptr<StatelessWorkerEndpoint> stateless_worker_endpoint_ptr{nullptr};
+    String stateless_worker_endpoint_name;
+    if (config().getBool("stateless_worker_server.enabled", false))
+    {
+        String stateless_worker_endpoint = config().getString("stateless_worker_server.endpoint", "localhost");
+        stateless_worker_endpoint_ptr = std::make_shared<StatelessWorkerEndpoint>();
+        stateless_worker_endpoint_name = stateless_worker_endpoint_ptr->getId(stateless_worker_endpoint);
+        global_context->getInterserverIOHandler().addEndpoint(stateless_worker_endpoint_name, stateless_worker_endpoint_ptr);
+        LOG_DEBUG(log, "Added stateless worker endpoint '{}'.", stateless_worker_endpoint_name);
+    }
+
+    SCOPE_EXIT({
+        if (stateless_worker_endpoint_ptr)
+        {
+            /// Remove the same endpoint that was registered (the configured name may differ from "localhost").
+            LOG_DEBUG(log, "Shutting down stateless worker endpoint '{}'.", stateless_worker_endpoint_name);
+            global_context->getInterserverIOHandler().removeEndpointIfExists(stateless_worker_endpoint_name);
+
+            stateless_worker_endpoint_ptr->blocker.cancelForever();
+            stateless_worker_endpoint_ptr->shutdown();
+            /// Acquire the lock to wait for all in-flight requests to finish.
+            std::lock_guard lock(stateless_worker_endpoint_ptr->rwlock);
+        }
+        stateless_worker_endpoint_ptr.reset();
+    });
+
+    #ifdef OS_LINUX
+    ExchangeConnectionsPtr exchange_connections_ptr = ExchangeConnections::instance();
+    std::vector<std::shared_ptr<ExchangeServer>> exchange_servers;
+    if (auto streaming_exchange_port = config().getUInt("distributed_query.streaming_exchange_port", 0))
+    {
+        if (streaming_exchange_port > 65535)
+            throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER,
+                "`distributed_query.streaming_exchange_port` must be in range 1..65535, got {}", streaming_exchange_port);
+
+        /// The exchange handshake is unauthenticated, so the listener is never bound to all interfaces
+        /// implicitly: the streaming exchange is enabled only when explicit listen host(s) are given.
+        Strings exchange_listen_hosts = DB::getMultipleValuesFromConfig(config(), "distributed_query", "streaming_exchange_listen_host");
+        if (exchange_listen_hosts.empty())
+        {
+            LOG_ERROR(log, "`distributed_query.streaming_exchange_port` is set but no "
+                "`distributed_query.streaming_exchange_listen_host` is configured; the streaming exchange "
+                "server is not started. Specify a listen host to enable it.");
+        }
+        else
+        {
+            for (const auto & listen_host : exchange_listen_hosts)
+            {
+                try
+                {
+                    exchange_servers.emplace_back(std::make_shared<ExchangeServer>(listen_host, streaming_exchange_port, exchange_connections_ptr));
+                    exchange_servers.back()->start();
+                }
+                catch (Poco::Exception & e)
+                {
+                    LOG_INFO(log, "Failed to start exchange server on {}:{}: {}",
+                        listen_host, streaming_exchange_port, e.displayText());
+                }
+            }
+            if (exchange_servers.empty())
+                throw Exception(ErrorCodes::NETWORK_ERROR, "Failed to start ExchangeServer on port {}", streaming_exchange_port);
+        }
+    }
+
+    SCOPE_EXIT({
+        for (auto & exchange_server_ptr : exchange_servers)
+        {
+            exchange_server_ptr->stop();
+            exchange_server_ptr.reset();
+        }
+    });
+    #else
+    if (config().getUInt("distributed_query.streaming_exchange_port", 0))
+        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "ExchangeServer is not supported on non-linux platform");
+    #endif
+
     /// Set up caches.
 
     JemallocCacheArena::setEnabled(server_settings[ServerSetting::use_separate_cache_arena]);
@@ -2571,6 +2653,9 @@ try
                     std::lock_guard lock(servers_lock);
                     updateServers(config(), new_server_settings, server_pool, *async_metrics, servers, servers_to_start_before_tables);
                 }
+
+                if (config().has("startup_scripts"))
+                    loadStartupScripts(config(), new_server_settings, global_context, log);
             }
 
             global_context->updateStorageConfiguration(config());

programs/server/config.xml

@@ -280,6 +280,25 @@
         <password></password>
     </interserver_http_credentials>-->
 
+    <!-- Multi-stage distributed query execution (experimental, enabled per query with the
+         `make_distributed_plan` setting). When a query uses streaming exchanges, workers connect
+         directly to each other on this port to pass intermediate results.
+
+         Security: the streaming exchange handshake is NOT authenticated. The port must be treated
+         like the interserver port: reachable only from trusted ClickHouse nodes in the same cluster,
+         never from untrusted networks. For that reason the listener is started only when an explicit
+         `streaming_exchange_listen_host` is configured; setting only the port logs an error and does
+         not start the listener. Bind it to a private interface and protect it with a firewall.
+         Exchanges are keyed by a random per-query identifier, which makes blind injection into a
+         running query impractical, but this is defense in depth, not an access-control mechanism.
+      -->
+    <!--
+    <distributed_query>
+        <streaming_exchange_port>9223</streaming_exchange_port>
+        <streaming_exchange_listen_host>127.0.0.1</streaming_exchange_listen_host>
+    </distributed_query>
+    -->
+
     <!-- Listen specified address.
          Use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere.
          Notes:

src/Analyzer/TableExpressionModifiers.cpp

@@ -4,6 +4,8 @@
 
 #include <Core/Streaming/CursorTree.h>
 
+#include <IO/ReadBuffer.h>
+#include <IO/ReadHelpers.h>
 #include <IO/WriteBuffer.h>
 #include <IO/WriteHelpers.h>
 #include <IO/Operators.h>
@@ -63,6 +65,20 @@ void TableExpressionModifiers::updateTreeHash(SipHash & hash_state) const
     }
 }
 
+void serializeRational(TableExpressionModifiers::Rational val, WriteBuffer & out)
+{
+    writeIntBinary(val.numerator, out);
+    writeIntBinary(val.denominator, out);
+}
+
+TableExpressionModifiers::Rational deserializeRational(ReadBuffer & in)
+{
+    TableExpressionModifiers::Rational val;
+    readIntBinary(val.numerator, in);
+    readIntBinary(val.denominator, in);
+    return val;
+}
+
 String TableExpressionModifiers::formatForErrorMessage() const
 {
     WriteBufferFromOwnString buffer;

src/Analyzer/TableExpressionModifiers.h

@@ -7,6 +7,9 @@
 namespace DB
 {
 
+class ReadBuffer;
+class WriteBuffer;
+
 /** Modifiers that can be used for table, table function and subquery in JOIN TREE.
   *
   * Example: SELECT * FROM test_table SAMPLE 0.1 OFFSET 0.1 FINAL
@@ -97,6 +100,9 @@ private:
     std::optional<StreamSettings> stream_settings;
 };
 
+void serializeRational(TableExpressionModifiers::Rational val, WriteBuffer & out);
+TableExpressionModifiers::Rational deserializeRational(ReadBuffer & in);
+
 inline bool operator==(const TableExpressionModifiers::StreamSettings & lhs, const TableExpressionModifiers::StreamSettings & rhs)
 {
     if ((lhs.cursor_tree == nullptr) != (rhs.cursor_tree == nullptr))

src/Backups/BackupEntriesCollector.cpp

@@ -15,8 +15,6 @@
 #include <Parsers/ASTCreateQuery.h>
 #include <Storages/IStorage.h>
 #include <Storages/MergeTree/extractZooKeeperPathFromReplicatedTableDef.h>
-#include <Storages/StorageMaterializedView.h>
-#include <Common/typeid_cast.h>
 #include <base/chrono_io.h>
 #include <base/insertAtEnd.h>
 #include <base/scope_guard.h>
@@ -570,10 +568,16 @@ void BackupEntriesCollector::gatherTablesMetadata()
             res_table_info.metadata_path_in_backup = metadata_path_in_backup;
             res_table_info.data_path_in_backup = data_path_in_backup;
 
-            if (const auto * mv = typeid_cast<const StorageMaterializedView *>(storage.get()))
+            /// Record REPLACE targets of refreshable materialized views from the create query, not
+            /// the storage object: for Replicated/Shared databases `getTablesForBackup` resolves it
+            /// from a ZooKeeper snapshot and may return null if not yet created on this replica.
+            if (create.is_materialized_view && create.refresh_strategy && !create.refresh_strategy->append
+                && create.hasTargetTableID(ViewTarget::To))
             {
-                if (mv->isRefreshable() && !mv->isAppendRefreshStrategy())
-                    rmv_replace_target_ids.insert(mv->getTargetTableId());
+                StorageID target_id = create.getTargetTableID(ViewTarget::To);
+                if (!target_id.hasDatabase())
+                    target_id.database_name = database_name;
+                rmv_replace_target_ids.insert(target_id);
             }
         }
     }
@@ -883,17 +887,15 @@ void BackupEntriesCollector::makeBackupEntriesForTableData(const QualifiedTableN
 
 bool BackupEntriesCollector::shouldBackupTableData(
     const QualifiedTableName & table_name,
-    const StoragePtr & storage,
+    /// Used in the Cloud build.
+    [[maybe_unused]] const StoragePtr & storage,
     const std::unordered_set<StorageID, StorageID::DatabaseAndTableNameHash, StorageID::DatabaseAndTableNameEqual> & rmv_replace_target_ids) const
 {
     if (backup_settings.structure_only)
         return false;
 
-    if (!storage)
-        return true;
-
     if (!backup_settings.backup_data_from_refreshable_materialized_view_targets
-        && rmv_replace_target_ids.contains(storage->getStorageID()))
+        && rmv_replace_target_ids.contains(StorageID{table_name.database, table_name.table}))
     {
         LOG_TRACE(log, "Skipping table data for {} (a target of a refreshable materialized view)", table_name.getFullName());
         return false;

src/Backups/BackupEntriesCollector.h

@@ -99,7 +99,8 @@ private:
     void makeBackupEntriesForTableData(const QualifiedTableName & table_name);
     bool shouldBackupTableData(
         const QualifiedTableName & table_name,
-        const StoragePtr & storage,
+        /// Used in the Cloud build.
+        [[maybe_unused]] const StoragePtr & storage,
         const std::unordered_set<StorageID, StorageID::DatabaseAndTableNameHash, StorageID::DatabaseAndTableNameEqual> & rmv_replace_target_ids) const;
 
     void addBackupEntryUnlocked(const String & file_name, BackupEntryPtr backup_entry);

src/Backups/BackupsWorker.cpp

@@ -285,9 +285,8 @@ public:
         size_t max_free_threads = 0;
         size_t queue_size = use_queue ? 0 : max_threads;
         auto thread_pool = std::make_unique<ThreadPool>(metric_threads, metric_active_threads, metric_scheduled_threads, max_threads, max_free_threads, queue_size);
-        auto * thread_pool_ptr = thread_pool.get();
-        thread_pools.emplace(thread_pool_id, std::move(thread_pool));
-        return *thread_pool_ptr;
+        it = thread_pools.emplace(thread_pool_id, std::move(thread_pool)).first;
+        return *it->second;
     }
 
     /// Waits for all threads to finish.

src/CMakeLists.txt

@@ -440,6 +440,8 @@ if (TARGET ch_contrib::arrow_flight)
     add_object_library(clickhouse_server_arrowflight Server/ArrowFlight)
 endif()
 add_object_library(clickhouse_server_embedded_client Server/ClientEmbedded)
+add_object_library(clickhouse_server_statelessworker Server/StatelessWorker)
+add_object_library(clickhouse_server_distributedquery Server/DistributedQuery)
 add_object_library(clickhouse_formats Formats)
 add_object_library(clickhouse_processors Processors)
 add_object_library(clickhouse_processors_executors Processors/Executors)

src/Client/BuzzHouse/Generator/SessionSettings.cpp

@@ -716,7 +716,7 @@ std::unordered_map<String, CHSetting> serverSettings = {
     {"distributed_insert_skip_read_only_replicas", trueOrFalseSettingNoOracle},
     {"distributed_plan_default_reader_bucket_count",
      CHSetting(
-         [](RandomGenerator & rg, FuzzConfig &) { return std::to_string(rg.thresholdGenerator<uint64_t>(0.2, 0.2, 0, 128)); }, {}, false)},
+         [](RandomGenerator & rg, FuzzConfig &) { return std::to_string(rg.thresholdGenerator<uint64_t>(0.2, 0.2, 1, 128)); }, {}, false)},
     {"distributed_plan_default_shuffle_join_bucket_count",
      CHSetting(
          [](RandomGenerator & rg, FuzzConfig &) { return std::to_string(rg.thresholdGenerator<uint64_t>(0.2, 0.2, 1, 128)); }, {}, false)},

src/Client/BuzzHouse/Generator/TableSettings.cpp

@@ -57,7 +57,6 @@ static std::unordered_map<String, CHSetting> mergeTreeTableSettings = {
     {"allow_coalescing_columns_in_partition_or_order_key", trueOrFalseSetting},
     {"allow_commit_order_projection", trueOrFalseSetting},
     {"allow_experimental_replacing_merge_with_cleanup", trueOrFalseSetting},
-    {"allow_experimental_reverse_key", trueOrFalseSetting},
     {"allow_floating_point_partition_key", trueOrFalseSetting},
     {"allow_nullable_key", trueOrFalseSetting},
     {"allow_part_offset_column_in_projections", trueOrFalseSetting},

src/Client/ClientBase.cpp

@@ -1007,10 +1007,6 @@ void ClientBase::setDefaultFormatsAndCompressionFromConfiguration()
             default_output_format = *format_from_file_name;
         else
             default_output_format = "TSV";
-
-        std::optional<String> file_name = tryGetFileNameFromFileDescriptor(stdout_fd);
-        if (file_name)
-            default_output_compression_method = chooseCompressionMethod(*file_name, "");
     }
     else if (is_interactive)
     {
@@ -1021,6 +1017,17 @@ void ClientBase::setDefaultFormatsAndCompressionFromConfiguration()
         default_output_format = "TSV";
     }
 
+    /// Detect output compression independently of format selection.
+    /// Even when the user specifies --output-format or --format explicitly,
+    /// stdout may still be redirected to a compressed file (e.g., output.gz).
+    if (default_output_compression_method == CompressionMethod::None
+        && isFileDescriptorSuitableForInput(stdout_fd))
+    {
+        std::optional<String> file_name = tryGetFileNameFromFileDescriptor(stdout_fd);
+        if (file_name)
+            default_output_compression_method = chooseCompressionMethod(*file_name, "");
+    }
+
     if (getClientConfiguration().has("input-format"))
     {
         default_input_format = getClientConfiguration().getString("input-format");
@@ -1045,7 +1052,13 @@ void ClientBase::setDefaultFormatsAndCompressionFromConfiguration()
             default_input_format = *format_from_file_name;
         else
             default_input_format = "auto";
+    }
 
+    /// Detect input compression independently of format selection.
+    /// Even when the user specifies --input-format or --format explicitly,
+    /// stdin may still be redirected from a compressed file (e.g., input.gz).
+    if (default_input_compression_method == CompressionMethod::None)
+    {
         std::optional<String> file_name = tryGetFileNameFromFileDescriptor(stdin_fd);
         if (file_name)
             default_input_compression_method = chooseCompressionMethod(*file_name, "");

src/Columns/ColumnConst.h

@@ -310,7 +310,8 @@ public:
     }
 
     bool isNullable() const override { return isColumnNullable(*data); }
-    bool onlyNull() const override { return data->isNullAt(0); }
+    /// Delegate to `data->onlyNull` so that e.g. `Const(Nullable(Nothing))` reports itself as only-null.
+    bool onlyNull() const override { return data->isNullAt(0) || data->onlyNull(); }
     bool isNumeric() const override { return data->isNumeric(); }
     bool isFixedAndContiguous() const override { return data->isFixedAndContiguous(); }
     bool valuesHaveFixedSize() const override { return data->valuesHaveFixedSize(); }

src/Columns/ColumnNullable.h

@@ -95,7 +95,17 @@ public:
     void insertDefault() override
     {
         getNestedColumn().insertDefault();
-        getNullMapData().push_back(true);
+        /// Keep the nested column and the null map in sync even if appending to the
+        /// null map throws (e.g. on a memory limit).
+        try
+        {
+            getNullMapData().push_back(true);
+        }
+        catch (...)
+        {
+            getNestedColumn().popBack(1);
+            throw;
+        }
     }
 
     void popBack(size_t n) override;

src/Columns/ColumnObject.cpp

@@ -833,20 +833,55 @@ void ColumnObject::deserializeValueFromSharedData(const ColumnString * shared_da
 
 void ColumnObject::insertDefault()
 {
-    for (auto & [_, column] : typed_paths)
-        column->insertDefault();
-    for (auto & [_, column] : dynamic_paths_ptrs)
-        column->insertDefault();
-    shared_data->insertDefault();
+    /// Exception-safe: if some sub-column's insertDefault throws (e.g. on a memory limit),
+    /// roll back the sub-columns that were already modified, otherwise the object is left
+    /// with sub-columns of different sizes and popBack would over-pop the shorter ones.
+    size_t prev_size = size();
+    try
+    {
+        for (auto & [_, column] : typed_paths)
+            column->insertDefault();
+        for (auto & [_, column] : dynamic_paths_ptrs)
+            column->insertDefault();
+        shared_data->insertDefault();
+    }
+    catch (...)
+    {
+        for (auto & [_, column] : typed_paths)
+            if (column->size() > prev_size)
+                column->popBack(column->size() - prev_size);
+        for (auto & [_, column] : dynamic_paths_ptrs)
+            if (column->size() > prev_size)
+                column->popBack(column->size() - prev_size);
+        if (shared_data->size() > prev_size)
+            shared_data->popBack(shared_data->size() - prev_size);
+        throw;
+    }
 }
 
 void ColumnObject::insertManyDefaults(size_t length)
 {
-    for (auto & [_, column] : typed_paths)
-        column->insertManyDefaults(length);
-    for (auto & [_, column] : dynamic_paths_ptrs)
-        column->insertManyDefaults(length);
-    shared_data->insertManyDefaults(length);
+    size_t prev_size = size();
+    try
+    {
+        for (auto & [_, column] : typed_paths)
+            column->insertManyDefaults(length);
+        for (auto & [_, column] : dynamic_paths_ptrs)
+            column->insertManyDefaults(length);
+        shared_data->insertManyDefaults(length);
+    }
+    catch (...)
+    {
+        for (auto & [_, column] : typed_paths)
+            if (column->size() > prev_size)
+                column->popBack(column->size() - prev_size);
+        for (auto & [_, column] : dynamic_paths_ptrs)
+            if (column->size() > prev_size)
+                column->popBack(column->size() - prev_size);
+        if (shared_data->size() > prev_size)
+            shared_data->popBack(shared_data->size() - prev_size);
+        throw;
+    }
 }
 
 void ColumnObject::popBack(size_t n)

src/Columns/ColumnTuple.cpp

@@ -279,9 +279,24 @@ void ColumnTuple::doInsertManyFrom(const IColumn & src, size_t position, size_t
 
 void ColumnTuple::insertDefault()
 {
+    /// Must be exception-safe: if some nested column throws (e.g. on a memory limit),
+    /// the already modified nested columns have to be rolled back, otherwise the tuple
+    /// is left with nested columns of different sizes, which later leads to over-popping
+    /// in popBack during rollback of a partially read row.
+    size_t i = 0;
+    try
+    {
+        for (; i < columns.size(); ++i)
+            columns[i]->insertDefault();
+    }
+    catch (...)
+    {
+        for (size_t rollback = 0; rollback < i; ++rollback)
+            columns[rollback]->popBack(1);
+        throw;
+    }
+
     ++column_length;
-    for (auto & column : columns)
-        column->insertDefault();
 }
 
 void ColumnTuple::popBack(size_t n)

src/Columns/ColumnVariant.cpp

@@ -718,14 +718,32 @@ void ColumnVariant::deserializeBinaryIntoVariant(ColumnVariant::Discriminator gl
 void ColumnVariant::insertDefault()
 {
     getLocalDiscriminators().push_back(NULL_DISCRIMINATOR);
-    getOffsets().emplace_back();
+    /// Keep local_discriminators and offsets in sync even if appending to offsets throws
+    /// (e.g. on a memory limit), otherwise popBack would over-pop the shorter one.
+    try
+    {
+        getOffsets().emplace_back();
+    }
+    catch (...)
+    {
+        getLocalDiscriminators().pop_back();
+        throw;
+    }
 }
 
 void ColumnVariant::insertManyDefaults(size_t length)
 {
-    size_t size = local_discriminators->size();
-    getLocalDiscriminators().resize_fill(size + length, NULL_DISCRIMINATOR);
-    getOffsets().resize_fill(size + length);
+    size_t prev_size = local_discriminators->size();
+    getLocalDiscriminators().resize_fill(prev_size + length, NULL_DISCRIMINATOR);
+    try
+    {
+        getOffsets().resize_fill(prev_size + length);
+    }
+    catch (...)
+    {
+        getLocalDiscriminators().resize_assume_reserved(prev_size);
+        throw;
+    }
 }
 
 void ColumnVariant::popBack(size_t n)

src/Columns/tests/gtest_column_tuple.cpp

@@ -1,8 +1,15 @@
 #include <Columns/ColumnTuple.h>
 #include <Columns/ColumnsNumber.h>
+#include <Columns/ColumnFixedString.h>
 
 #include <gtest/gtest.h>
+#include <base/types.h>
+#include <Common/CurrentThread.h>
 #include <Common/Exception.h>
+#include <Common/MemoryTracker.h>
+#include <Common/ThreadStatus.h>
+#include <Common/assert_cast.h>
+#include <Common/scope_guard_safe.h>
 
 using namespace DB;
 
@@ -77,6 +84,38 @@ TEST(ColumnTuple, EmptyTupleExpand)
     ASSERT_EQ(expanded_inv->size(), mask.size());
 }
 
+TEST(ColumnTuple, InsertDefaultIsExceptionSafe)
+{
+    /// First element is plain: its insertDefault succeeds.
+    /// Second element is a wide FixedString: its insertDefault triggers a large allocation
+    /// that overshoots a clamped memory limit and throws MEMORY_LIMIT_EXCEEDED - the same
+    /// way a real query hits a memory limit in the middle of a default insert.
+    static constexpr size_t huge = 64 * 1024 * 1024;
+
+    auto first = ColumnFloat64::create();
+    first->reserve(1);
+    MutableColumns elements;
+    elements.push_back(std::move(first));
+    elements.push_back(ColumnFixedString::create(huge));
+    auto tuple = ColumnTuple::create(std::move(elements));
+
+    {
+        ThreadStatus thread_status;
+        const Int64 prev_hard_limit = total_memory_tracker.getHardLimit();
+        SCOPE_EXIT_SAFE(total_memory_tracker.setHardLimit(prev_hard_limit));
+        total_memory_tracker.setHardLimit(total_memory_tracker.get() + 1024);
+
+        ASSERT_THROW(tuple->insertDefault(), Exception);
+    }
+
+    /// The first element must have been rolled back so that all nested columns stay in sync,
+    /// otherwise a later popBack would over-pop the shorter element.
+    const auto & tuple_concrete = assert_cast<const ColumnTuple &>(*tuple);
+    ASSERT_EQ(tuple_concrete.getColumn(0).size(), 0);
+    ASSERT_EQ(tuple_concrete.getColumn(1).size(), 0);
+    ASSERT_EQ(tuple->size(), 0);
+}
+
 TEST(ColumnTuple, EmptyTuplePermute)
 {
     ColumnPtr tuple_col = ColumnTuple::create(5);

src/Common/CurrentMetrics.cpp

@@ -505,6 +505,9 @@
     M(StatelessWorkerThreads, "Number of threads in the stateless worker thread pool.") \
     M(StatelessWorkerThreadsActive, "Number of threads in the stateless worker thread pool running a task.") \
     M(StatelessWorkerThreadsScheduled, "Number of queued or active jobs in the stateless worker thread pool.") \
+    M(ExchangeServerThreads, "Number of threads in the distributed exchange server handshake thread pool.") \
+    M(ExchangeServerThreadsActive, "Number of threads in the distributed exchange server handshake thread pool running a task.") \
+    M(ExchangeServerThreadsScheduled, "Number of queued or active jobs in the distributed exchange server handshake thread pool.") \
     M(ReadonlyDisks, "Number of disks that were marked as readonly during disk check.") \
     M(BrokenDisks, "Number of disks disks that were marked as broken during disk check.") \
     M(TaskTrackerThreads, "Number of threads used by the distributed query remote task tracker.") \

src/Common/DateLUTImpl.cpp

@@ -240,6 +240,48 @@ unsigned int DateLUTImpl::toMillisecond(const DB::DateTime64 & datetime, Int64 s
 }
 
 
+unsigned int DateLUTImpl::toMicrosecond(const DB::DateTime64 & datetime, Int64 scale_multiplier) const
+{
+    constexpr Int64 microsecond_multiplier = 1'000'000;
+
+    auto components = DB::DecimalUtils::splitWithScaleMultiplier(datetime, scale_multiplier);
+
+    if (datetime.value < 0 && components.fractional)
+    {
+        components.fractional = scale_multiplier + (components.whole ? Int64(-1) : Int64(1)) * components.fractional;
+        --components.whole;
+    }
+    Int64 fractional = components.fractional;
+    if (scale_multiplier > microsecond_multiplier)
+        fractional = fractional / (scale_multiplier / microsecond_multiplier);
+    else if (scale_multiplier < microsecond_multiplier)
+        fractional = fractional * (microsecond_multiplier / scale_multiplier);
+
+    return static_cast<unsigned>(fractional);
+}
+
+
+unsigned int DateLUTImpl::toNanosecond(const DB::DateTime64 & datetime, Int64 scale_multiplier) const
+{
+    constexpr Int64 nanosecond_multiplier = 1'000'000'000;
+
+    auto components = DB::DecimalUtils::splitWithScaleMultiplier(datetime, scale_multiplier);
+
+    if (datetime.value < 0 && components.fractional)
+    {
+        components.fractional = scale_multiplier + (components.whole ? Int64(-1) : Int64(1)) * components.fractional;
+        --components.whole;
+    }
+    Int64 fractional = components.fractional;
+    if (scale_multiplier > nanosecond_multiplier)
+        fractional = fractional / (scale_multiplier / nanosecond_multiplier);
+    else if (scale_multiplier < nanosecond_multiplier)
+        fractional = fractional * (nanosecond_multiplier / scale_multiplier);
+
+    return static_cast<unsigned>(fractional);
+}
+
+
 /// Prefer to load timezones from blobs linked to the binary.
 /// The blobs are provided by "tzdata" library.
 /// This allows to avoid dependency on system tzdata.

src/Common/DateLUTImpl.h

@@ -600,6 +600,10 @@ public:
 
     unsigned toMillisecond(const DB::DateTime64 & datetime, Int64 scale_multiplier) const;
 
+    unsigned toMicrosecond(const DB::DateTime64 & datetime, Int64 scale_multiplier) const;
+
+    unsigned toNanosecond(const DB::DateTime64 & datetime, Int64 scale_multiplier) const;
+
     unsigned toMinute(Time t) const
     {
         if (t >= 0 && offset_is_whole_number_of_hours_during_epoch)

src/Common/FailPoint.cpp

@@ -189,7 +189,8 @@ static struct InitFiu
     REGULAR(wide_part_writer_fail_in_add_streams) \
     REGULAR(compact_part_writer_fail_in_add_streams) \
     REGULAR(transaction_force_unknown_state_after_commit) \
-    PAUSEABLE(transaction_after_commit_pause)
+    PAUSEABLE(transaction_after_commit_pause) \
+    REGULAR(mt_mutate_task_can_skip_conversion_to_nullable_force_null_column_desc)
 
 namespace FailPoints
 {

src/Common/IntervalKind.cpp

@@ -3,6 +3,9 @@
 
 #include <base/EnumReflection.h>
 
+#include <string_view>
+#include <unordered_map>
+
 
 namespace DB
 {
@@ -247,63 +250,49 @@ const char * IntervalKind::toNameOfFunctionExtractTimePart() const
 }
 
 
+bool IntervalKind::tryParseFromNameOfFunctionExtractTimePart(std::string_view name, IntervalKind::Kind & result)
+{
+    /// Mirrors `toNameOfFunctionExtractTimePart`.
+    static const std::unordered_map<std::string_view, IntervalKind::Kind> lookup = {
+        {"toNanosecond",  IntervalKind::Kind::Nanosecond},
+        {"toMicrosecond", IntervalKind::Kind::Microsecond},
+        {"toMillisecond", IntervalKind::Kind::Millisecond},
+        {"toSecond",      IntervalKind::Kind::Second},
+        {"toMinute",      IntervalKind::Kind::Minute},
+        {"toHour",        IntervalKind::Kind::Hour},
+        {"toDayOfMonth",  IntervalKind::Kind::Day},
+        {"toISOWeek",     IntervalKind::Kind::Week},
+        {"toMonth",       IntervalKind::Kind::Month},
+        {"toQuarter",     IntervalKind::Kind::Quarter},
+        {"toYear",        IntervalKind::Kind::Year},
+    };
+    auto it = lookup.find(name);
+    if (it == lookup.end())
+        return false;
+    result = it->second;
+    return true;
+}
+
+
 bool IntervalKind::tryParseString(const std::string & kind, IntervalKind::Kind & result)
 {
-    if ("nanosecond" == kind)
-    {
-        result = IntervalKind::Kind::Nanosecond;
-        return true;
-    }
-    if ("microsecond" == kind)
-    {
-        result = IntervalKind::Kind::Microsecond;
-        return true;
-    }
-    if ("millisecond" == kind)
-    {
-        result = IntervalKind::Kind::Millisecond;
-        return true;
-    }
-    if ("second" == kind)
-    {
-        result = IntervalKind::Kind::Second;
-        return true;
-    }
-    if ("minute" == kind)
-    {
-        result = IntervalKind::Kind::Minute;
-        return true;
-    }
-    if ("hour" == kind)
-    {
-        result = IntervalKind::Kind::Hour;
-        return true;
-    }
-    if ("day" == kind)
-    {
-        result = IntervalKind::Kind::Day;
-        return true;
-    }
-    if ("week" == kind)
-    {
-        result = IntervalKind::Kind::Week;
-        return true;
-    }
-    if ("month" == kind)
-    {
-        result = IntervalKind::Kind::Month;
-        return true;
-    }
-    if ("quarter" == kind)
-    {
-        result = IntervalKind::Kind::Quarter;
-        return true;
-    }
-    if ("year" == kind)
-    {
-        result = IntervalKind::Kind::Year;
-        return true;
-    }
-    return false;
+    static const std::unordered_map<std::string_view, IntervalKind::Kind> lookup = {
+        {"nanosecond",  IntervalKind::Kind::Nanosecond},
+        {"microsecond", IntervalKind::Kind::Microsecond},
+        {"millisecond", IntervalKind::Kind::Millisecond},
+        {"second",      IntervalKind::Kind::Second},
+        {"minute",      IntervalKind::Kind::Minute},
+        {"hour",        IntervalKind::Kind::Hour},
+        {"day",         IntervalKind::Kind::Day},
+        {"week",        IntervalKind::Kind::Week},
+        {"month",       IntervalKind::Kind::Month},
+        {"quarter",     IntervalKind::Kind::Quarter},
+        {"year",        IntervalKind::Kind::Year},
+    };
+    auto it = lookup.find(kind);
+    if (it == lookup.end())
+        return false;
+    result = it->second;
+    return true;
 }
 }

src/Common/IntervalKind.h

@@ -67,6 +67,13 @@ struct IntervalKind
     /// returns "toDayOfMonth".
     const char * toNameOfFunctionExtractTimePart() const;
 
+    /// Inverse of `toNameOfFunctionExtractTimePart`: given a function name like
+    /// "toYear", "toMonth", "toDayOfMonth", ... sets `result` to the matching
+    /// `IntervalKind` and returns true. Returns false for any other name.
+    /// Used to recognise calendar-field extractor functions whose `EXTRACT`-style
+    /// dispatch can be redirected onto an `Interval` operand.
+    static bool tryParseFromNameOfFunctionExtractTimePart(std::string_view name, IntervalKind::Kind & result);
+
     /// Converts the string representation of an interval kind to its IntervalKind equivalent.
     /// Returns false if the conversion did not succeed.
     /// For example, `IntervalKind::tryParseString('second', result)` returns `result` equals `IntervalKind::Kind::Second`.

src/Common/MemoryTracker.cpp

@@ -133,6 +133,7 @@ void AllocationTrace::onFreeImpl(void * ptr, size_t size) const
 namespace ProfileEvents
 {
     extern const Event QueryMemoryLimitExceeded;
+    extern const Event GlobalMemoryLimitExceeded;
     extern const Event PageCacheOvercommitResize;
     extern const Event MemoryAllocatedWithoutCheck;
     extern const Event MemoryAllocatedWithoutCheckBytes;
@@ -232,6 +233,8 @@ void MemoryTracker::injectFault() const
     MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global);
 
     ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded);
+    if (level == VariableContext::Global)
+        ProfileEvents::increment(ProfileEvents::GlobalMemoryLimitExceeded);
     const auto * description = description_ptr.load(std::memory_order_relaxed);
     throw DB::Exception(
         DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED,
@@ -348,6 +351,8 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool enforce_memory_limit,
             MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global);
 
             ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded);
+            if (level == VariableContext::Global)
+                ProfileEvents::increment(ProfileEvents::GlobalMemoryLimitExceeded);
             const auto * description = description_ptr.load(std::memory_order_relaxed);
             throw DB::Exception(
                 DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED,
@@ -440,6 +445,8 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool enforce_memory_limit,
                 /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc
                 MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global);
                 ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded);
+                if (level == VariableContext::Global)
+                    ProfileEvents::increment(ProfileEvents::GlobalMemoryLimitExceeded);
                 const auto * description = description_ptr.load(std::memory_order_relaxed);
                 throw DB::Exception(
                     DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED,

src/Common/ProfileEvents.cpp

@@ -779,6 +779,7 @@ The server successfully detected this situation and will download merged part fr
     M(WriteBufferFromS3RequestsErrors, "Number of exceptions while writing to S3.", ValueType::Number) \
     M(WriteBufferFromS3WaitInflightLimitMicroseconds, "Time spent on waiting while some of the current requests are done when its number reached the limit defined by s3_max_inflight_parts_for_one_file.", ValueType::Microseconds) \
     M(QueryMemoryLimitExceeded, "Number of times when memory limit exceeded for query.", ValueType::Number) \
+    M(GlobalMemoryLimitExceeded, "Number of times the global memory limit was exceeded.", ValueType::Number) \
     M(MemoryAllocatedWithoutCheck, "Number of times memory has been allocated without checking for memory constraints.", ValueType::Number) \
     M(MemoryAllocatedWithoutCheckBytes, "Amount of bytes that has been allocated without checking for memory constraints.", ValueType::Number) \
     \
@@ -1387,13 +1388,13 @@ The server successfully detected this situation and will download merged part fr
     M(WasmModuleInstatiate, "Number of WebAssembly compartments created", ValueType::Number) \
     M(WasmMemoryAllocated, "Total memory allocated for WebAssembly compartments", ValueType::Bytes) \
     \
-    M(ExecutableUserDefinedFunctionInvocations, "Number of executable_pool user-defined function invocations.", ValueType::Number) \
-    M(ExecutableUserDefinedFunctionElapsedMicroseconds, "Wall clock time spent executing executable_pool user-defined functions, in microseconds.", ValueType::Microseconds) \
-    M(ExecutableUserDefinedFunctionUserTimeMicroseconds, "User mode CPU time consumed by executable_pool user-defined function child processes, in microseconds.", ValueType::Microseconds) \
-    M(ExecutableUserDefinedFunctionSystemTimeMicroseconds, "Kernel mode CPU time consumed by executable_pool user-defined function child processes, in microseconds.", ValueType::Microseconds) \
-    M(ExecutableUserDefinedFunctionPeakMemoryByteSeconds, "Per-process peak memory used by executable_pool user-defined function child processes integrated over wall clock time, in byte-seconds.", ValueType::Number) \
-    M(ExecutableUserDefinedFunctionInputBytes, "Total bytes written to the stdin of executable_pool user-defined function child processes.", ValueType::Bytes) \
-    M(ExecutableUserDefinedFunctionOutputBytes, "Total bytes read from the stdout of executable_pool user-defined function child processes.", ValueType::Bytes) \
+    M(ExecutableUserDefinedFunctionInvocations, "Number of executable and executable_pool user-defined function invocations.", ValueType::Number) \
+    M(ExecutableUserDefinedFunctionElapsedMicroseconds, "Wall clock time spent executing executable and executable_pool user-defined functions, in microseconds.", ValueType::Microseconds) \
+    M(ExecutableUserDefinedFunctionUserTimeMicroseconds, "User mode CPU time consumed by executable and executable_pool user-defined function child processes, in microseconds.", ValueType::Microseconds) \
+    M(ExecutableUserDefinedFunctionSystemTimeMicroseconds, "Kernel mode CPU time consumed by executable and executable_pool user-defined function child processes, in microseconds.", ValueType::Microseconds) \
+    M(ExecutableUserDefinedFunctionPeakMemoryByteSeconds, "Per-process peak memory used by executable and executable_pool user-defined function child processes integrated over wall clock time, in byte-seconds.", ValueType::Number) \
+    M(ExecutableUserDefinedFunctionInputBytes, "Total bytes written to the stdin of executable and executable_pool user-defined function child processes.", ValueType::Bytes) \
+    M(ExecutableUserDefinedFunctionOutputBytes, "Total bytes read from the stdout of executable and executable_pool user-defined function child processes.", ValueType::Bytes) \
     M(ExecutableUserDefinedFunctionPoolWaitMicroseconds, "Time spent waiting on tryBorrowObject when the executable_pool user-defined function pool is exhausted, in microseconds.", ValueType::Microseconds) \
     \
     M(ParquetReadRowGroups, "The total number of row groups read from parquet data", ValueType::Number) \
@@ -1599,12 +1600,11 @@ void Counters::setTraceProfileEvent(Event event)
     {
         /// It is very unlikely that it will be allocated twice, since we set it at the beginning of the query
         auto fresh = std::make_unique<std::atomic_bool[]>(num_counters);
-        auto * fresh_raw = fresh.get();
         std::atomic_bool * expected = nullptr;
-        if (should_trace_array.compare_exchange_strong(expected, fresh_raw, std::memory_order_release, std::memory_order_relaxed))
+        if (should_trace_array.compare_exchange_strong(expected, fresh.get(), std::memory_order_release, std::memory_order_relaxed))
         {
             should_trace_holder = std::move(fresh);
-            trace_array = fresh_raw;
+            trace_array = should_trace_holder.get();
         }
         else
             trace_array = expected;

src/Common/QueryFuzzer.cpp

@@ -971,7 +971,6 @@ void QueryFuzzer::fuzzCreateQuery(ASTCreateQuery & create)
                "add_minmax_index_for_string_columns",
                "add_minmax_index_for_temporal_columns",
                "allow_coalescing_columns_in_partition_or_order_key",
-               "allow_experimental_reverse_key",
                "allow_floating_point_partition_key",
                "allow_nullable_key",
                "allow_summing_columns_in_partition_or_order_key",

src/Common/ReplicasReconnector.cpp

@@ -23,6 +23,7 @@ ReplicasReconnector::ReplicasReconnector(ContextPtr context)
     : task_handle(context->getSchedulePool().createTask(StorageID::createEmpty(), "ReplicasReconnector", [this]{ run(); }))
     , log(getLogger("ReplicasReconnector"))
 {
+    instance_ptr = this;
 }
 
 ReplicasReconnector::~ReplicasReconnector()
@@ -37,9 +38,7 @@ std::unique_ptr<ReplicasReconnector> ReplicasReconnector::init(ContextPtr contex
     if (instance_ptr)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Replicas reconnector is already initialized.");
 
-    std::unique_ptr<ReplicasReconnector> ret(new ReplicasReconnector(context));
-    instance_ptr = ret.get();
-    return ret;
+    return std::unique_ptr<ReplicasReconnector>(new ReplicasReconnector(context));
 }
 
 ReplicasReconnector & ReplicasReconnector::instance()

src/Common/ShellCommand.cpp

@@ -1,3 +1,16 @@
+/// `wait4` is declared under `_DEFAULT_SOURCE` on Linux glibc, which the
+/// `-std=c++23` strict mode otherwise hides. Define it before the first system
+/// header that guards it. It is a libc feature-test macro, hence the reserved
+/// name; suppress the diagnostics that would otherwise reject our own define.
+#if defined(OS_LINUX) && !defined(_DEFAULT_SOURCE)
+#   pragma clang diagnostic push
+#   pragma clang diagnostic ignored "-Wreserved-macro-identifier"
+#   pragma clang diagnostic ignored "-Wunused-macros"
+#   define _DEFAULT_SOURCE // NOLINT(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp)
+#   pragma clang diagnostic pop
+#endif
+
+#include <sys/resource.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <dlfcn.h>
@@ -338,7 +351,7 @@ int ShellCommand::tryWait()
     return tryWaitImpl(true).retcode;
 }
 
-ShellCommand::tryWaitResult ShellCommand::tryWaitImpl(bool blocking)
+ShellCommand::tryWaitResult ShellCommand::tryWaitImpl(bool blocking, bool check_exit_status)
 {
     LOG_TRACE(getLogger(), "Will wait for shell command pid {}", pid);
 
@@ -347,12 +360,32 @@ ShellCommand::tryWaitResult ShellCommand::tryWaitImpl(bool blocking)
     int options = ((!blocking) ? WNOHANG : 0);
     int status = 0;
     int waitpid_retcode = -1;
+    ::rusage local_rusage{};
 
     while (waitpid_retcode < 0)
     {
-        waitpid_retcode = waitpid(pid, &status, options);
+        /// Reap the child. With `Config::collect_resource_usage` (executable UDFs),
+        /// use `wait4` to also collect the child's `rusage`: it is `waitpid` plus an
+        /// `rusage` out-parameter and shares its pid/status/options/EINTR semantics.
+        /// Without the flag, reap with plain `waitpid` and collect no usage.
+        if (config.collect_resource_usage)
+            waitpid_retcode = wait4(pid, &status, options, &local_rusage);
+        else
+            waitpid_retcode = waitpid(pid, &status, options);
         if (waitpid_retcode > 0)
         {
+            /// A reaped pid may be reused immediately, so `wait_called` must be set the
+            /// moment the child is reaped — before any operation that can throw — so the
+            /// destructor never waits on or signals an unrelated process.
+            wait_called = true;
+            if (config.collect_resource_usage)
+            {
+                child_user_time_us = static_cast<UInt64>(local_rusage.ru_utime.tv_sec) * 1000000ULL
+                    + static_cast<UInt64>(local_rusage.ru_utime.tv_usec);
+                child_system_time_us = static_cast<UInt64>(local_rusage.ru_stime.tv_sec) * 1000000ULL
+                    + static_cast<UInt64>(local_rusage.ru_stime.tv_usec);
+                child_resource_usage_captured = true;
+            }
             break;
         }
         if (!blocking && !waitpid_retcode)
@@ -366,8 +399,6 @@ ShellCommand::tryWaitResult ShellCommand::tryWaitImpl(bool blocking)
 
     LOG_TRACE(getLogger(), "Wait for shell command pid {} completed with status {}", pid, status);
 
-    wait_called = true;
-
     result.is_process_terminated = true;
     in.close();
     out.close();
@@ -379,6 +410,12 @@ ShellCommand::tryWaitResult ShellCommand::tryWaitImpl(bool blocking)
     for (auto & [_, fd] : read_fds)
         fd.close();
 
+    /// When `check_exit_status` is false the caller only wants the reaped `rusage`;
+    /// skip decoding/validating the status so a non-zero or signalled child is not
+    /// reported as an error.
+    if (!check_exit_status)
+        return result;
+
     if (WIFEXITED(status))
     {
         result.retcode = WEXITSTATUS(status);
@@ -430,6 +467,12 @@ bool ShellCommand::waitIfProccesTerminated()
 }
 
 
+bool ShellCommand::tryReapWithoutStatusCheck()
+{
+    return tryWaitImpl(/*blocking=*/false, /*check_exit_status=*/false).is_process_terminated;
+}
+
+
 void ShellCommand::wait()
 {
     int retcode = tryWaitImpl(true).retcode;
@@ -437,4 +480,23 @@ void ShellCommand::wait()
 }
 
 
+bool ShellCommand::wasChildResourceUsageCaptured() const noexcept
+{
+    return child_resource_usage_captured;
 }
+
+
+UInt64 ShellCommand::getChildUserTimeMicroseconds() const noexcept
+{
+    return child_user_time_us;
+}
+
+
+UInt64 ShellCommand::getChildSystemTimeMicroseconds() const noexcept
+{
+    return child_system_time_us;
+}
+
+
+}
+

src/Common/ShellCommand.h

@@ -5,6 +5,7 @@
 #include <IO/ReadBufferFromFile.h>
 #include <IO/WriteBufferFromFile.h>
 #include <Common/VectorWithMemoryTracking.h>
+#include <base/types.h>
 
 
 namespace DB
@@ -68,6 +69,12 @@ public:
         size_t pipe_capacity = 0;
 
         DestructorStrategy terminate_in_destructor_strategy = DestructorStrategy(false, 0);
+
+        /// When true, `tryWaitImpl` reaps with `wait4` and captures the child's
+        /// `rusage` (read back via `getChild*`/`wasChildResourceUsageCaptured`).
+        /// When false (the default) it reaps with plain `waitpid` and allocates
+        /// nothing. Set for executable (non-pool) UDFs, which read the usage.
+        bool collect_resource_usage = false;
     };
 
     pid_t getPid() const
@@ -85,6 +92,18 @@ public:
         do_not_terminate = true;
     }
 
+    /// True once the child has been reaped by `tryWaitImpl` and its
+    /// resource usage was captured.
+    bool wasChildResourceUsageCaptured() const noexcept;
+
+    /// User-mode CPU time consumed by the reaped child. Zero if
+    /// `wasChildResourceUsageCaptured` returns false.
+    UInt64 getChildUserTimeMicroseconds() const noexcept;
+
+    /// Kernel-mode CPU time consumed by the reaped child. Zero if
+    /// `wasChildResourceUsageCaptured` returns false.
+    UInt64 getChildSystemTimeMicroseconds() const noexcept;
+
     /// Run the command using /bin/sh -c.
     /// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
     static std::unique_ptr<ShellCommand> execute(const Config & config);
@@ -103,6 +122,11 @@ public:
     /// If process terminated, then handle return code.
     bool waitIfProccesTerminated();
 
+    /// Non-blocking reap: if the child has already terminated, collect its `rusage`
+    /// without inspecting the exit status, so a non-zero or signalled exit is not
+    /// raised as an error. Returns whether the child was reaped.
+    bool tryReapWithoutStatusCheck();
+
     WriteBufferFromFile in;        /// If the command reads from stdin, do not forget to call in.close() after writing all the data there.
     ReadBufferFromFile out;
     ReadBufferFromFile err;
@@ -116,12 +140,19 @@ private:
     bool wait_called = false;
     bool do_not_terminate = false;
 
+    /// CPU time of the reaped child, taken from `wait4` rusage and stored by value
+    /// at reap time. The reap path performs no allocation, so a memory-limit
+    /// `exception` can never fail a query whose child has already exited.
+    bool child_resource_usage_captured = false;
+    UInt64 child_user_time_us = 0;
+    UInt64 child_system_time_us = 0;
+
     ShellCommand(pid_t pid_, int & in_fd_, int & out_fd_, int & err_fd_, const Config & config);
 
     bool tryWaitProcessWithTimeout(size_t timeout_in_seconds);
     struct tryWaitResult;
 
-    tryWaitResult tryWaitImpl(bool blocking);
+    tryWaitResult tryWaitImpl(bool blocking, bool check_exit_status = true);
 
     void handleProcessRetcode(int retcode) const;
 

src/Common/ThreadGroupSwitcher.cpp

@@ -20,6 +20,22 @@ ThreadGroupPtr getCurrentThreadGroup()
 }
 
 ThreadGroupSwitcher::ThreadGroupSwitcher(ThreadGroupPtr thread_group_, ThreadName thread_name, bool allow_existing_group) noexcept
+    : ThreadGroupSwitcher(std::move(thread_group_), allow_existing_group)
+{
+    if (!thread_group)
+        return;
+
+    try
+    {
+        setThreadName(thread_name);
+    }
+    catch (...)
+    {
+        DB::tryLogCurrentException(__PRETTY_FUNCTION__);
+    }
+}
+
+ThreadGroupSwitcher::ThreadGroupSwitcher(ThreadGroupPtr thread_group_, bool allow_existing_group) noexcept
     : thread_group(std::move(thread_group_))
 {
     try
@@ -38,18 +54,17 @@ ThreadGroupSwitcher::ThreadGroupSwitcher(ThreadGroupPtr thread_group_, ThreadNam
                 return;
             }
             else if (!allow_existing_group)
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Thread ({}) is already attached to a group (master_thread_id {})", thread_name, prev_thread_group->master_thread_id);
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Thread is already attached to a group (master_thread_id {})", prev_thread_group->master_thread_id);
             else
                 CurrentThread::detachFromGroupIfNotDetached();
         }
 
         if (!prev_thread)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Tried to attach thread ({}) to a group, but the ThreadStatus is not initialized", thread_name);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Tried to attach thread to a group, but the ThreadStatus is not initialized");
 
         LockMemoryExceptionInThread lock_memory_tracker(VariableContext::Global);
 
         CurrentThread::attachToGroup(thread_group);
-        setThreadName(thread_name);
     }
     catch (...)
     {

src/Common/ThreadGroupSwitcher.h

@@ -39,6 +39,7 @@ public:
     ///  * If true, remembers the current group and restores it in destructor.
     /// If thread_name is not empty, calls setThreadName along the way; should be at most 15 bytes long.
     ThreadGroupSwitcher(ThreadGroupPtr thread_group_, ThreadName thread_name, bool allow_existing_group = false) noexcept;
+    explicit ThreadGroupSwitcher(ThreadGroupPtr thread_group_, bool allow_existing_group = false) noexcept;
     ~ThreadGroupSwitcher();
 
 private:

src/Common/ThreadPoolTaskTracker.cpp

@@ -104,6 +104,12 @@ void TaskTracker::waitIfAny()
 
 void TaskTracker::add(Callback && func)
 {
+    {
+        std::lock_guard lock(mutex);
+        chassert(!final_task_added && "add must not be called after addFinal");
+        ++tasks_added;
+    }
+
     /// All this fuzz is about 2 things. This is the most critical place of TaskTracker.
     /// The first is not to fail insertion in the list `futures`.
     /// In order to face it, the element is allocated at the end of the list `futures` in advance.
@@ -118,14 +124,33 @@ void TaskTracker::add(Callback && func)
     /// preallocation for the second issue
     FinishedList pre_allocated_finished {future_placeholder};
 
-    Callback func_with_notification = [&, my_func = std::move(func), my_pre_allocated_finished = std::move(pre_allocated_finished)]() mutable
+    Callback func_with_notification = [this, my_func = std::move(func), my_pre_allocated_finished = std::move(pre_allocated_finished)]() mutable
     {
         SCOPE_EXIT({
-            DENY_ALLOCATIONS_IN_SCOPE;
-
-            std::lock_guard lock(mutex);
-            finished_futures.splice(finished_futures.end(), my_pre_allocated_finished);
-            has_finished.notify_one();
+            std::shared_ptr<std::packaged_task<void()>> maybe_final_task;
+            {
+                DENY_ALLOCATIONS_IN_SCOPE;
+                std::lock_guard lock(mutex);
+                finished_futures.splice(finished_futures.end(), my_pre_allocated_finished);
+                ++tasks_finished;
+                if (final_task && tasks_finished == tasks_added)
+                {
+                    maybe_final_task = std::move(final_task);
+                }
+                has_finished.notify_one();
+            }
+            if (maybe_final_task)
+            {
+                try
+                {
+                    scheduler([pt = std::move(maybe_final_task)]() mutable { (*pt)(); }, Priority{});
+                }
+                catch (...)
+                {
+                    /// SCOPE_EXIT is noexcept; let the final task's future report broken_promise.
+                    tryLogCurrentException(__PRETTY_FUNCTION__);
+                }
+            }
         });
 
         my_func();
@@ -137,6 +162,33 @@ void TaskTracker::add(Callback && func)
     waitTilInflightShrink();
 }
 
+void TaskTracker::addFinal(Callback && func)
+{
+    auto pt = std::make_shared<std::packaged_task<void()>>(std::move(func));
+    futures.emplace_back(pt->get_future());
+
+    bool run_final_task_now = false;
+    {
+        std::lock_guard lock(mutex);
+        chassert(!final_task_added && "addFinal must be called at most once");
+        final_task_added = true;
+        if (tasks_finished == tasks_added)
+        {
+            /// Every previously added task has already finished.
+            /// There will be no SCOPE_EXIT to trigger the final callback, so run it here.
+            run_final_task_now = true;
+        }
+        else
+        {
+            final_task = pt;
+        }
+    }
+    if (run_final_task_now)
+    {
+        scheduler([p = std::move(pt)]() mutable { (*p)(); }, Priority{});
+    }
+}
+
 void TaskTracker::waitTilInflightShrink()
 {
     if (!max_tasks_inflight)

src/Common/ThreadPoolTaskTracker.h

@@ -39,6 +39,11 @@ public:
 
     void add(Callback && func);
 
+    /// Must be called at most once, after all `add()` calls.
+    /// The callback runs on a worker thread once every previously-added task has completed.
+    /// May or may not run if prior tasks threw.
+    void addFinal(Callback && func);
+
 private:
     /// waitTilInflightShrink waits til the number of in-flight tasks beyond the limit `max_tasks_inflight`.
     void waitTilInflightShrink() TSA_NO_THREAD_SAFETY_ANALYSIS;
@@ -57,6 +62,18 @@ private:
     std::condition_variable has_finished TSA_GUARDED_BY(mutex);
     using FinishedList = std::list<FutureList::iterator>;
     FinishedList finished_futures TSA_GUARDED_BY(mutex);
+
+    /// Monotonic counters of regular tasks submitted via add (excludes the final task).
+    /// Worker threads compare these to decide when the final task is ready, so they
+    /// never need to read `futures`.
+    size_t tasks_added TSA_GUARDED_BY(mutex) = 0;
+    size_t tasks_finished TSA_GUARDED_BY(mutex) = 0;
+
+    /// A packaged task for the callback added by addFinal. A non-null value means
+    /// the callback has been added, but not yet scheduled.
+    std::shared_ptr<std::packaged_task<void()>> final_task TSA_GUARDED_BY(mutex);
+
+    bool final_task_added = false;
 };
 
 }

src/Common/UDFProcessSubtreeSampler.cpp

@@ -4,6 +4,7 @@
 #include <IO/ReadHelpers.h>
 
 #include <base/arithmeticOverflow.h>
+#include <base/scope_guard.h>
 
 #include <dirent.h>
 #include <fcntl.h>
@@ -67,6 +68,11 @@ std::vector<pid_t> walkSubtree(pid_t root_pid, bool & truncated)
         if (!dir)
             continue;
 
+        /// Release the handle on every path, including an exception thrown while
+        /// building `children_path` or opening the `ifstream` under a memory limit;
+        /// otherwise the swallowed sample failure would leak a directory fd.
+        SCOPE_EXIT(::closedir(dir));
+
         struct dirent * entry = nullptr;
         /// NOLINTNEXTLINE(concurrency-mt-unsafe) -- `dir` is a local `DIR *` not shared across threads.
         while ((entry = ::readdir(dir)) != nullptr)
@@ -101,8 +107,6 @@ std::vector<pid_t> walkSubtree(pid_t root_pid, bool & truncated)
             if (result.size() >= MAX_PIDS)
                 break;
         }
-
-        ::closedir(dir);
     }
 #endif
 
@@ -262,7 +266,7 @@ void UDFProcessSubtreeSampler::recordPidAcquired(pid_t root_pid_)
     bool walk_truncated = false;
     auto pids = UDFProcfs::walkSubtree(root_pid, walk_truncated);
     if (walk_truncated)
-        subtree_truncated_any = true;
+        subtree_truncated_any.store(true, std::memory_order_relaxed);
     for (pid_t pid : pids)
     {
         pre_walk_pids.insert(pid);
@@ -338,7 +342,7 @@ void UDFProcessSubtreeSampler::recordReleased()
     bool walk_truncated = false;
     auto pids = UDFProcfs::walkSubtree(root_pid, walk_truncated);
     if (walk_truncated)
-        subtree_truncated_any = true;
+        subtree_truncated_any.store(true, std::memory_order_relaxed);
 
     UInt64 post_utime_sum = 0;
     UInt64 post_stime_sum = 0;
@@ -381,7 +385,7 @@ void UDFProcessSubtreeSampler::recordReleased()
         if (UDFProcfs::readPeakRss(pid, hwm_bytes))
             peak_rss = std::max(peak_rss, hwm_bytes);
         else
-            read_peak_rss_failed_any = true;
+            read_peak_rss_failed_any.store(true, std::memory_order_relaxed);
     }
 
     if (post_utime_sum >= pre_utime_sum)
@@ -410,4 +414,119 @@ void UDFProcessSubtreeSampler::recordReleased()
         peak_memory_byte_seconds = product / 1000000ULL;
 }
 
+
+void UDFProcessSubtreeSampler::recordExecutablePid(pid_t root_pid_) noexcept
+{
+    executable_root_pid = root_pid_;
+}
+
+
+void UDFProcessSubtreeSampler::sampleExecutablePeak([[maybe_unused]] bool is_final) noexcept
+{
+    if (executable_root_pid <= 0)
+        return;
+
+#if defined(OS_LINUX)
+    /// Called on every IO buffer fill; a 4 KiB read buffer would otherwise trigger
+    /// a subtree walk per fill while streaming large output. VmHWM is monotonic, so
+    /// the running max survives sparse sampling: throttle to at most one walk per
+    /// ~5 ms. The first call always samples (last_sample_us == 0) so a
+    /// short-lived child is measured at least once. The final (EOF) call bypasses
+    /// the throttle to ensure the tail sample always executes.
+    if (!is_final)
+    {
+        static constexpr UInt64 sample_interval_us = 5000;
+
+        const UInt64 now_us = entry_watch.elapsedMicroseconds();
+        UInt64 last = last_sample_us.load(std::memory_order_relaxed);
+        if (last != 0 && now_us < last + sample_interval_us)
+            return;
+        if (!last_sample_us.compare_exchange_strong(last, now_us, std::memory_order_relaxed))
+            return;
+    }
+
+    /// walkSubtree and readPeakRss allocate (path strings, ifstream); an exception
+    /// thrown under a memory limit must not cross this noexcept boundary.
+    try
+    {
+        bool walk_truncated = false;
+        const auto pids = UDFProcfs::walkSubtree(executable_root_pid, walk_truncated);
+        if (walk_truncated)
+            subtree_truncated_any.store(true, std::memory_order_relaxed);
+
+        for (pid_t pid : pids)
+        {
+            UInt64 hwm_bytes = 0;
+            if (!UDFProcfs::readPeakRss(pid, hwm_bytes))
+            {
+                /// A read can fail because the pid exited since `walkSubtree` (a
+                /// short-lived child, or the root after it closed stdout) or because
+                /// a sandbox denies `/proc`. The executable path cannot tell these
+                /// apart — the child's lifetime is not under our control — so it does
+                /// not raise the degradation flag here, unlike the pool path where the
+                /// borrowed worker is provably alive during sampling.
+                continue;
+            }
+
+            UInt64 current = executable_peak_rss_bytes.load(std::memory_order_relaxed);
+            while (hwm_bytes > current)
+            {
+                if (executable_peak_rss_bytes.compare_exchange_weak(
+                        current, hwm_bytes,
+                        std::memory_order_relaxed, std::memory_order_relaxed))
+                    break;
+            }
+        }
+    }
+    catch (...)
+    {
+        /// Dropping a sample is ok: the peak is a best-effort monotonic high-water mark.
+        return;
+    }
+#endif
+}
+
+
+void UDFProcessSubtreeSampler::recordExecutableElapsed() noexcept
+{
+    if (executable_finished)
+        return;
+
+    /// Wall time from sampler construction to `ShellCommandSource` cleanup (includes
+    /// spawn, output parsing, and IO). The executable path spawns a fresh child per
+    /// invocation, so there is no pool-wait interval to subtract.
+    elapsed_us = entry_watch.elapsedMicroseconds();
+
+    /// Peak RSS comes from /proc VmHWM sampled during IO into executable_peak_rss_bytes.
+    /// The send-data threads are joined before cleanup reaches this point, and the
+    /// read side ran on this same (cleanup) thread, so the relaxed load sees all
+    /// sampled writes.
+    const UInt64 peak_rss_bytes = executable_peak_rss_bytes.load(std::memory_order_relaxed);
+
+    UInt64 product = 0;
+    if (common::mulOverflow(peak_rss_bytes, elapsed_us, product))
+        peak_memory_byte_seconds = std::numeric_limits<UInt64>::max();
+    else
+        peak_memory_byte_seconds = product / 1000000ULL;
+
+    executable_finished = true;
+}
+
+
+void UDFProcessSubtreeSampler::recordExecutableFinished(
+    UInt64 user_time_us_, UInt64 system_time_us_) noexcept
+{
+    if (executable_rusage_recorded)
+        return;
+
+    /// Stamp elapsed first; `recordExecutableElapsed` is idempotent so calling
+    /// it here is safe even if the caller already invoked it.
+    recordExecutableElapsed();
+
+    user_time_us = user_time_us_;
+    system_time_us = system_time_us_;
+
+    executable_rusage_recorded = true;
+}
+
 }

src/Common/UDFProcessSubtreeSampler.h

@@ -14,29 +14,52 @@
 namespace DB
 {
 
-/** Per-borrow resource accounting for an executable_pool UDF call.
+/** Per-invocation resource accounting for executable UDF calls.
   *
   * Created by `UserDefinedExecutableFunctionFactory::executeImpl` and passed
   * via `ShellCommandSourceConfiguration` into the borrow/IO machinery. The
   * factory queries the accumulators after the pipeline drains and feeds them
   * into ProfileEvents.
   *
-  * Lifecycle (callbacks tolerate being skipped — e.g. `tryBorrowObject`
-  * may time out, in which case only `recordPoolWaitDone` fires):
-  *   ctor                    -- implicitly starts the wall clock for pool wait
+  * Two paths share this class:
+  *
+  * Pool path (`executable_pool`): a persistent worker process is borrowed from
+  * a pool for the duration of one UDF invocation. CPU and memory are measured
+  * by sampling `/proc/<pid>/{stat,status}` for the worker and all descendants.
+  * The pre-snapshot zeroes `VmHWM` via `/proc/<pid>/clear_refs` (mode 5) so
+  * `VmHWM` at release reflects the peak observed during this borrow, not the
+  * worker's lifetime peak. Lifecycle callbacks:
+  *   ctor                    -- starts the wall clock for pool-wait measurement
   *   recordPoolWaitDone      -- `tryBorrowObject` returned (success OR timeout)
   *   recordPidAcquired       -- `buildCommand` returned; pid known (success only)
   *   recordInputBytes / recordOutputBytes -- IO buffers report bytes pushed
   *   recordReleased          -- ShellCommandSource cleanup, before returnObject
   *
-  * CPU and memory deltas are derived by sampling /proc/<pid>/{stat,status} for
-  * every descendant of the pool's child process. The pre-snapshot zeroes
-  * VmHWM via /proc/<pid>/clear_refs (mode 5) so VmHWM at release reflects the
-  * peak observed during this borrow, not the lifetime peak of the worker.
+  * Executable path (non-pool): a fresh child is spawned per invocation via
+  * `vfork`+`exec`. Peak RSS is measured by sampling `/proc/<pid>/VmHWM` across
+  * the child's subtree during IO (not `wait4 ru_maxrss`, which reports the
+  * parent's RSS for a `vfork`+`exec` child because the kernel folds the pre-exec
+  * mm's RSS into `signal->maxrss` at exec). CPU still comes from `wait4` rusage.
+  * Lifecycle callbacks:
+  *   ctor                    -- starts the wall clock for elapsed measurement
+  *   recordExecutablePid     -- process created; stores pid for subtree sampling
+  *   recordInputBytes / recordOutputBytes -- IO buffers report bytes pushed
+  *   sampleExecutablePeak    -- called on each IO read and input write (child
+  *                              provably alive), plus one best-effort tail sample
+  *                              at stdout EOF (child may have already exited);
+  *                              keeps a running max of VmHWM across the subtree
+  *   recordExecutableElapsed -- ShellCommandSource cleanup; stamps elapsed_us
+  *                              and computes peak_memory_byte_seconds
+  *   recordExecutableFinished -- records CPU from wait4 rusage
   *
-  * Procfs failures (ENOENT after a worker died, EACCES, malformed) are
-  * silently skipped: CPU/memory increments are dropped but the other events
-  * still fire.
+  * Procfs failures (ENOENT after a worker died, EACCES, malformed content) are
+  * silently skipped on both the pool and executable paths: CPU/memory increments
+  * are dropped but the other events still fire. On the pool path, a failed read
+  * for the root pid signals genuine degradation (e.g. seccomp blocking `/proc`)
+  * because the borrowed worker is provably alive during sampling. On the
+  * executable path, `readPeakRss` failures are never flagged as degradation:
+  * the child's lifetime is not under our control and a root read failure is
+  * indistinguishable from a legitimately exited short-lived UDF.
   */
 class UDFProcessSubtreeSampler
 {
@@ -49,11 +72,71 @@ public:
     void recordOutputBytes(size_t bytes) noexcept;
     void recordReleased();
 
+    /// Executable (non-pool) path: store the child pid for subtree sampling.
+    /// Called immediately after the child process is created. Does not perform
+    /// any /proc I/O — the child's VmHWM starts at zero and is sampled during IO.
+    void recordExecutablePid(pid_t root_pid) noexcept;
+
+    /// Executable (non-pool) path: walk the subtree from `executable_root_pid`,
+    /// read `VmHWM` from `/proc/<pid>/status` for each pid, and keep a running
+    /// max in `executable_peak_rss_bytes`. Invoked from both IO loops (stdout
+    /// read and stdin write). The first call always samples; subsequent non-final
+    /// calls are throttled to at most one subtree walk per ~5 ms. `VmHWM`
+    /// monotonicity keeps the running max correct under sparse sampling. No-op
+    /// when `executable_root_pid <= 0`. Thread-safe: may be called concurrently
+    /// from the write-buffer send thread and the read-buffer pull thread.
+    ///
+    /// When the stdout read reaches EOF, the caller passes `is_final=true` to
+    /// take one best-effort concluding ("tail") sample that bypasses the throttle.
+    /// This is a single attempt at the point the child has closed stdout and is
+    /// typically still resident; it is NOT continuous sampling during the
+    /// post-output reap. A failed `/proc` read on the final call is expected
+    /// (the child may have already exited) and does not flag degradation.
+    /// Memory the child allocates strictly after closing stdout is captured only
+    /// on a best-effort basis: it may or may not be observed depending on whether
+    /// the child is still resident when EOF is detected.
+    ///
+    /// Limitation: only the UDF process and its live descendants at a sample
+    /// point are captured. A short-lived child that allocates and is reaped
+    /// between two consecutive throttled samples is best-effort and may be
+    /// under-counted. A process that is alive at least once after reaching its
+    /// peak is captured exactly, because `VmHWM` is monotonic.
+    /// Additionally, a short-lived UDF that allocates, writes small output
+    /// that fits entirely in the pipe buffer, and exits before ClickHouse reads
+    /// its stdout may report a near-zero peak: the IO-phase samples never fire
+    /// because the child is already gone when the read side runs. CPU and
+    /// elapsed time are unaffected (taken from `wait4` rusage).
+    void sampleExecutablePeak([[maybe_unused]] bool is_final = false) noexcept;
+
+    /// Executable (non-pool) path: stamp `elapsed_us` from the wall clock at
+    /// `ShellCommandSource` cleanup and compute `peak_memory_byte_seconds` from
+    /// `executable_peak_rss_bytes` (sampled from `/proc VmHWM` during IO).
+    /// Safe to call even when the child was never reaped. Idempotent; a second
+    /// call is a no-op.
+    void recordExecutableElapsed() noexcept;
+
+    /// Executable (non-pool) path: fill `user_time_us` and `system_time_us`
+    /// from the scalars returned by `ShellCommand::getChild*` (CPU from `wait4`
+    /// rusage). Two independent idempotency guards operate here:
+    ///   - `recordExecutableElapsed` is guarded by `executable_finished`: it
+    ///     stamps `elapsed_us` and `peak_memory_byte_seconds` on the first call
+    ///     and is a no-op thereafter.
+    ///   - `recordExecutableFinished` itself is guarded by
+    ///     `executable_rusage_recorded`: it stores CPU scalars on the first call
+    ///     and is a no-op thereafter, even if `recordExecutableElapsed` was
+    ///     already invoked by the caller.
+    /// `recordExecutableElapsed` is always called from here so that elapsed and
+    /// peak are stamped even when CPU is recorded first.
+    /// Peak memory is taken from `/proc VmHWM` sampled during IO; this method
+    /// does not receive or touch it.
+    void recordExecutableFinished(UInt64 user_time_us, UInt64 system_time_us) noexcept;
+
     /// Pool wait = entry → borrow acquired.
     /// Zero if the borrow never happened (caller should still report 0).
     UInt64 getPoolWaitMicroseconds() const noexcept { return pool_wait_us; }
 
-    /// Borrow-internal wall = borrow acquired → released.
+    /// Pool mode: borrow acquired → released (excludes the pool-wait interval).
+    /// Executable mode: ctor → ShellCommandSource cleanup (includes spawn, output parsing and IO).
     UInt64 getElapsedMicroseconds() const noexcept { return elapsed_us; }
 
     UInt64 getUserTimeMicroseconds() const noexcept { return user_time_us; }
@@ -64,6 +147,7 @@ public:
 
     bool poolWaitDone() const noexcept { return pool_wait_done; }
     bool borrowAcquired() const noexcept { return borrow_acquired; }
+    bool executableFinished() const noexcept { return executable_finished; }
 
     /// Whether at least one pid in this borrow's subtree saw the
     /// corresponding procfs operation fail. Caller-side state can use these
@@ -71,13 +155,13 @@ public:
     /// a sandbox / seccomp profile silently degrades metrics to zero.
     bool clearRefsFailedAnyPid() const noexcept { return clear_refs_failed_any; }
     bool readStatFailedAnyPid() const noexcept { return read_stat_failed_any; }
-    bool readPeakRssFailedAnyPid() const noexcept { return read_peak_rss_failed_any; }
+    bool readPeakRssFailedAnyPid() const noexcept { return read_peak_rss_failed_any.load(std::memory_order_relaxed); }
 
     /// Whether `walkSubtree` hit the `MAX_PIDS` cap during this borrow and
     /// a candidate pid had to be dropped. When true, the subtree enumeration
     /// is incomplete, so CPU and `PeakMemoryByteSeconds` will under-count
     /// the unvisited descendants.
-    bool subtreeWalkTruncated() const noexcept { return subtree_truncated_any; }
+    bool subtreeWalkTruncated() const noexcept { return subtree_truncated_any.load(std::memory_order_relaxed); }
 
 private:
     Stopwatch entry_watch;
@@ -86,6 +170,8 @@ private:
     pid_t root_pid = -1;
     bool pool_wait_done = false;
     bool borrow_acquired = false;
+    bool executable_finished = false;
+    bool executable_rusage_recorded = false;
 
     UInt64 pool_wait_us = 0;
     UInt64 elapsed_us = 0;
@@ -95,12 +181,30 @@ private:
 
     bool clear_refs_failed_any = false;
     bool read_stat_failed_any = false;
-    bool read_peak_rss_failed_any = false;
-    bool subtree_truncated_any = false;
+
+    /// Written by sampleExecutablePeak (called from two threads: the write-buffer
+    /// send thread and the read-buffer pull thread) and read after both threads are
+    /// joined. Must be atomic to avoid a data race.
+    std::atomic<bool> read_peak_rss_failed_any{false};
+    std::atomic<bool> subtree_truncated_any{false};
 
     std::atomic<UInt64> input_bytes{0};
     std::atomic<UInt64> output_bytes{0};
 
+    /// Executable path only: pid set by recordExecutablePid; <= 0 means sampling
+    /// has not been requested (pool path or no process yet). Guarded by sequential
+    /// writes (recordExecutablePid called once before IO) and reads from IO threads.
+    pid_t executable_root_pid = -1;
+
+    /// Running max VmHWM across the executable child's subtree, updated atomically
+    /// by sampleExecutablePeak from two IO threads. Read after both threads join.
+    std::atomic<UInt64> executable_peak_rss_bytes{0};
+
+    /// Wall-clock microseconds (from entry_watch) of the last subtree sample, or 0
+    /// if none taken yet. Throttles sampleExecutablePeak off the per-buffer IO path;
+    /// updated with a relaxed CAS from the two IO threads.
+    std::atomic<UInt64> last_sample_us{0};
+
     struct PreSnapshot
     {
         UInt64 utime_us = 0;

src/Common/setThreadName.h

@@ -56,6 +56,7 @@ namespace DB
     M(DISTRIBUTED_SCHEDULE_POOL, "BgDistSchPool") \
     M(DISTRIBUTED_SINK, "DistrOutStrProc") \
     M(DISTRIBUTED_INDEX_ANALYSIS, "DistIdxAnalysis") \
+    M(DISTRIBUTED_QUERY_TASK, "DistQueryTask") \
     M(DROP_TABLES, "DropTables") \
     M(DWARF_DECODER, "DWARFDecoder") \
     M(ERROR_LOG, "ErrorLog") \

src/Common/tests/gtest_allocation_interceptors.cpp

@@ -7,7 +7,9 @@
 #include <Common/Exception.h>
 #include <Common/LockMemoryExceptionInThread.h>
 #include <Common/MemoryTracker.h>
+#include <Common/ProfileEvents.h>
 #include <Common/ThreadStatus.h>
+#include <base/scope_guard.h>
 #include <limits>
 
 namespace DB::ErrorCodes
@@ -15,6 +17,12 @@ namespace DB::ErrorCodes
     extern const int MEMORY_LIMIT_EXCEEDED;
 }
 
+namespace ProfileEvents
+{
+    extern const Event GlobalMemoryLimitExceeded;
+    extern const Event QueryMemoryLimitExceeded;
+}
+
 using namespace DB;
 
 /// NOLINTBEGIN
@@ -222,6 +230,53 @@ TEST(AllocationInterceptors, MinAllocSizeToThrowDoesNotAffectExplicitAllocPath)
     }, DB::Exception);
 }
 
+TEST(AllocationInterceptors, GlobalMemoryLimitExceededOnlyIncrementsForGlobalTracker)
+{
+    MainThreadStatus::getInstance();
+
+    MemoryLimitGuard limit_guard;
+    auto & thread_tracker = CurrentThread::get().memory_tracker;
+    MemoryTracker query_tracker(&total_memory_tracker, VariableContext::Process, /*log_peak_memory_usage_in_destructor=*/ false);
+    MemoryTracker * prev_parent = thread_tracker.getParent();
+    Int64 prev_hard_limit = thread_tracker.getHardLimit();
+    const auto reset_test_counters = [&]
+    {
+        total_memory_tracker.resetCounters();
+        thread_tracker.resetCounters();
+        query_tracker.resetCounters();
+        CurrentThread::getProfileEvents().resetCounters();
+    };
+
+    CurrentThread::flushUntrackedMemory();
+
+    SCOPE_EXIT({
+        CurrentThread::flushUntrackedMemory();
+        reset_test_counters();
+        thread_tracker.setHardLimit(prev_hard_limit);
+        thread_tracker.setParent(prev_parent);
+    });
+
+    thread_tracker.setParent(&query_tracker);
+
+    reset_test_counters();
+
+    query_tracker.setHardLimit(query_tracker.get() + 1024);
+    EXPECT_THROW({
+        std::ignore = CurrentMemoryTracker::alloc(allocation_size);
+    }, DB::Exception);
+    EXPECT_EQ(1, CurrentThread::getProfileEvents()[ProfileEvents::QueryMemoryLimitExceeded]);
+    EXPECT_EQ(0, CurrentThread::getProfileEvents()[ProfileEvents::GlobalMemoryLimitExceeded]);
+
+    reset_test_counters();
+
+    total_memory_tracker.setHardLimit(total_memory_tracker.get() + 1024);
+    EXPECT_THROW({
+        std::ignore = CurrentMemoryTracker::alloc(allocation_size);
+    }, DB::Exception);
+    EXPECT_EQ(1, CurrentThread::getProfileEvents()[ProfileEvents::QueryMemoryLimitExceeded]);
+    EXPECT_EQ(1, CurrentThread::getProfileEvents()[ProfileEvents::GlobalMemoryLimitExceeded]);
+}
+
 TEST(AllocationInterceptors, MinAllocSizeToThrowDoesNotAffectMalloc)
 {
     MemoryLimitGuard guard;

src/Common/tests/gtest_thread_pool_task_tracker.cpp

@@ -0,0 +1,203 @@
+#include <Common/ThreadPoolTaskTracker.h>
+#include <Common/setThreadName.h>
+
+#include <gtest/gtest.h>
+
+namespace CurrentMetrics
+{
+    extern const Metric LocalThread;
+    extern const Metric LocalThreadActive;
+    extern const Metric LocalThreadScheduled;
+}
+
+using namespace DB;
+
+namespace
+{
+
+LogSeriesLimiterPtr makeTestLogger()
+{
+    return std::make_shared<LogSeriesLimiter>(getLogger("TaskTrackerTest"), 1, 5);
+}
+
+struct AsyncTracker
+{
+    ThreadPool pool;
+    TaskTracker tracker;
+
+    explicit AsyncTracker(size_t threads, size_t max_inflight = 0)
+        : pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, threads)
+        , tracker(threadPoolCallbackRunnerUnsafe<void>(pool, ThreadName::UNKNOWN), max_inflight, makeTestLogger())
+    {}
+};
+
+}
+
+TEST(TaskTrackerAddFinal, AsyncHappyPathRunsFinalAfterAllPriors)
+{
+    AsyncTracker t{/*threads=*/4};
+    constexpr size_t N = 16;
+    std::atomic<size_t> ran{0};
+    std::atomic<bool> final_ran{false};
+    std::atomic<size_t> count_observed_by_final{0};
+
+    for (size_t i = 0; i < N; ++i)
+        t.tracker.add([&] {
+            std::this_thread::sleep_for(std::chrono::milliseconds(5));
+            ran.fetch_add(1);
+        });
+
+    t.tracker.addFinal([&] {
+        count_observed_by_final = ran.load();
+        final_ran = true;
+    });
+
+    t.tracker.waitAll();
+
+    EXPECT_EQ(ran.load(), N);
+    EXPECT_TRUE(final_ran.load());
+    EXPECT_EQ(count_observed_by_final.load(), N);
+}
+
+TEST(TaskTrackerAddFinal, AsyncZeroPriorsStillRunsFinal)
+{
+    AsyncTracker t{/*threads=*/2};
+    std::atomic<bool> final_ran{false};
+
+    t.tracker.addFinal([&] { final_ran = true; });
+    t.tracker.waitAll();
+
+    EXPECT_TRUE(final_ran.load());
+}
+
+TEST(TaskTrackerAddFinal, AsyncAllPriorsAlreadyDoneBeforeAddFinal)
+{
+    AsyncTracker t{/*threads=*/2};
+    constexpr size_t N = 4;
+    std::atomic<size_t> ran{0};
+    std::atomic<bool> final_ran{false};
+
+    for (size_t i = 0; i < N; ++i)
+        t.tracker.add([&] { ran.fetch_add(1); });
+
+    /// Give workers a chance to finish before we call addFinal — exercises the path where
+    /// `finished_futures.size() == futures.size()` is already true at the time addFinal is called.
+    while (ran.load() != N)
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+
+    t.tracker.addFinal([&] { final_ran = true; });
+    t.tracker.waitAll();
+
+    EXPECT_TRUE(final_ran.load());
+}
+
+TEST(TaskTrackerAddFinal, AsyncPriorFailureDoesNotDeadlock)
+{
+    AsyncTracker t{/*threads=*/2};
+
+    t.tracker.add([] { /* ok */ });
+    t.tracker.add([] { throw std::runtime_error("boom"); });
+    t.tracker.add([] { /* ok */ });
+    t.tracker.addFinal([] {});
+
+    EXPECT_THROW(t.tracker.waitAll(), std::runtime_error);
+    /// The final callback is best-effort on failures — we only assert no deadlock and clean shutdown.
+    t.tracker.safeWaitAll();
+}
+
+TEST(TaskTrackerAddFinal, AsyncFinalCallbackException)
+{
+    AsyncTracker t{/*threads=*/2};
+    t.tracker.add([] { /* ok */ });
+    t.tracker.addFinal([] { throw std::runtime_error("final boom"); });
+
+    EXPECT_THROW(t.tracker.waitAll(), std::runtime_error);
+    t.tracker.safeWaitAll();
+}
+
+TEST(TaskTrackerAddFinal, WaitIfAnyBeforeFinalTask)
+{
+    TaskTracker tracker(/*scheduler_=*/{}, /*max_tasks_inflight=*/0, makeTestLogger());
+    ASSERT_FALSE(tracker.isAsync());
+
+    constexpr size_t N = 2;
+    std::atomic<size_t> ran{0};
+    std::atomic<bool> final_ran{false};
+
+    for (size_t i = 0; i < N; ++i)
+        tracker.add([&] { ran.fetch_add(1); });
+
+    /// Sync runner has already run every task body and populated `finished_futures`.
+    ASSERT_EQ(ran.load(), N);
+
+    tracker.waitIfAny();
+
+    tracker.addFinal([&] { final_ran = true; });
+
+    tracker.waitAll();
+
+    EXPECT_EQ(ran.load(), N);
+    EXPECT_TRUE(final_ran.load());
+}
+
+TEST(TaskTrackerAddFinal, AsyncWithInflightLimit)
+{
+    /// Exercise the path where `waitTilInflightShrink` is hit on the owner thread between adds.
+    AsyncTracker t{/*threads=*/4, /*max_inflight=*/3};
+    constexpr size_t N = 32;
+    std::atomic<size_t> ran{0};
+    std::atomic<bool> final_ran{false};
+
+    for (size_t i = 0; i < N; ++i)
+        t.tracker.add([&] {
+            std::this_thread::sleep_for(std::chrono::milliseconds(2));
+            ran.fetch_add(1);
+        });
+
+    size_t observed_in_final = 0;
+    t.tracker.addFinal([&] {
+        observed_in_final = ran.load();
+        final_ran = true;
+    });
+    t.tracker.waitAll();
+
+    EXPECT_EQ(ran.load(), N);
+    EXPECT_TRUE(final_ran.load());
+    EXPECT_EQ(observed_in_final, N);
+}
+
+TEST(TaskTrackerAddFinal, SyncRunnerHappyPath)
+{
+    TaskTracker tracker(/*scheduler_=*/{}, /*max_tasks_inflight=*/0, makeTestLogger());
+    ASSERT_FALSE(tracker.isAsync());
+
+    constexpr size_t N = 5;
+    std::atomic<size_t> ran{0};
+    std::atomic<bool> final_ran{false};
+    size_t observed_in_final = 0;
+
+    for (size_t i = 0; i < N; ++i)
+        tracker.add([&] { ran.fetch_add(1); });
+
+    tracker.addFinal([&] {
+        observed_in_final = ran.load();
+        final_ran = true;
+    });
+    tracker.waitAll();
+
+    EXPECT_EQ(ran.load(), N);
+    EXPECT_TRUE(final_ran.load());
+    EXPECT_EQ(observed_in_final, N);
+}
+
+TEST(TaskTrackerAddFinal, SyncRunnerPriorFailureDoesNotDeadlock)
+{
+    TaskTracker tracker(/*scheduler_=*/{}, /*max_tasks_inflight=*/0, makeTestLogger());
+
+    tracker.add([] { /* ok */ });
+    tracker.add([] { throw std::runtime_error("boom"); });
+    tracker.addFinal([] {});
+
+    EXPECT_THROW(tracker.waitAll(), std::runtime_error);
+    tracker.safeWaitAll();
+}

src/Common/waitForPid.cpp

@@ -5,6 +5,7 @@
 /// for abortOnFailedAssertion() via chassert() (dependency chain looks odd)
 #include <Common/Exception.h>
 #include <base/defines.h>
+#include <base/scope_guard.h>
 
 #include <fcntl.h>
 #include <sys/wait.h>
@@ -105,18 +106,30 @@ static PollPidResult pollPid(pid_t pid, int timeout_in_ms)
         }
     }
 
+    /// Releases pid_fd on every return path, including poll timeout and error.
+    SCOPE_EXIT(
+    {
+        [[maybe_unused]] int err = close(pid_fd);
+        chassert(!err || errno == EINTR);
+    });
+
     struct pollfd pollfd{};
     pollfd.fd = pid_fd;
     pollfd.events = POLLIN;
 
-    int ready = HANDLE_EINTR(poll(&pollfd, 1, timeout_in_ms));
+    /// A signal interrupting `poll` must not restart with the full timeout; returning
+    /// RESTART lets `waitForPid` re-evaluate the deadline-bounded remaining budget.
+    int ready = poll(&pollfd, 1, timeout_in_ms);
 
+    if (ready < 0 && errno == EINTR)
+        return PollPidResult::RESTART;
+
+    /// `ready == 0` is a poll timeout; `ready < 0` (non-EINTR) is a real error.
+    /// Both return FAILED: `waitForPid`'s outer deadline loop treats either as
+    /// "stop waiting" — the timeout is re-evaluated there, not here.
     if (ready <= 0)
         return PollPidResult::FAILED;
 
-    [[maybe_unused]] int err = close(pid_fd);
-    chassert(!err || errno == EINTR);
-
     return PollPidResult::RESTART;
 }
 
@@ -136,6 +149,9 @@ static PollPidResult pollPid(pid_t pid, int timeout_in_ms)
     if (kq == -1)
         return PollPidResult::FAILED;
 
+    /// Releases the kqueue fd on every return path, including early filter-add errors.
+    SCOPE_EXIT({ close(kq); });
+
     struct kevent change{};
     change.ident = 0;
 
@@ -154,12 +170,18 @@ static PollPidResult pollPid(pid_t pid, int timeout_in_ms)
     event.ident = 0;
 
     struct timespec remaining_timespec = {.tv_sec = timeout_in_ms / 1000, .tv_nsec = (timeout_in_ms % 1000) * 1000000};
-    int ready = HANDLE_EINTR(kevent(kq, nullptr, 0, &event, 1, &remaining_timespec));
-    PollPidResult result = ready < 0 ? PollPidResult::FAILED : PollPidResult::RESTART;
 
-    close(kq);
+    /// A signal interrupting `kevent` must not restart with the full timeout; returning
+    /// RESTART lets `waitForPid` re-evaluate the deadline-bounded remaining budget.
+    int ret = kevent(kq, nullptr, 0, &event, 1, &remaining_timespec);
 
-    return result;
+    if (ret < 0 && errno == EINTR)
+        return PollPidResult::RESTART;
+
+    if (ret <= 0)
+        return PollPidResult::FAILED;
+
+    return PollPidResult::RESTART;
 }
 #elif defined(OS_SUNOS)
 
@@ -217,12 +239,15 @@ bool waitForPid(pid_t pid, size_t timeout_in_seconds)
         return process_terminated_normally;
     }
 
-    /// If timeout is positive try waitpid without block in loop until
-    /// process is normally terminated or waitpid return error
+    /// If timeout is positive, poll until the process exits or the total wall
+    /// clock since function entry exceeds the limit. The remaining budget is
+    /// derived from the `watch` started at function entry (never reset) so
+    /// that the `/proc/<pid>` fallback — whose directory fd is always instantly
+    /// ready, causing `pollPid` to return in ~0 ms — still subtracts real
+    /// elapsed time and cannot busy-spin until the child exits on its own.
 
-    /// NOTE: timeout cast to int, since poll() accept int for timeout
-    int timeout_in_ms = static_cast<int>(timeout_in_seconds * 1000);
-    while (timeout_in_ms > 0)
+    const Int64 total_timeout_ms = static_cast<Int64>(timeout_in_seconds * 1000);
+    while (true)
     {
         int waitpid_res = HANDLE_EINTR(waitpid(pid, &status, WNOHANG));
         bool process_terminated_normally = (waitpid_res == pid);
@@ -232,17 +257,14 @@ bool waitForPid(pid_t pid, size_t timeout_in_seconds)
         if (waitpid_res != 0)
             return false;
 
-        watch.restart();
-
-        PollPidResult result = pollPid(pid, timeout_in_ms);
-
-        if (result == PollPidResult::FAILED)
+        const Int64 remaining_ms = total_timeout_ms - static_cast<Int64>(watch.elapsedMilliseconds());
+        if (remaining_ms <= 0)
             return false;
 
-        timeout_in_ms -= watch.elapsedMilliseconds();
+        PollPidResult result = pollPid(pid, static_cast<int>(remaining_ms));
+        if (result == PollPidResult::FAILED)
+            return false;
     }
-
-    return false;
 }
 
 }

src/Coordination/CoordinationSettings.cpp

@@ -89,6 +89,7 @@ namespace ErrorCodes
     DECLARE(UInt64, dispatch_busy_wait_sleep_us, 100, "Sleep duration in microseconds for busy-wait loops in the dispatch and response threads", 0) \
     DECLARE(Milliseconds, stream_suspect_retry_delay_ms, 1000, "Delay before reconnecting to the leader after a stream breaks while the new stream is suspected to be unhealthy", 0) \
     DECLARE(Milliseconds, stream_in_flight_drain_timeout_ms, 5000, "Maximum time to wait for in-flight requests to drain after a stream break (e.g. leader change) before dropping them", 0) \
+    DECLARE(Bool, nuraft_use_bg_thread_for_snapshot_io, false, "Use a background thread for NuRaft snapshot IO instead of reading snapshot objects synchronously from Raft worker threads.", 0) \
     DECLARE(Bool, nuraft_streaming_mode, true, "Enable NuRaft streaming mode, which allows multiple in-flight AppendEntries requests to followers instead of strict one-by-one pipeline. Increases write throughput, especially in high-latency environments (e.g. cross-zone Kubernetes).", 0) \
     DECLARE(UInt64, nuraft_max_log_gap_in_stream, 0, "Maximum number of in-flight log entries per follower when streaming mode is enabled. Acts as a throttling cap. Only effective when nuraft_streaming_mode is true.", 0) \
     DECLARE(UInt64, commit_profiler_real_time_period_ns, 0, "Period for real clock timer of the query profiler on the Keeper commit thread (in nanoseconds). The profiling results appear in system.trace_log with query_id = 'KeeperCommit'. 0 means disabled.", 0) \

src/Coordination/Keeper4LWInfo.h

@@ -3,6 +3,8 @@
 #include <base/types.h>
 #include <Common/Exception.h>
 
+#include <optional>
+
 namespace DB
 {
 
@@ -79,4 +81,15 @@ struct KeeperLogInfo
     uint64_t commit_logs_cache_size{};
 };
 
+struct KeeperClusterMemberInfo
+{
+    int32_t server_id{};
+    String endpoint;
+    bool is_observer{};
+    int32_t priority{};
+    bool is_leader{};
+    bool is_self{};
+    std::optional<uint64_t> last_log_index;
+};
+
 }

src/Coordination/KeeperDispatcher.h

@@ -238,6 +238,11 @@ public:
         return server->getKeeperLogInfo();
     }
 
+    std::vector<KeeperClusterMemberInfo> getClusterMembersInfo() const
+    {
+        return server->getClusterMembersInfo();
+    }
+
     std::vector<KeeperChangelogStatus> getChangelogsStatus() const
     {
         return server->getChangelogsStatus();

src/Coordination/KeeperServer.cpp

@@ -24,6 +24,7 @@
 #include <libnuraft/ptr.hxx>
 #include <libnuraft/peer.hxx>
 #include <libnuraft/raft_server.hxx>
+#include <libnuraft/snapshot_sync_ctx.hxx>
 #include <Poco/Util/AbstractConfiguration.h>
 #include <Poco/Util/Application.h>
 #include <Common/Exception.h>
@@ -87,6 +88,7 @@ namespace CoordinationSetting
     extern const CoordinationSettingsUInt64 stale_log_gap;
     extern const CoordinationSettingsMilliseconds startup_timeout;
     extern const CoordinationSettingsBool nuraft_test_mode;
+    extern const CoordinationSettingsBool nuraft_use_bg_thread_for_snapshot_io;
     extern const CoordinationSettingsBool nuraft_streaming_mode;
     extern const CoordinationSettingsUInt64 nuraft_max_log_gap_in_stream;
     extern const CoordinationSettingsUInt64 nuraft_max_bytes_in_flight_in_stream;
@@ -229,6 +231,17 @@ void setSSLParams(nuraft::asio_service::options & asio_opts)
     const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config();
     asio_opts.ssl_context_provider_server_ = getSslContextProvider(config, "server");
     asio_opts.ssl_context_provider_client_ = getSslContextProvider(config, "client");
+
+    const String client_verification_mode_property = "openSSL.client.verificationMode";
+    if (config.has(client_verification_mode_property))
+    {
+        /// `NuRaft` overrides the client `SSL_CTX` verify mode per connection.
+        /// Only an explicitly configured `none` disables peer verification;
+        /// an absent key keeps the historical secure-by-default behavior.
+        asio_opts.skip_verification_
+            = Poco::Net::Utility::convertVerificationMode(config.getString(client_verification_mode_property))
+                == Poco::Net::Context::VERIFY_NONE;
+    }
 }
 #endif
 
@@ -579,6 +592,7 @@ void KeeperServer::launchRaftServer(const Poco::Util::AbstractConfiguration & co
         = getValueOrMaxInt32AndLogWarning(coordination_settings[CoordinationSetting::max_requests_append_size], "max_requests_append_size", log);
     params.max_append_size_bytes_ = coordination_settings[CoordinationSetting::max_requests_append_bytes_size];
 
+    params.use_bg_thread_for_snapshot_io_ = coordination_settings[CoordinationSetting::nuraft_use_bg_thread_for_snapshot_io];
     params.max_log_gap_in_stream_
         = getValueOrMaxInt32AndLogWarning(coordination_settings[CoordinationSetting::nuraft_max_log_gap_in_stream], "nuraft_max_log_gap_in_stream", log);
     params.max_bytes_in_flight_in_stream_
@@ -616,6 +630,10 @@ void KeeperServer::launchRaftServer(const Poco::Util::AbstractConfiguration & co
     {
 #if USE_SSL
         setSSLParams(asio_opts);
+        if (asio_opts.skip_verification_)
+            LOG_WARNING(
+                log,
+                "Keeper Raft peer certificate verification is disabled because `openSSL.client.verificationMode` is set to `none`");
 #else
         throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSL support for NuRaft is disabled because ClickHouse was built without SSL support.");
 #endif
@@ -772,6 +790,7 @@ void KeeperServer::shutdownRaftServer()
 void KeeperServer::shutdown()
 {
     shutdownRaftServer();
+    nuraft::snapshot_io_mgr::shutdown_instance();
     state_manager->flushAndShutDownLogStore();
     state_machine->shutdownStorage();
 }
@@ -1478,6 +1497,32 @@ int64_t KeeperServer::getLeaderID() const
     return raft_instance->get_leader();
 }
 
+std::vector<KeeperClusterMemberInfo> KeeperServer::getClusterMembersInfo() const
+{
+    const auto cluster_config = state_manager->getClusterConfig();
+    const auto & servers = cluster_config->get_servers();
+
+    const int32_t leader_id = static_cast<int32_t>(raft_instance->get_leader());
+    const uint64_t self_log_idx = raft_instance->get_last_log_idx();
+
+    std::vector<KeeperClusterMemberInfo> result;
+    result.reserve(servers.size());
+    for (const auto & cfg : servers)
+    {
+        KeeperClusterMemberInfo info;
+        info.server_id = cfg->get_id();
+        info.endpoint = cfg->get_endpoint();
+        info.is_observer = cfg->is_learner();
+        info.priority = cfg->get_priority();
+        info.is_leader = (cfg->get_id() == leader_id);
+        info.is_self = (cfg->get_id() == server_id);
+        if (info.is_self)
+            info.last_log_index = self_log_idx;
+        result.push_back(std::move(info));
+    }
+    return result;
+}
+
 void KeeperServer::yieldLeadership()
 {
     if (isLeader())

src/Coordination/KeeperServer.h

@@ -205,6 +205,8 @@ public:
 
     KeeperLogInfo getKeeperLogInfo();
 
+    std::vector<KeeperClusterMemberInfo> getClusterMembersInfo() const;
+
     std::vector<KeeperChangelogStatus> getChangelogsStatus() const;
 
     bool requestLeader();

src/Coordination/KeeperStateManager.h

@@ -88,6 +88,12 @@ public:
         return configuration_wrapper.cluster_config->get_servers().size();
     }
 
+    ClusterConfigPtr getClusterConfig() const
+    {
+        std::lock_guard lock(configuration_wrapper_mutex);
+        return configuration_wrapper.cluster_config;
+    }
+
     /// Read all log entries in log store from the begging and return latest config (with largest log_index)
     ClusterConfigPtr getLatestConfigFromLogStore() const;
 

src/Core/ProtocolDefines.h

@@ -54,7 +54,9 @@ static constexpr auto DBMS_MIN_REVISION_WITH_QUERY_AND_LINE_NUMBERS = 54475;
 
 static constexpr auto DBMS_MERGE_TREE_PART_INFO_VERSION = 1;
 
-static constexpr auto DBMS_QUERY_PLAN_SERIALIZATION_VERSION = 0;
+static constexpr auto DBMS_QUERY_PLAN_SERIALIZATION_VERSION = 1;
+/// Version 1 added the initiator's settings changes to the task.
+static constexpr auto DBMS_DISTRIBUTED_TASK_SERIALIZATION_VERSION = 1;
 
 static constexpr auto DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET = 54441;
 

src/Core/Settings.cpp

@@ -755,6 +755,8 @@ Possible values:
 )", 0) \
     DECLARE(Bool, enable_hdfs_pread, true, R"(
 Enable or disables pread for HDFS files. By default, `hdfsPread` is used. If disabled, `hdfsRead` and `hdfsSeek` will be used to read hdfs files.)", 0) \
+    DECLARE(Bool, use_reader_executor, false, R"(
+Experimental. Route reads through the new pipeline `ReaderExecutor` instead of the legacy matryoshka of read buffers. Falls back to the legacy path for configurations the executor does not yet support.)", EXPERIMENTAL) \
     DECLARE(Bool, azure_skip_empty_files, false, R"(
 Enables or disables skipping empty files in S3 engine.
 
@@ -2437,11 +2439,11 @@ When set to 1, a random seed is generated, when set to a value > 1, that value i
 This is intended for testing to find errors caused by different join orderings.
 )", EXPERIMENTAL) \
     \
-    DECLARE(Bool, enable_join_transitive_predicates, false, R"(
+    DECLARE(Bool, enable_join_transitive_predicates, true, R"(
 Infer transitive equi-join predicates from existing join conditions.
 For example, given `A.x = B.x` and `B.x = C.x`, a synthetic `A.x = C.x` predicate
 is added so the join order optimizer can consider direct (A JOIN C) plans.
-)", EXPERIMENTAL) \
+)", BETA) \
     \
     DECLARE(Bool, query_plan_join_shard_by_pk_ranges, false, R"(
 Apply sharding for JOIN if join keys contain a prefix of PRIMARY KEY for both tables. Supported for hash, parallel_hash and full_sorting_merge algorithms. Usually does not speed up queries but may lower memory consumption.
@@ -8110,7 +8112,7 @@ Run all tasks of a distributed query plan locally. Useful for testing and debugg
     DECLARE(NonZeroUInt64, distributed_plan_default_shuffle_join_bucket_count, 8, R"(
 Default number of buckets for distributed shuffle-hash-join.
 )", EXPERIMENTAL) \
-    DECLARE(UInt64, distributed_plan_default_reader_bucket_count, 8, R"(
+    DECLARE(NonZeroUInt64, distributed_plan_default_reader_bucket_count, 8, R"(
 Default number of tasks for parallel reading in distributed query. Tasks are spread across between replicas.
 )", EXPERIMENTAL) \
     DECLARE(Bool, distributed_plan_optimize_exchanges, true, R"(

src/Core/SettingsChangesHistory.cpp

@@ -58,9 +58,11 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
             {"query_plan_push_limit_by_into_sort", false, true, "New setting that pushes a per-stream LIMIT BY into the sort pipeline when LIMIT BY's columns are a prefix of ORDER BY, reducing rows flowing through the final merge."},
             {"analyzer_compatibility_prefer_alias_over_subcolumn", false, false, "New compatibility setting"},
             {"query_plan_max_set_size_for_projection_match", 0, 10000, "Added new setting that bounds the cost of content-hashing IN-clause sets in the projection matcher (today: aggregate projection). Sets larger than the limit are treated as non-matching. Zero disables content-hash comparison entirely (compatibility value: projection match never succeeds for nodes with IN-sets)."},
+            {"use_reader_executor", false, false, "New experimental setting to route reads through the new pipeline ReaderExecutor instead of the legacy matryoshka of read buffers."},
             {"function_base58_max_input_size", 0, 10000, "New setting that limits the input size of `base58Encode`, `base58Decode` and `tryBase58Decode` (whose conversion is quadratic in the input length) to 10 KB by default. The compatibility value `0` disables the limit, restoring the previous behavior of accepting arbitrarily large inputs."},
             {"format_avro_schema_registry_max_retries", 0, 5, "New setting controlling the maximum number of retries for transient failures (transport timeouts, connection refused, DNS errors, HTTP 5xx/408/429) when communicating with the Confluent Schema Registry. Set to 0 to disable retries. Previous behavior (no retries) is preserved by `compatibility = '26.5'`."},
             {"format_avro_schema_registry_retry_initial_backoff_ms", 100, 100, "New setting controlling the initial backoff (in milliseconds) before retrying a failed Confluent Schema Registry request. The backoff doubles on each retry, capped at 10 seconds. Has no effect when `format_avro_schema_registry_max_retries = 0` (the pre-26.6 behavior restored by `compatibility = '26.5'`)."},
+            {"enable_join_transitive_predicates", false, true, "Turn on enable_join_transitive_predicates by default"},
         });
         addSettingsChanges(settings_changes_history, "26.5",
         {

src/DataTypes/DataTypesNumber.h

@@ -24,9 +24,7 @@ public:
     bool equals(const IDataType & rhs) const override { return typeid(rhs) == typeid(*this); }
 
     bool canBeUsedAsVersion() const override { return true; }
-    /// Custom data types with numeric representations are not necessarily summable.
-    /// For example, the `Bool` type isn't.
-    bool isSummable() const override { return !this->hasCustomName(); }
+    bool isSummable() const override { return true; }
     bool canBeUsedInBitOperations() const override { return true; }
     bool canBeUsedInBooleanContext() const override { return WhichDataType(TypeToTypeIndex<T>).isNativeNumber(); }
     bool canBeInsideNullable() const override { return true; }

src/DataTypes/Serializations/SerializationDateTime.cpp

@@ -56,13 +56,13 @@ readText(time_t & x, ReadBuffer & istr, const FormatSettings & settings, const D
             break;
     }
 
-    x = std::max<time_t>(0, x);
+    x = std::clamp<time_t>(x, 0, static_cast<time_t>(0xFFFFFFFF));
 }
 
 inline void readAsIntText(time_t & x, ReadBuffer & istr)
 {
     readIntText(x, istr);
-    x = std::max<time_t>(0, x);
+    x = std::clamp<time_t>(x, 0, static_cast<time_t>(0xFFFFFFFF));
 }
 
 inline bool tryReadText(
@@ -82,7 +82,7 @@ inline bool tryReadText(
             break;
     }
 
-    x = std::max<time_t>(0, x);
+    x = std::clamp<time_t>(x, 0, static_cast<time_t>(0xFFFFFFFF));
     return res;
 }
 
@@ -90,7 +90,7 @@ inline bool tryReadAsIntText(time_t & x, ReadBuffer & istr)
 {
     if (!tryReadIntText(x, istr))
         return false;
-    x = std::max<time_t>(0, x);
+    x = std::clamp<time_t>(x, 0, static_cast<time_t>(0xFFFFFFFF));
     return true;
 }
 
@@ -245,7 +245,7 @@ bool SerializationDateTime::tryDeserializeTextJSON(IColumn & column, ReadBuffer
     }
     else
     {
-        if (!tryReadIntText(x, istr))
+        if (!tryReadAsIntText(x, istr))
             return false;
     }
 

src/Databases/DataLake/DatabaseDataLake.cpp

@@ -71,6 +71,7 @@ namespace DatabaseDataLakeSetting
     extern const DatabaseDataLakeSettingsString onelake_tenant_id;
     extern const DatabaseDataLakeSettingsString onelake_client_id;
     extern const DatabaseDataLakeSettingsString onelake_client_secret;
+    extern const DatabaseDataLakeSettingsBool onelake_use_blob_endpoint;
     extern const DatabaseDataLakeSettingsString dlf_access_key_id;
     extern const DatabaseDataLakeSettingsString dlf_access_key_secret;
     extern const DatabaseDataLakeSettingsString google_project_id;
@@ -631,7 +632,8 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con
         azure_configuration->setInitializationAsOneLake(
             rest_catalog->getClientId(),
             rest_catalog->getClientSecret(),
-            rest_catalog->getTenantId()
+            rest_catalog->getTenantId(),
+            settings[DatabaseDataLakeSetting::onelake_use_blob_endpoint].value
         );
 #else
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server does not contain support for storage type Azure for Iceberg OneLake catalog");

src/Databases/DataLake/DatabaseDataLakeSettings.cpp

@@ -35,6 +35,7 @@ namespace ErrorCodes
     DECLARE(String, onelake_tenant_id, "", "Tenant id from azure", 0) \
     DECLARE(String, onelake_client_id, "", "Client id from azure", 0) \
     DECLARE(String, onelake_client_secret, "", "Client secret from azure", 0) \
+    DECLARE(Bool, onelake_use_blob_endpoint, true, "Use the Blob endpoint (.blob.fabric.microsoft.com) for OneLake. When disabled, the DFS endpoint (.dfs.fabric.microsoft.com) is used instead", 0) \
     DECLARE(String, google_project_id, "", "Google Cloud project ID for BigLake. Required for BigLake catalog. Used in x-goog-user-project header. If not set and google_adc_quota_project_id is provided, it latter will be used", 0) \
     DECLARE(String, google_service_account, "", "Google Cloud service account email for metadata service authentication. Default: 'default'. Only used when ADC credentials are not provided", 0) \
     DECLARE(String, google_metadata_service, "", "Google Cloud metadata service endpoint for token retrieval. Default: 'metadata.google.internal'. Only used when ADC credentials are not provided", 0) \

src/Disks/DiskEncrypted.h

@@ -3,6 +3,8 @@
 #include "config.h"
 
 #if USE_SSL
+#include <mutex>
+
 #include <Disks/IDisk.h>
 #include <Common/MultiVersion.h>
 #include <Disks/FakeDiskTransaction.h>
@@ -373,7 +375,14 @@ public:
 
     MetadataStoragePtr getMetadataStorage() override
     {
-        return std::make_shared<MetadataStorageWithPathWrapper>(delegate->getMetadataStorage(), disk_path);
+        /// Cache the wrapper so that the metadata storage pointer is stable across calls (like `DiskObjectStorage`),
+        /// which code such as `isStoredOnTheSameDisk` relies on to hardlink parts instead of copying them; lazily,
+        /// because a non-object-storage delegate does not implement `getMetadataStorage`.
+        std::call_once(metadata_storage_init_flag, [this]
+        {
+            metadata_storage = std::make_shared<MetadataStorageWithPathWrapper>(delegate->getMetadataStorage(), disk_path);
+        });
+        return metadata_storage;
     }
 
     std::unordered_map<String, String> getSerializedMetadata(const std::vector<String> & paths) const override;
@@ -413,6 +422,10 @@ private:
     const String disk_absolute_path;
     MultiVersion<DiskEncryptedSettings> current_settings;
     bool use_fake_transaction;
+
+    /// Lazily-initialized stable wrapper returned by getMetadataStorage(); see the comment there.
+    std::once_flag metadata_storage_init_flag;
+    MetadataStoragePtr metadata_storage;
 };
 
 }

src/Disks/DiskLocal.cpp

@@ -379,13 +379,13 @@ void DiskLocal::prepareRead(
 
     /// Do not fail eagerly if the file doesn't exist or can't be stat'd.
     /// The error should come at read time with the proper error code
-    /// (e.g. FILE_DOESNT_EXIST for broken projections).
+    /// (e.g. FILE_DOESNT_EXIST for broken projections). Mark the size unknown in
+    /// that case so the reader opens the file rather than treating a zero size as
+    /// an empty file.
     std::error_code ec;
     auto file_size = fs::file_size(full_path, ec);
-    if (ec)
-        file_size = 0;
 
-    StoredObject obj(full_path.string(), full_path.string(), file_size);
+    StoredObject obj(full_path.string(), full_path.string(), ec ? StoredObject::UnknownSize : file_size);
 
     /// No gather for local disk — the source buffer is returned directly.
     pipeline.setLocalFileSource(

src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp

@@ -691,15 +691,17 @@ void AzureSettingsByEndpoint::loadFromConfig(
 
     for (const String & key : config_keys)
     {
+        /// Accept both the modern `<object_storage_type>azure</object_storage_type>` and the legacy
+        /// `<type>azure_blob_storage</type>` declaration forms. Without the latter, an endpoint
+        /// declared the legacy way is never loaded into the map, so its settings are silently lost.
+        String disk_type;
         if (config.has(config_prefix + "." + key + ".object_storage_type"))
-        {
-            const auto &object_storage_type = config.getString(config_prefix + "." + key + ".object_storage_type");
-            if (object_storage_type != "azure" && object_storage_type != "azure_blob_storage")
-            {
-                /// Then its not an azure config
-                continue;
-            }
+            disk_type = config.getString(config_prefix + "." + key + ".object_storage_type");
+        else if (config.has(config_prefix + "." + key + ".type"))
+            disk_type = config.getString(config_prefix + "." + key + ".type");
 
+        if (disk_type == "azure" || disk_type == "azure_blob_storage")
+        {
             const auto key_path = config_prefix + "." + key;
             String endpoint_path = key_path + ".connection_string";
 
@@ -714,7 +716,7 @@ void AzureSettingsByEndpoint::loadFromConfig(
                     if (!config.has(endpoint_path))
                     {
                         throw Exception(ErrorCodes::LOGICAL_ERROR, "URL not provided for azure blob storage disk {}",
-                                        object_storage_type);
+                                        disk_type);
                     }
                 }
             }

src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp

@@ -12,10 +12,13 @@
 #include <Common/getRandomASCIIString.h>
 #include <Disks/IO/ReadBufferFromAzureBlobStorage.h>
 #include <Disks/IO/WriteBufferFromAzureBlobStorage.h>
+#include <Disks/IO/WriteBufferFromAzureDataLakeStorage.h>
 #include <Disks/IO/ReadBufferFromRemoteFSGather.h>
 #include <Disks/IO/AsynchronousBoundedReadBuffer.h>
 #include <IO/AzureBlobStorage/copyAzureBlobStorageFile.h>
 
+#include <azure/storage/files/datalake/datalake_file_client.hpp>
+#include <azure/storage/files/datalake/datalake_options.hpp>
 
 #include <IO/WriteBufferFromString.h>
 #include <IO/copyData.h>
@@ -267,7 +270,17 @@ SmallObjectDataWithMetadata AzureObjectStorage::readSmallObjectAndGetObjectMetad
 }
 
 
-/// Open the file for write and return WriteBufferFromFileBase object.
+std::unique_ptr<Azure::Storage::Files::DataLake::DataLakeFileClient>
+AzureObjectStorage::buildDataLakeFileClient(const String & blob_path) const
+{
+    return std::make_unique<Azure::Storage::Files::DataLake::DataLakeFileClient>(
+        makeAdlsGen2FileClient(
+            connection_params.endpoint,
+            auth_method,
+            connection_params.client_options,
+            blob_path));
+}
+
 std::unique_ptr<WriteBufferFromFileBase> AzureObjectStorage::writeObject( /// NOLINT
     const StoredObject & object,
     WriteMode mode,
@@ -280,13 +293,27 @@ std::unique_ptr<WriteBufferFromFileBase> AzureObjectStorage::writeObject( /// NO
 
     LOG_TEST(log, "Writing file: {}", object.remote_path);
 
+    auto blob_storage_log = BlobStorageLogWriter::create(name);
+    if (blob_storage_log)
+        blob_storage_log->local_path = object.local_path;
+
     ThreadPoolCallbackRunnerUnsafe<void> scheduler;
     if (write_settings.azure_allow_parallel_part_upload)
         scheduler = threadPoolCallbackRunnerUnsafe<void>(getThreadPoolWriter(), ThreadName::REMOTE_FS_WRITE_THREAD_POOL);
 
-    auto blob_storage_log = BlobStorageLogWriter::create(name);
-    if (blob_storage_log)
-        blob_storage_log->local_path = object.local_path;
+    if (isAdlsGen2Endpoint(connection_params.endpoint))
+    {
+        return std::make_unique<WriteBufferFromAzureDataLakeStorage>(
+            connection_params.endpoint,
+            auth_method,
+            connection_params.client_options,
+            object.remote_path,
+            write_settings.use_adaptive_write_buffer ? write_settings.adaptive_write_buffer_initial_size : buf_size,
+            patchSettings(write_settings),
+            settings.get(),
+            connection_params.getContainer(),
+            std::move(blob_storage_log));
+    }
 
     return std::make_unique<WriteBufferFromAzureBlobStorage>(
         client.get(),
@@ -318,12 +345,20 @@ void AzureObjectStorage::removeObjectImpl(
     bool success = false;
     try
     {
-        auto delete_info = client_ptr->GetBlobClient(path).Delete();
-        success = delete_info.Value.Deleted;
-        if (!if_exists && !delete_info.Value.Deleted)
-            throw Exception(
-                ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Failed to delete file (path: {}) in AzureBlob Storage, reason: {}",
-                path, delete_info.RawResponse ? delete_info.RawResponse->GetReasonPhrase() : "Unknown");
+        if (isAdlsGen2Endpoint(connection_params.endpoint))
+        {
+            buildDataLakeFileClient(path)->Delete();
+            success = true;
+        }
+        else
+        {
+            auto delete_info = client_ptr->GetBlobClient(path).Delete();
+            success = delete_info.Value.Deleted;
+            if (!if_exists && !delete_info.Value.Deleted)
+                throw Exception(
+                    ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Failed to delete file (path: {}) in AzureBlob Storage, reason: {}",
+                    path, delete_info.RawResponse ? delete_info.RawResponse->GetReasonPhrase() : "Unknown");
+        }
     }
     catch (const Azure::Storage::StorageException & e)
     {
@@ -471,6 +506,13 @@ void AzureObjectStorage::removeObjectsIfExist(const StoredObjects & objects)
     auto client_ptr = client.get();
     auto blob_storage_log = BlobStorageLogWriter::create(name);
 
+    if (isAdlsGen2Endpoint(connection_params.endpoint))
+    {
+        for (const auto & object : objects)
+            removeObjectImpl(object, client_ptr, /*if_exists=*/ true, blob_storage_log);
+        return;
+    }
+
     removeObjectsBatchIfExists(objects, client_ptr, blob_storage_log);
 }
 
@@ -518,6 +560,7 @@ ObjectMetadata AzureObjectStorage::getObjectMetadata(const std::string & path, b
 
     ObjectMetadata result;
     result.size_bytes = properties.BlobSize;
+    result.etag = properties.ETag.ToString();
     if (!properties.Metadata.empty())
     {
         result.attributes.emplace();

src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h

@@ -8,6 +8,7 @@
 #include <Common/BlobStorageLogWriter.h>
 #include <Common/MultiVersion.h>
 #include <azure/storage/blobs.hpp>
+#include <azure/storage/files/datalake/datalake_file_client.hpp>
 #include <azure/core/http/curl_transport.hpp>
 #include <Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
 
@@ -140,6 +141,8 @@ private:
         const std::shared_ptr<const AzureBlobStorage::ContainerClient> & client_ptr,
         BlobStorageLogWriterPtr blob_storage_log);
 
+    std::unique_ptr<Azure::Storage::Files::DataLake::DataLakeFileClient> buildDataLakeFileClient(const String & blob_path) const;
+
     const String name;
     AzureBlobStorage::AuthMethod auth_method;
     /// client used to access the files in the Blob Storage cloud

src/Disks/DiskObjectStorage/ObjectStorages/Cached/CachedObjectStorage.cpp

@@ -69,6 +69,19 @@ std::unique_ptr<ReadBufferFromFileBase> CachedObjectStorage::readObject( /// NOL
     return object_storage->readObject(object, patchSettings(read_settings), read_hint, use_external_buffer, restrict_seek);
 }
 
+SmallObjectDataWithMetadata CachedObjectStorage::readSmallObjectAndGetObjectMetadata( /// NOLINT
+    const StoredObject & object,
+    const ReadSettings & read_settings,
+    size_t max_size_bytes,
+    std::optional<size_t> read_hint) const
+{
+    /// Delegate to the underlying storage so its override populates `ObjectMetadata`
+    /// (notably `etag`). The base `IObjectStorage` implementation only reads the bytes
+    /// and leaves metadata empty, which silently breaks Iceberg `version-hint.text`
+    /// CAS updates on cached S3/Azure storage.
+    return object_storage->readSmallObjectAndGetObjectMetadata(object, patchSettings(read_settings), max_size_bytes, read_hint);
+}
+
 void CachedObjectStorage::prepareRead(
     ObjectStoragePtr /* storage */,
     const StoredObjects & objects,

src/Disks/DiskObjectStorage/ObjectStorages/Cached/CachedObjectStorage.h

@@ -38,6 +38,12 @@ public:
         bool use_external_buffer = false,
         bool restrict_seek = false) const override;
 
+    SmallObjectDataWithMetadata readSmallObjectAndGetObjectMetadata( /// NOLINT
+        const StoredObject & object,
+        const ReadSettings & read_settings,
+        size_t max_size_bytes,
+        std::optional<size_t> read_hint = {}) const override;
+
     void prepareRead(
         ObjectStoragePtr storage,
         const StoredObjects & objects,

src/Disks/DiskObjectStorage/ObjectStorages/StoredObject.h

@@ -15,16 +15,23 @@ namespace DB
 /// Object metadata: path, size, path_key_for_cache.
 struct StoredObject
 {
+    /// Sentinel meaning "size is not known yet", set by callers that can't
+    /// determine the size up-front (S3 `HEAD` without `Content-Length`, local
+    /// `stat` failure). Consumers doing offset arithmetic (`OffsetMap`,
+    /// `ReaderExecutor`) must check for it and stream until EOF instead of
+    /// treating it as a real file size.
+    static constexpr uint64_t UnknownSize = std::numeric_limits<uint64_t>::max();
+
     String remote_path; /// abs path
     String local_path; /// or equivalent "metadata_path"
 
     /// NOTE: the type must stay uint64_t — MetadataStorageFromDisk removal log serializes it as UInt64 LE.
-    uint64_t bytes_size = std::numeric_limits<uint64_t>::max();
+    uint64_t bytes_size = UnknownSize;
 
     explicit StoredObject(
         const String & remote_path_ = "",
         const String & local_path_ = "",
-        uint64_t bytes_size_ = std::numeric_limits<uint64_t>::max())
+        uint64_t bytes_size_ = UnknownSize)
         : remote_path(remote_path_)
         , local_path(local_path_)
         , bytes_size(bytes_size_)

src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp

@@ -46,6 +46,7 @@ namespace ErrorCodes
     extern const int ARGUMENT_OUT_OF_BOUND;
     extern const int UNKNOWN_FILE_SIZE;
     extern const int CACHE_CANNOT_WRITE_TO_CACHE_DISK;
+    extern const int CANNOT_READ_ALL_DATA;
 }
 
 CachedOnDiskReadBufferFromFile::ReadInfo::ReadInfo(
@@ -250,6 +251,19 @@ namespace
 using ReadType = CachedOnDiskReadBufferFromFile::ReadType;
 using ReadInfo = CachedOnDiskReadBufferFromFile::ReadInfo;
 
+/// Format the remote object's last modification time for diagnostics, alongside its size: a recent
+/// timestamp confirms the object was overwritten between listing and reading. Returns "None" when the
+/// metadata is unavailable (e.g. a non-remote buffer).
+String formatRemoteFileLastModified(const std::optional<RemoteFileMetadata> & metadata)
+{
+    if (!metadata.has_value())
+        return "None";
+
+    WriteBufferFromOwnString out;
+    writeDateTimeText(metadata->last_modification_time, out);
+    return fmt::format("{} (unix time {})", out.str(), metadata->last_modification_time);
+}
+
 std::shared_ptr<ReadBufferFromFileBase> getCacheReadBuffer(
     const FileSegment & file_segment,
     ReadInfo & info)
@@ -792,16 +806,66 @@ bool CachedOnDiskReadBufferFromFile::predownloadForFileSegment(
             if (!state.bytes_to_predownload || !has_more_data)
             {
                 if (state.bytes_to_predownload)
+                {
+                    /// We failed to predownload all the bytes that were needed, so this downloader
+                    /// will not continue. Release the segment (transition it to
+                    /// PARTIALLY_DOWNLOADED_NO_CONTINUATION) before throwing, so that other readers
+                    /// waiting on this segment can take over instead of waiting for a download that
+                    /// will never finish. This mirrors the EOF handling in readFromFileSegment.
+                    if (file_segment.isDownloader())
+                        file_segment.setDownloadFinishedWithoutContinuation();
+
+                    /// The remote object may have been overwritten with shorter content
+                    /// between listing and reading. Check the actual remote file metadata (size and
+                    /// last modification time, fetched in a single request) to distinguish that from
+                    /// a genuine logic error.
+                    const auto remote_metadata = state.buf->getRemoteFileMetadata();
+                    /// The object size known at buffer creation time (from listing). Logged alongside
+                    /// the actual size so a truncation is immediately visible in diagnostics.
+                    const auto expected_object_size = state.buf->tryGetFileSize();
+                    if (remote_metadata.has_value()
+                        && remote_metadata->size == file_segment.getCurrentWriteOffset())
+                    {
+                        /// We reached the real end of the (now shorter) object while
+                        /// predownloading, before reaching the bytes the reader actually
+                        /// needs: those lie beyond the truncated object, because
+                        /// offset == current_write_offset + bytes_to_predownload > remote_metadata->size.
+                        ///
+                        /// There is no valid data to return for `offset`. We must NOT treat
+                        /// this as EOF (return a short/zero read): the caller would then
+                        /// consume buffer bytes that were never written -- a
+                        /// use-of-uninitialized-value under MSan, and silent garbage in
+                        /// release. Fail with a regular CANNOT_READ_ALL_DATA instead of a
+                        /// LOGICAL_ERROR: it does not alert as a server bug and is retryable
+                        /// while the object is being concurrently replaced.
+                        throw Exception(
+                            ErrorCodes::CANNOT_READ_ALL_DATA,
+                            "Remote object was truncated between listing and reading: "
+                            "actual object size {}, expected object size {}, last modified {}, "
+                            "but need to read until offset {}. Current file segment: {}",
+                            remote_metadata->size,
+                            expected_object_size ? std::to_string(*expected_object_size) : "None",
+                            formatRemoteFileLastModified(remote_metadata),
+                            offset,
+                            file_segment.getInfoForLog());
+                    }
+
                     throw Exception(
                         ErrorCodes::LOGICAL_ERROR,
                         "Failed to predownload remaining {} bytes. Current file segment: {}, "
-                        "current download offset: {}, expected: {}, eof: {}, internal buffer size: {}",
+                        "current download offset: {}, expected: {}, eof: {}, "
+                        "internal buffer size: {}, remote object size: {}, "
+                        "expected object size: {}, remote object last modified: {}",
                         state.bytes_to_predownload,
                         file_segment.range().toString(),
                         file_segment.getCurrentWriteOffset(),
                         offset,
                         state.buf->eof(),
-                        state.buf->internalBuffer().size());
+                        state.buf->internalBuffer().size(),
+                        remote_metadata ? std::to_string(remote_metadata->size) : "None",
+                        expected_object_size ? std::to_string(*expected_object_size) : "None",
+                        formatRemoteFileLastModified(remote_metadata));
+                }
 
                 chassert(!state.buf->hasPendingData());
                 auto result = state.buf->hasPendingData();
@@ -1372,12 +1436,12 @@ size_t CachedOnDiskReadBufferFromFile::readFromFileSegment(
         else
             download_finished_time = "None";
 
-        std::optional<size_t> object_size;
+        std::optional<RemoteFileMetadata> remote_metadata;
         std::optional<size_t> impl_read_until_position;
         std::optional<std::string> impl_read_stop_reason;
         if (state.read_type != ReadType::CACHED)
         {
-            object_size = state.buf->getRemoteFileSize();
+            remote_metadata = state.buf->getRemoteFileMetadata();
 
 #if USE_AWS_S3
             if (const auto * s3_buf = dynamic_cast<const ReadBufferFromS3 *>(state.buf.get()))
@@ -1388,21 +1452,26 @@ size_t CachedOnDiskReadBufferFromFile::readFromFileSegment(
 #endif
         }
 
-        if (object_size.has_value() && *object_size == offset)
+        if (remote_metadata.has_value() && remote_metadata->size == offset)
         {
             /// The remote object is smaller than file_size_ indicated, e.g. the object was
-            /// overwritten with shorter content between listing and reading.
-            /// Treat this as a legitimate EOF rather than a logic error.
-            LOG_WARNING(
-                log,
-                "Remote object is smaller than expected: read {} bytes but expected to read until position {}. "
-                "Actual object size: {}, expected size: {}, stop reason: {}. Treating as EOF.",
-                offset, info.read_until_position, *object_size, file_size_,
-                impl_read_stop_reason ? *impl_read_stop_reason : "None");
+            /// overwritten with shorter content between listing and reading. This downloader
+            /// will not continue: release the segment so other readers waiting on it can take
+            /// over, then fail with a regular CANNOT_READ_ALL_DATA -- retryable while the object
+            /// is being concurrently replaced, and (unlike a LOGICAL_ERROR) it does not alert as a
+            /// server bug. The predownloadForFileSegment and readBigAt paths fail the same way.
             if (file_segment.isDownloader())
                 file_segment.setDownloadFinishedWithoutContinuation();
-            info.read_until_position = offset;
-            return 0;
+            throw Exception(
+                ErrorCodes::CANNOT_READ_ALL_DATA,
+                "Remote object was truncated between listing and reading: "
+                "read until offset {} but expected to read until position {}. "
+                "Actual object size {}, expected object size {}, last modified {}, stop reason: {}. "
+                "Current file segment: {}",
+                offset, info.read_until_position, remote_metadata->size, file_size_,
+                formatRemoteFileLastModified(remote_metadata),
+                impl_read_stop_reason ? *impl_read_stop_reason : "None",
+                file_segment.getInfoForLog());
         }
 
         throw Exception(
@@ -1411,6 +1480,7 @@ size_t CachedOnDiskReadBufferFromFile::readFromFileSegment(
             "file offset: {}, "
             "read bytes: {}, "
             "object size: {}, "
+            "object last modified: {}, "
             "expected object size: {}, "
             "reading until: {}, "
             "read type: {}, "
@@ -1428,7 +1498,8 @@ size_t CachedOnDiskReadBufferFromFile::readFromFileSegment(
             "current file segment: {}",
             offset + size,
             size,
-            object_size ? std::to_string(*object_size) : "None",
+            remote_metadata ? std::to_string(remote_metadata->size) : "None",
+            formatRemoteFileLastModified(remote_metadata),
             file_size_,
             info.read_until_position,
             toString(state.read_type),
@@ -1585,6 +1656,9 @@ size_t CachedOnDiskReadBufferFromFile::readBigAt(
 
         if (!size)
         {
+            /// A truncated remote object (overwritten with shorter content between listing and
+            /// reading) is handled in readFromFileSegment, which throws CANNOT_READ_ALL_DATA before
+            /// returning here. Reaching this point with a zero-sized read is therefore unexpected.
             throw Exception(
                 ErrorCodes::LOGICAL_ERROR,
                 "Cannot read all data. Offset: {} (initial offset: {}), read bytes {}/{}",

src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp

@@ -344,9 +344,15 @@ std::optional<size_t> ReadBufferFromAzureBlobStorage::tryGetFileSize()
     return file_size;
 }
 
-std::optional<size_t> ReadBufferFromAzureBlobStorage::getRemoteFileSize() const
+std::optional<RemoteFileMetadata> ReadBufferFromAzureBlobStorage::getRemoteFileMetadata() const
 {
-    return static_cast<size_t>(blob_container_client->GetBlobClient(path).GetProperties().Value.BlobSize);
+    const auto properties = blob_container_client->GetBlobClient(path).GetProperties().Value;
+    const auto last_modification_time = std::chrono::duration_cast<std::chrono::seconds>(
+        static_cast<std::chrono::system_clock::time_point>(properties.LastModified).time_since_epoch())
+        .count();
+    return RemoteFileMetadata{
+        .size = static_cast<size_t>(properties.BlobSize),
+        .last_modification_time = static_cast<time_t>(last_modification_time)};
 }
 
 size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t range_begin, const std::function<bool(size_t)> & /*progress_callback*/) const

src/Disks/IO/ReadBufferFromAzureBlobStorage.h

@@ -53,7 +53,7 @@ public:
 
     std::optional<size_t> tryGetFileSize() override;
 
-    std::optional<size_t> getRemoteFileSize() const override;
+    std::optional<RemoteFileMetadata> getRemoteFileMetadata() const override;
 
     size_t readBigAt(char * to, size_t n, size_t range_begin, const std::function<bool(size_t)> & progress_callback) const override;
 

src/Disks/IO/WriteBufferFromAzureDataLakeStorage.cpp

@@ -0,0 +1,303 @@
+#include "config.h"
+
+#if USE_AZURE_BLOB_STORAGE
+
+#include <base/sleep.h>
+
+#include <Disks/IO/WriteBufferFromAzureDataLakeStorage.h>
+#include <Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h>
+#include <IO/AzureBlobStorage/isRetryableAzureException.h>
+#include <Common/logger_useful.h>
+#include <Common/Stopwatch.h>
+
+#include <azure/core/io/body_stream.hpp>
+#include <azure/storage/files/datalake/datalake_options.hpp>
+
+
+namespace ProfileEvents
+{
+    extern const Event AzureUpload;
+}
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int AZURE_BLOB_STORAGE_ERROR;
+}
+
+namespace
+{
+
+Azure::Storage::Files::DataLake::DataLakeClientOptions toDataLakeOptions(
+    const Azure::Storage::Blobs::BlobClientOptions & blob_client_options)
+{
+    Azure::Storage::Files::DataLake::DataLakeClientOptions out;
+    static_cast<Azure::Core::_internal::ClientOptions &>(out)
+        = static_cast<const Azure::Core::_internal::ClientOptions &>(blob_client_options);
+    return out;
+}
+
+String prefixedBlobPath(const AzureBlobStorage::Endpoint & endpoint, const String & blob_path_)
+{
+    String full = endpoint.prefix;
+    if (!full.empty() && !full.ends_with('/'))
+        full += '/';
+    full += blob_path_;
+    return full;
+}
+
+String buildAdlsGen2FileUrl(const AzureBlobStorage::Endpoint & endpoint, const String & blob_path_)
+{
+    Poco::URI uri(endpoint.getContainerEndpoint());
+    String path = uri.getPath();
+    if (!path.empty() && path.back() != '/')
+        path += '/';
+    path += prefixedBlobPath(endpoint, blob_path_);
+    uri.setPath(path);
+    return uri.toString();
+}
+
+}
+
+constexpr size_t ADLFS_MAX_RETRIES = 10;
+
+bool isAdlsGen2Endpoint(const AzureBlobStorage::Endpoint & endpoint)
+{
+    return endpoint.storage_account_url.contains("dfs.fabric.microsoft.com")
+        || endpoint.storage_account_url.contains("blob.fabric.microsoft.com");
+}
+
+Azure::Storage::Files::DataLake::DataLakeFileClient makeAdlsGen2FileClient(
+    const AzureBlobStorage::Endpoint & endpoint,
+    const AzureBlobStorage::AuthMethod & auth_method,
+    const Azure::Storage::Blobs::BlobClientOptions & blob_client_options,
+    const String & blob_path)
+{
+    using namespace Azure::Storage::Files::DataLake;
+    auto datalake_options = toDataLakeOptions(blob_client_options);
+
+    if (!endpoint.sas_auth.empty() || !endpoint.additional_params.empty())
+        return DataLakeFileClient(buildAdlsGen2FileUrl(endpoint, blob_path), datalake_options);
+
+    return std::visit(
+        [&]<typename T>(const T & auth) -> DataLakeFileClient
+        {
+            if constexpr (std::is_same_v<T, AzureBlobStorage::ConnectionString>)
+            {
+                return DataLakeFileClient::CreateFromConnectionString(
+                    auth.toUnderType(),
+                    endpoint.container_name,
+                    prefixedBlobPath(endpoint, blob_path),
+                    datalake_options);
+            }
+            else
+            {
+                return DataLakeFileClient(buildAdlsGen2FileUrl(endpoint, blob_path), auth, datalake_options);
+            }
+        },
+        auth_method);
+}
+
+WriteBufferFromAzureDataLakeStorage::WriteBufferFromAzureDataLakeStorage(
+    const AzureBlobStorage::Endpoint & endpoint_,
+    const AzureBlobStorage::AuthMethod & auth_method_,
+    const Azure::Storage::Blobs::BlobClientOptions & blob_client_options_,
+    const String & blob_path_,
+    size_t buf_size_,
+    const WriteSettings & write_settings_,
+    std::shared_ptr<const AzureBlobStorage::RequestSettings> settings_,
+    const String & container_for_logging_,
+    BlobStorageLogWriterPtr blob_log_)
+    : WriteBufferFromFileBase(buf_size_, nullptr, 0)
+    , log(getLogger("WriteBufferFromAzureDataLakeStorage"))
+    , file_client(makeAdlsGen2FileClient(endpoint_, auth_method_, blob_client_options_, blob_path_))
+    , blob_path(blob_path_)
+    , write_settings(write_settings_)
+    , max_unexpected_write_error_retries(settings_->max_unexpected_write_error_retries)
+    , container_for_logging(container_for_logging_)
+    , blob_log(std::move(blob_log_))
+{
+}
+
+WriteBufferFromAzureDataLakeStorage::~WriteBufferFromAzureDataLakeStorage()
+{
+    if (canceled)
+    {
+        LOG_INFO(log, "WriteBufferFromAzureDataLakeStorage was canceled. File `{}` may be left in an incomplete state.", blob_path);
+    }
+    else if (!finalized)
+    {
+        LOG_INFO(log, "WriteBufferFromAzureDataLakeStorage is not finalized in destructor. File `{}` may not be written to ADLS Gen2.", blob_path);
+    }
+}
+
+void WriteBufferFromAzureDataLakeStorage::runWithRetries(
+    const std::function<void()> & op,
+    const char * what,
+    BlobStorageLogElement::EventType event_type,
+    size_t data_size)
+{
+    auto log_event = [&](Int32 error_code, const String & error_message, size_t elapsed_us)
+    {
+        if (blob_log)
+            blob_log->addEvent(
+                event_type,
+                /* bucket */ container_for_logging,
+                /* remote_path */ blob_path,
+                /* local_path */ {},
+                /* data_size */ data_size,
+                /* elapsed_microseconds */ elapsed_us,
+                error_code,
+                error_message);
+    };
+
+    Stopwatch watch;
+    size_t backoff_ms = 100;
+    for (size_t attempt = 1; attempt < ADLFS_MAX_RETRIES; ++attempt)
+    {
+        LOG_TRACE(log, "ADLS Gen2 {} attempt {} for `{}`", what, attempt, blob_path);
+        try
+        {
+            op();
+            log_event(/*error_code=*/ 0, /*error_message=*/ {}, watch.elapsedMicroseconds());
+            LOG_TRACE(log, "ADLS Gen2 {} attempt {} for `{}` succeeded", what, attempt, blob_path);
+            return;
+        }
+        catch (const Azure::Core::RequestFailedException & e)
+        {
+            const bool retryable = isRetryableAzureException(e, write_settings.is_initial_access_check);
+            if (!retryable || attempt >= max_unexpected_write_error_retries)
+            {
+                log_event(static_cast<Int32>(e.StatusCode), e.Message, watch.elapsedMicroseconds());
+                throw Exception(
+                    ErrorCodes::AZURE_BLOB_STORAGE_ERROR,
+                    "ADLS Gen2 {} failed for `{}`: HTTP {}: {}",
+                    what,
+                    blob_path,
+                    static_cast<int>(e.StatusCode),
+                    e.Message);
+            }
+
+            LOG_WARNING(log, "ADLS Gen2 {} attempt {} for `{}` failed: HTTP {}: {}. Retrying after {} ms.",
+                what, attempt, blob_path, static_cast<int>(e.StatusCode), e.Message, backoff_ms);
+
+            sleepForMilliseconds(backoff_ms);
+            backoff_ms *= 2;
+        }
+    }
+    log_event(static_cast<Int32>(ErrorCodes::AZURE_BLOB_STORAGE_ERROR), "retries exhausted", watch.elapsedMicroseconds());
+    throw Exception(
+        ErrorCodes::AZURE_BLOB_STORAGE_ERROR,
+        "ADLS Gen2 {} failed for `{}`",
+        what,
+        blob_path);
+}
+
+void WriteBufferFromAzureDataLakeStorage::ensureCreated()
+{
+    if (file_created)
+        return;
+
+    Azure::Storage::Files::DataLake::CreateFileOptions create_options;
+    if (!write_settings.object_storage_write_if_none_match.empty())
+        create_options.AccessConditions.IfNoneMatch = Azure::ETag(write_settings.object_storage_write_if_none_match);
+    if (!write_settings.object_storage_write_if_match.empty())
+        create_options.AccessConditions.IfMatch = Azure::ETag(write_settings.object_storage_write_if_match);
+
+    LOG_TRACE(log, "Entering Create for ADLS Gen2 file `{}`", blob_path);
+    runWithRetries(
+        [&]() { file_client.Create(create_options); },
+        "Create",
+        BlobStorageLogElement::EventType::MultiPartUploadCreate,
+        /*data_size=*/ 0);
+    file_created = true;
+    LOG_DEBUG(log, "Created ADLS Gen2 file `{}`", blob_path);
+}
+
+void WriteBufferFromAzureDataLakeStorage::appendBufferedData()
+{
+    const size_t to_append = offset();
+    if (to_append == 0)
+        return;
+
+    ensureCreated();
+
+    const auto * data_ptr = reinterpret_cast<const uint8_t *>(working_buffer.begin());
+    const int64_t offset_for_append = bytes_appended;
+
+    ProfileEvents::increment(ProfileEvents::AzureUpload);
+
+    LOG_TRACE(log, "Entering Append for `{}`: offset={}, len={}", blob_path, offset_for_append, to_append);
+    runWithRetries(
+        [&]()
+        {
+            Azure::Core::IO::MemoryBodyStream stream(data_ptr, to_append);
+            file_client.Append(stream, offset_for_append);
+        },
+        "Append",
+        BlobStorageLogElement::EventType::MultiPartUploadWrite,
+        to_append);
+
+    bytes_appended += static_cast<int64_t>(to_append);
+    LOG_TRACE(log, "Appended for `{}`: bytes_appended={}", blob_path, bytes_appended);
+}
+
+void WriteBufferFromAzureDataLakeStorage::nextImpl()
+{
+    if (is_prefinalized)
+        throw Exception(
+            ErrorCodes::AZURE_BLOB_STORAGE_ERROR,
+            "Cannot write to prefinalized buffer for ADLS Gen2, the file `{}` has already been flushed",
+            blob_path);
+
+    appendBufferedData();
+}
+
+void WriteBufferFromAzureDataLakeStorage::preFinalize()
+{
+    if (is_prefinalized)
+        return;
+
+    LOG_DEBUG(log, "Entering preFinalize for ADLS Gen2 file `{}`", blob_path);
+    next();
+
+    is_prefinalized = true;
+    WriteBuffer::set(fake_buffer_when_prefinalized, sizeof(fake_buffer_when_prefinalized));
+    ensureCreated();
+    runWithRetries(
+        [&]() { file_client.Flush(bytes_appended); },
+        "Flush",
+        BlobStorageLogElement::EventType::MultiPartUploadComplete,
+        /*data_size=*/ 0);
+    LOG_DEBUG(log, "Flushed ADLS Gen2 file `{}` ({} bytes)", blob_path, bytes_appended);
+}
+
+void WriteBufferFromAzureDataLakeStorage::finalizeImpl()
+{
+    if (!is_prefinalized)
+        preFinalize();
+}
+
+void WriteBufferFromAzureDataLakeStorage::cancelImpl() noexcept
+{
+    WriteBufferFromFileBase::cancelImpl();
+
+    if (file_created && !finalized)
+    {
+        try
+        {
+            LOG_INFO(log, "Deleting incomplete ADLS Gen2 file `{}` after cancel", blob_path);
+            file_client.DeleteIfExists();
+        }
+        catch (...)
+        {
+            tryLogCurrentException(log, fmt::format("Failed to delete incomplete ADLS Gen2 file `{}`", blob_path));
+        }
+    }
+}
+
+}
+
+#endif

src/Disks/IO/WriteBufferFromAzureDataLakeStorage.h

@@ -0,0 +1,80 @@
+#pragma once
+
+#include "config.h"
+
+#if USE_AZURE_BLOB_STORAGE
+
+#include <memory>
+
+#include <IO/WriteBufferFromFileBase.h>
+#include <IO/WriteBuffer.h>
+#include <IO/WriteSettings.h>
+#include <Common/BlobStorageLogWriter.h>
+#include <Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
+#include <Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h>
+
+#include <azure/storage/files/datalake/datalake_file_client.hpp>
+
+namespace DB
+{
+
+class WriteBufferFromAzureDataLakeStorage : public WriteBufferFromFileBase
+{
+public:
+    WriteBufferFromAzureDataLakeStorage(
+        const AzureBlobStorage::Endpoint & endpoint_,
+        const AzureBlobStorage::AuthMethod & auth_method_,
+        const Azure::Storage::Blobs::BlobClientOptions & blob_client_options_,
+        const String & blob_path_,
+        size_t buf_size_,
+        const WriteSettings & write_settings_,
+        std::shared_ptr<const AzureBlobStorage::RequestSettings> settings_,
+        const String & container_for_logging_ = {},
+        BlobStorageLogWriterPtr blob_log_ = {});
+
+    ~WriteBufferFromAzureDataLakeStorage() override;
+
+    void nextImpl() override;
+    void finalizeImpl() override;
+    void preFinalize() override;
+    void cancelImpl() noexcept override;
+    std::string getFileName() const override { return blob_path; }
+    void sync() override { next(); }
+
+private:
+    void ensureCreated();
+    void appendBufferedData();
+    void runWithRetries(
+        const std::function<void()> & op,
+        const char * what,
+        BlobStorageLogElement::EventType event_type,
+        size_t data_size);
+
+    LoggerPtr log;
+
+    Azure::Storage::Files::DataLake::DataLakeFileClient file_client;
+    const std::string blob_path;
+    const WriteSettings write_settings;
+    const size_t max_unexpected_write_error_retries;
+
+    bool file_created = false;
+    bool is_prefinalized = false;
+    int64_t bytes_appended = 0;
+
+    char fake_buffer_when_prefinalized[1] = {};
+
+    String container_for_logging;
+    BlobStorageLogWriterPtr blob_log;
+};
+
+Azure::Storage::Files::DataLake::DataLakeFileClient makeAdlsGen2FileClient(
+    const AzureBlobStorage::Endpoint & endpoint,
+    const AzureBlobStorage::AuthMethod & auth_method,
+    const Azure::Storage::Blobs::BlobClientOptions & blob_client_options,
+    const String & blob_path);
+
+bool isAdlsGen2Endpoint(const AzureBlobStorage::Endpoint & endpoint);
+
+}
+
+#endif

src/Formats/JSONExtractTree.cpp

@@ -721,7 +721,17 @@ public:
                 return false;
             }
 
-            value = element.isInt64() ? element.getInt64() : element.getUInt64();
+            if (element.isInt64())
+            {
+                value = element.getInt64();
+            }
+            else
+            {
+                /// Clamp in the unsigned domain before narrowing to time_t,
+                /// because values above INT64_MAX would wrap to negative on cast.
+                UInt64 raw = element.getUInt64();
+                value = static_cast<time_t>(std::min(raw, UInt64(0xFFFFFFFF)));
+            }
         }
         else
         {
@@ -729,7 +739,7 @@ public:
             return false;
         }
 
-        assert_cast<ColumnDateTime &>(column).insert(value);
+        assert_cast<ColumnDateTime &>(column).insert(std::clamp<time_t>(value, 0, static_cast<time_t>(0xFFFFFFFF)));
         return true;
     }
 
@@ -2264,6 +2274,7 @@ private:
                     }
                 }
 
+                if (format_settings.try_infer_datetimes)
                 {
                     DateTime64 value;
                     if (tryInferDateTime64FromString(data, value, format_settings, time_zone_for_schema_inference, utc_time_zone_for_schema_inference))

src/Functions/DateTimeTransforms.h

@@ -75,6 +75,28 @@ struct ZeroTransform
     static UInt16 execute(UInt16, const DateLUTImpl &) { return 0; }
 };
 
+/// Returns the factor transform value used by monotonicity analysis (see getMonotonicityForRange).
+///
+/// `Transform::execute` clamps an out-of-range result to a representable Date. For example,
+/// rounding a pre-epoch Date32/DateTime64 down with toMonday/toStartOfMonth/toStartOfYear gives a
+/// day number before 1970-01-01, which is clamped to 1970-01-01 (day 0); a day number after
+/// 2149-06-06 is clamped to 2149-06-06 (day 0xFFFF). That clamp is correct for the value the
+/// function returns, but it is wrong for monotonicity: it collapses every out-of-range
+/// week/month/year to the same factor, so a function that actually wraps within those periods
+/// (toDayOfWeek, toMonth, toDayOfMonth, ...) would be reported as monotonic and corrupt
+/// index/partition pruning.
+///
+/// `executeExtendedResult` keeps the unclamped, order-preserving extended day number, so prefer it
+/// when the factor transform provides it; otherwise fall back to the plain `execute`.
+template <typename FactorTransform, typename ArgType>
+auto extendedFactorForMonotonicity(ArgType arg, const DateLUTImpl & date_lut)
+{
+    if constexpr (requires { FactorTransform::executeExtendedResult(arg, date_lut); })
+        return FactorTransform::executeExtendedResult(arg, date_lut);
+    else
+        return FactorTransform::execute(arg, date_lut);
+}
+
 template <FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior = default_date_time_overflow_behavior>
 struct ToDateImpl
 {
@@ -215,17 +237,17 @@ struct ToMondayImpl
 
     static UInt16 execute(Int64 t, const DateLUTImpl & time_zone)
     {
-        //return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t));
-        return static_cast<UInt16>(time_zone.toFirstDayNumOfWeek(t));
+        const int res = time_zone.toFirstDayNumOfWeek(t);
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
     {
-        //return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t));
         return time_zone.toFirstDayNumOfWeek(t);
     }
     static UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
     {
-        return static_cast<UInt16>(time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d)));
+        const int res = time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d));
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
@@ -248,7 +270,8 @@ struct ToStartOfMonthImpl
 
     static UInt16 execute(Int64 t, const DateLUTImpl & time_zone)
     {
-        return static_cast<UInt16>(time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t)));
+        const int res = time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t));
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
     {
@@ -256,7 +279,8 @@ struct ToStartOfMonthImpl
     }
     static UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
     {
-        return static_cast<UInt16>(time_zone.toFirstDayNumOfMonth(ExtendedDayNum(d)));
+        const int res = time_zone.toFirstDayNumOfMonth(ExtendedDayNum(d));
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
@@ -280,7 +304,8 @@ struct ToLastDayOfMonthImpl
 
     static UInt16 execute(Int64 t, const DateLUTImpl & time_zone)
     {
-        return static_cast<UInt16>(time_zone.toLastDayNumOfMonth(time_zone.toDayNum(t)));
+        const int res = time_zone.toLastDayNumOfMonth(time_zone.toDayNum(t));
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
     {
@@ -288,7 +313,8 @@ struct ToLastDayOfMonthImpl
     }
     static UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
     {
-        return static_cast<UInt16>(time_zone.toLastDayNumOfMonth(ExtendedDayNum(d)));
+        const int res = time_zone.toLastDayNumOfMonth(ExtendedDayNum(d));
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
@@ -311,7 +337,8 @@ struct ToStartOfQuarterImpl
 
     static UInt16 execute(Int64 t, const DateLUTImpl & time_zone)
     {
-        return static_cast<UInt16>(time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t)));
+        const int res = time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t));
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
     {
@@ -319,7 +346,8 @@ struct ToStartOfQuarterImpl
     }
     static UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
     {
-        return static_cast<UInt16>(time_zone.toFirstDayNumOfQuarter(ExtendedDayNum(d)));
+        const int res = time_zone.toFirstDayNumOfQuarter(ExtendedDayNum(d));
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
@@ -342,7 +370,8 @@ struct ToStartOfYearImpl
 
     static UInt16 execute(Int64 t, const DateLUTImpl & time_zone)
     {
-        return static_cast<UInt16>(time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t)));
+        const int res = time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t));
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt32 t, const DateLUTImpl & time_zone)
     {
@@ -350,7 +379,8 @@ struct ToStartOfYearImpl
     }
     static UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
     {
-        return static_cast<UInt16>(time_zone.toFirstDayNumOfYear(ExtendedDayNum(d)));
+        const int res = time_zone.toFirstDayNumOfYear(ExtendedDayNum(d));
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
     {
@@ -408,16 +438,17 @@ struct ToStartOfWeekImpl
     static UInt16 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
         const int res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
-        return static_cast<UInt16>(std::max(res, 0));
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
         const int res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
-        return static_cast<UInt16>(std::max(res, 0));
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
-        return static_cast<UInt16>(time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode));
+        const int res = time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode);
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
@@ -443,7 +474,8 @@ struct ToLastDayOfWeekImpl
 
     static UInt16 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
-        return static_cast<UInt16>(time_zone.toLastDayNumOfWeek(time_zone.toDayNum(t), week_mode));
+        const int res = time_zone.toLastDayNumOfWeek(time_zone.toDayNum(t), week_mode);
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
@@ -451,7 +483,8 @@ struct ToLastDayOfWeekImpl
     }
     static UInt16 execute(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
-        return static_cast<UInt16>(time_zone.toLastDayNumOfWeek(ExtendedDayNum(d), week_mode));
+        const int res = time_zone.toLastDayNumOfWeek(ExtendedDayNum(d), week_mode);
+        return static_cast<UInt16>(std::clamp(res, 0, DATE_LUT_MAX_DAY_NUM));
     }
     static UInt16 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
@@ -1882,6 +1915,76 @@ struct ToMillisecondImpl
     using FactorTransform = ZeroTransform;
 };
 
+struct ToMicrosecondImpl
+{
+    static constexpr auto name = "toMicrosecond";
+
+    static UInt32 execute(const DateTime64 & datetime64, Int64 scale_multiplier, const DateLUTImpl & time_zone)
+    {
+        return static_cast<UInt32>(time_zone.toMicrosecond(datetime64, scale_multiplier));
+    }
+
+    static UInt32 execute(const Time64 & time64, Int64 scale_multiplier, const DateLUTImpl & time_zone)
+    {
+        return static_cast<UInt32>(time_zone.toMicrosecond(time64, scale_multiplier));
+    }
+
+    static UInt32 execute(UInt32, const DateLUTImpl &)
+    {
+        return 0;
+    }
+    static UInt32 execute(Int32, const DateLUTImpl &)
+    {
+        throwDate32IsNotSupported(name);
+    }
+    static UInt32 execute(Int64, const DateLUTImpl &)
+    {
+        throwTimeIsNotSupported(name);
+    }
+    static UInt32 execute(UInt16, const DateLUTImpl &)
+    {
+        throwDateIsNotSupported(name);
+    }
+    static constexpr bool hasPreimage() { return false; }
+
+    using FactorTransform = ZeroTransform;
+};
+
+struct ToNanosecondImpl
+{
+    static constexpr auto name = "toNanosecond";
+
+    static UInt32 execute(const DateTime64 & datetime64, Int64 scale_multiplier, const DateLUTImpl & time_zone)
+    {
+        return static_cast<UInt32>(time_zone.toNanosecond(datetime64, scale_multiplier));
+    }
+
+    static UInt32 execute(const Time64 & time64, Int64 scale_multiplier, const DateLUTImpl & time_zone)
+    {
+        return static_cast<UInt32>(time_zone.toNanosecond(time64, scale_multiplier));
+    }
+
+    static UInt32 execute(UInt32, const DateLUTImpl &)
+    {
+        return 0;
+    }
+    static UInt32 execute(Int32, const DateLUTImpl &)
+    {
+        throwDate32IsNotSupported(name);
+    }
+    static UInt32 execute(Int64, const DateLUTImpl &)
+    {
+        throwTimeIsNotSupported(name);
+    }
+    static UInt32 execute(UInt16, const DateLUTImpl &)
+    {
+        throwDateIsNotSupported(name);
+    }
+    static constexpr bool hasPreimage() { return false; }
+
+    using FactorTransform = ZeroTransform;
+};
+
 struct ToISOYearImpl
 {
     static constexpr auto name = "toISOYear";

src/Functions/FunctionDateOrDateTimeToSomething.h

@@ -1,5 +1,8 @@
 #pragma once
+#include <DataTypes/DataTypeInterval.h>
+#include <DataTypes/DataTypesNumber.h>
 #include <Functions/IFunctionDateOrDateTime.h>
+#include <Interpreters/castColumn.h>
 
 namespace DB
 {
@@ -22,6 +25,34 @@ public:
         constexpr bool result_is_date_or_date32 = (std::is_same_v<ToDataType, DataTypeDate> || std::is_same_v<ToDataType, DataTypeDate32>);
         this->checkArguments(arguments, result_is_date_or_date32);
 
+        /// For Interval operands, validate the kind match here (at type
+        /// analysis) rather than only in `executeImpl`, so an invalid
+        /// expression like `toHour(<IntervalDay>)` is rejected before it can
+        /// be captured in a view's column type. Return the raw stored value
+        /// as `Int64`: the function's natural return type (e.g. `UInt8` for
+        /// `toDayOfMonth`) is sized for a calendar-field range (1..31), not
+        /// for arbitrary interval magnitudes or negative values, so widening
+        /// to `Int64` avoids silent overflow/wrap.
+        if (isInterval(arguments[0].type))
+        {
+            IntervalKind::Kind required = IntervalKind::Kind::Second;
+            if (!IntervalKind::tryParseFromNameOfFunctionExtractTimePart(this->getName(), required))
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Function {} does not support Interval arguments",
+                    this->getName());
+
+            const auto & interval_type = static_cast<const DataTypeInterval &>(*arguments[0].type);
+            if (interval_type.getKind() != required)
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Cannot extract {} from {}: the interval's unit does not match the requested unit",
+                    IntervalKind(required).toLowercasedKeyword(),
+                    interval_type.getName());
+
+            return std::make_shared<DataTypeInt64>();
+        }
+
         /// For DateTime results, if time zone is specified, attach it to type.
         /// If the time zone is specified but empty, throw an exception.
         if constexpr (std::is_same_v<ToDataType, DataTypeDateTime>)
@@ -75,15 +106,26 @@ public:
     DataTypePtr getReturnTypeForDefaultImplementationForDynamic() const override
     {
         /// If result type is DateTime or DateTime64 we don't know the timezone and scale without argument types.
-        if constexpr (!std::is_same_v<ToDataType, DataTypeDateTime> && !std::is_same_v<ToDataType, DataTypeTime> && !std::is_same_v<ToDataType, DataTypeDateTime64> && !std::is_same_v<ToDataType, DataTypeTime64>)
+        /// Keep this branch as the `if constexpr` so `std::make_shared<ToDataType>()` is not instantiated
+        /// for types without a default constructor (e.g. DataTypeDateTime64).
+        if constexpr (std::is_same_v<ToDataType, DataTypeDateTime> || std::is_same_v<ToDataType, DataTypeTime> || std::is_same_v<ToDataType, DataTypeDateTime64> || std::is_same_v<ToDataType, DataTypeTime64>)
+            return nullptr;
+        else
+        {
+            /// Extract-capable functions widen to Int64 for Interval; the narrow `ToDataType`
+            /// would wrap an out-of-range interval. Int64 fits every supported result.
+            if (this->acceptsIntervalArgument())
+                return std::make_shared<DataTypeInt64>();
             return std::make_shared<ToDataType>();
-        return nullptr;
+        }
     }
 
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         const IDataType * from_type = arguments[0].type.get();
 
+        if (isInterval(from_type))
+            return executeOnInterval(arguments, result_type, input_rows_count);
         if (isDate(from_type))
             return DateTimeTransformImpl<DataTypeDate, ToDataType, Transform>::execute(arguments, result_type, input_rows_count);
         if (isDate32(from_type))
@@ -113,6 +155,16 @@ public:
             this->getName());
     }
 
+private:
+    /// PostgreSQL-style `EXTRACT(<unit> FROM INTERVAL ...)`: kind matching is
+    /// validated in `getReturnTypeImpl`, so here we just return the underlying
+    /// Int64 values cast to the function's result type.
+    ColumnPtr executeOnInterval(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const
+    {
+        return castColumn(arguments[0], result_type);
+    }
+
+public:
     bool hasInformationAboutPreimage() const override { return Transform::hasPreimage(); }
 
     FieldIntervalPtr getPreimage(const IDataType & type, const Field & point) const override

src/Functions/FunctionsJSON.cpp

@@ -28,6 +28,7 @@
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypeTuple.h>
 #include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/DataTypeDynamic.h>
 #include <DataTypes/DataTypeObject.h>
 
 #include <Functions/IFunction.h>
@@ -259,16 +260,24 @@ public:
             auto merged_type = data_type_object.getSubcolumnType(combined_name);
             auto merged = data_type_object.getSubcolumn(combined_name, object_column);
 
+            /// Typed paths are always present in a JSON column, even when the key was missing
+            /// from the inserted JSON (they get the type's default value). For non-typed paths
+            /// the combined subcolumn returns a Dynamic column where NULL means absent.
+            bool is_typed_path = data_type_object.getTypedPaths().contains(path);
+
             /// JSONHas must be UInt8 {0,1} from path presence. The generic `else` below would
             /// cast the extracted value to UInt8 and silently return the value itself.
             constexpr bool is_has = std::string_view(TName::name) == std::string_view("JSONHas");
 
             if constexpr (is_has)
             {
+                if (is_typed_path)
+                    return DataTypeUInt8().createColumnConst(input_rows_count, 1u)->convertToFullColumnIfConst();
+
                 auto result = ColumnVector<UInt8>::create(input_rows_count);
                 auto & data = result->getData();
                 for (size_t i = 0; i < input_rows_count; ++i)
-                    data[i] = merged->isDefaultAt(i) ? 0 : 1;
+                    data[i] = merged->isNullAt(i) ? 0 : 1;
                 return result;
             }
 
@@ -293,7 +302,7 @@ public:
                 auto serialization = merged_type->getDefaultSerialization();
                 for (size_t i = 0; i < input_rows_count; ++i)
                 {
-                    if (merged->isDefaultAt(i))
+                    if (!is_typed_path && merged->isNullAt(i))
                     {
                         raw_col->insertDefault();
                     }