mirror of
https://github.com/hanzoai/dev.git
synced 2026-10-25 03:32:32 +00:00
Add native search: the hanzo-search engine and the grep tool
hanzo-search is Dev's one search path. It answers text (regex), file names (glob) and symbols (word-boundary identifiers, definitions first, with a Resolver hook for semantic confirmation) over tgrep-core's trigram index, linked in process and pinned by commit. Per path it asks a running tgrep server, else the on-disk index, else scans with tgrep's walker. Paths Dev names, and files changed after a tool that may write began, are read from disk instead of a lagging index. hanzo-grep gives the agent the engine as one native tool, grep, installed in the app server's extensions by an anchored edit. It marks every other tool start so the next search sees what the tool wrote. The base instructions send the model to grep, and to rg only on error. tgrep-core's rusqlite git pin would link a second SQLite beside sqlx's; the root manifest patches it to rusqlite 0.39, which shares 0.37.
This commit is contained in:
Generated
+244
-1
@@ -165,6 +165,12 @@ version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
||||
|
||||
[[package]]
|
||||
name = "ambient-authority"
|
||||
version = "0.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b"
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
@@ -1382,6 +1388,50 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0"
|
||||
|
||||
[[package]]
|
||||
name = "cap-primitives"
|
||||
version = "4.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18"
|
||||
dependencies = [
|
||||
"ambient-authority",
|
||||
"fs-set-times",
|
||||
"io-extras",
|
||||
"io-lifetimes 3.0.1",
|
||||
"ipnet",
|
||||
"maybe-owned",
|
||||
"rustix",
|
||||
"rustix-linux-procfs",
|
||||
"windows-sys 0.60.2",
|
||||
"winx",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cap-std"
|
||||
version = "4.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1ec78e242cfa2cfe276807ac2ecc00315a6c97786977414bcd1c3963b6c91b8"
|
||||
dependencies = [
|
||||
"cap-primitives",
|
||||
"io-extras",
|
||||
"io-lifetimes 3.0.1",
|
||||
"rustix",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cap-tempfile"
|
||||
version = "4.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "038b3f6a5d4b5641b1bb99d638eb7a22013de84f81c528d8930de06d7383fbd5"
|
||||
dependencies = [
|
||||
"cap-std",
|
||||
"rand 0.9.3",
|
||||
"rustix",
|
||||
"rustix-linux-procfs",
|
||||
"uuid",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "caseless"
|
||||
version = "0.2.2"
|
||||
@@ -1865,6 +1915,7 @@ dependencies = [
|
||||
"codex-websocket-auth",
|
||||
"codex-windows-sandbox",
|
||||
"futures",
|
||||
"hanzo-grep",
|
||||
"hanzo-kai",
|
||||
"hanzo-version",
|
||||
"libc",
|
||||
@@ -2890,7 +2941,7 @@ dependencies = [
|
||||
"clap",
|
||||
"crossbeam-channel",
|
||||
"hanzo-version",
|
||||
"ignore",
|
||||
"ignore 0.4.25 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"nucleo",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -5850,6 +5901,18 @@ dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||
|
||||
[[package]]
|
||||
name = "fancy-regex"
|
||||
version = "0.11.0"
|
||||
@@ -6156,6 +6219,27 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs-set-times"
|
||||
version = "0.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a"
|
||||
dependencies = [
|
||||
"io-lifetimes 2.0.4",
|
||||
"rustix",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs2"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs_extra"
|
||||
version = "1.3.0"
|
||||
@@ -7436,6 +7520,21 @@ dependencies = [
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hanzo-grep"
|
||||
version = "0.7.0-dev.1"
|
||||
dependencies = [
|
||||
"codex-exec-server",
|
||||
"codex-extension-api",
|
||||
"codex-protocol",
|
||||
"hanzo-search",
|
||||
"pretty_assertions",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hanzo-kai"
|
||||
version = "0.7.0-dev.1"
|
||||
@@ -7467,6 +7566,22 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hanzo-search"
|
||||
version = "0.7.0-dev.1"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"globset",
|
||||
"memchr",
|
||||
"pretty_assertions",
|
||||
"rayon",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"tgrep-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hanzo-tui"
|
||||
version = "0.7.0-dev.1"
|
||||
@@ -8200,6 +8315,21 @@ dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ignore"
|
||||
version = "0.4.25"
|
||||
source = "git+https://github.com/microsoft/tgrep?rev=c7b2ab827274f8a56ff95eadc005f69e0732ee92#c7b2ab827274f8a56ff95eadc005f69e0732ee92"
|
||||
dependencies = [
|
||||
"crossbeam-deque",
|
||||
"globset",
|
||||
"log",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"same-file",
|
||||
"walkdir",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "image"
|
||||
version = "0.25.9"
|
||||
@@ -8398,6 +8528,28 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "io-extras"
|
||||
version = "0.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f"
|
||||
dependencies = [
|
||||
"io-lifetimes 3.0.1",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "io-lifetimes"
|
||||
version = "2.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983"
|
||||
|
||||
[[package]]
|
||||
name = "io-lifetimes"
|
||||
version = "3.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96"
|
||||
|
||||
[[package]]
|
||||
name = "io_tee"
|
||||
version = "0.1.1"
|
||||
@@ -8971,6 +9123,12 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "maybe-owned"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4"
|
||||
|
||||
[[package]]
|
||||
name = "md-5"
|
||||
version = "0.11.0"
|
||||
@@ -11593,6 +11751,31 @@ dependencies = [
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rsqlite-vfs"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c"
|
||||
dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.39.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"fallible-iterator",
|
||||
"fallible-streaming-iterator",
|
||||
"hashlink",
|
||||
"libsqlite3-sys",
|
||||
"smallvec",
|
||||
"sqlite-wasm-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed"
|
||||
version = "8.11.0"
|
||||
@@ -11680,6 +11863,16 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix-linux-procfs"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"rustix",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.45"
|
||||
@@ -12537,6 +12730,18 @@ dependencies = [
|
||||
"der",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlite-wasm-rs"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"js-sys",
|
||||
"rsqlite-vfs",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlx"
|
||||
version = "0.9.0"
|
||||
@@ -13353,6 +13558,34 @@ dependencies = [
|
||||
"unicode-width 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tgrep-core"
|
||||
version = "1.2.1"
|
||||
source = "git+https://github.com/microsoft/tgrep?rev=c7b2ab827274f8a56ff95eadc005f69e0732ee92#c7b2ab827274f8a56ff95eadc005f69e0732ee92"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
"cap-std",
|
||||
"cap-tempfile",
|
||||
"encoding_rs",
|
||||
"fs2",
|
||||
"getrandom 0.4.2",
|
||||
"globset",
|
||||
"ignore 0.4.25 (git+https://github.com/microsoft/tgrep?rev=c7b2ab827274f8a56ff95eadc005f69e0732ee92)",
|
||||
"libc",
|
||||
"memchr",
|
||||
"memmap2",
|
||||
"rayon",
|
||||
"regex",
|
||||
"regex-syntax",
|
||||
"rusqlite",
|
||||
"same-file",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.69"
|
||||
@@ -15288,6 +15521,16 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ab703352da6a72f35c39a533526393725640575bb211f61987a2748323ad956"
|
||||
|
||||
[[package]]
|
||||
name = "winx"
|
||||
version = "0.36.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wiremock"
|
||||
version = "0.6.5"
|
||||
|
||||
@@ -5,6 +5,8 @@ members = [
|
||||
"crates/hanzo-tui",
|
||||
"crates/hanzo-kai",
|
||||
"crates/hanzo-loop",
|
||||
"crates/hanzo-grep",
|
||||
"crates/hanzo-search",
|
||||
"crates/hanzo-version",
|
||||
]
|
||||
exclude = ["upstream", "crates/hanzo-upstream"]
|
||||
@@ -501,3 +503,8 @@ tungstenite = { git = "https://github.com/openai-oss-forks/tungstenite-rs", rev
|
||||
|
||||
[patch."ssh://[email protected]/openai-oss-forks/tungstenite-rs.git"]
|
||||
tungstenite = { git = "https://github.com/openai-oss-forks/tungstenite-rs", rev = "4fffad30fe373adbdcffab9545e9e9bf4f2fc19f" }
|
||||
|
||||
# tgrep-core pins rusqlite by git commit, whose libsqlite3-sys 0.38 would link a second
|
||||
# SQLite beside the 0.37 sqlx bundles. rusqlite 0.39 is the release built on 0.37.
|
||||
[patch."https://github.com/rusqlite/rusqlite.git"]
|
||||
rusqlite = "=0.39.0"
|
||||
|
||||
@@ -129,6 +129,36 @@ engine with no catalog of its own is reviewed by the model it runs. Guardian v2'
|
||||
sync reviewer runs on that same choice; its `CodexResponsesHeaders` model only
|
||||
scopes an OpenAI-backend header. `crates/hanzo-dev/tests/review.rs` holds both.
|
||||
|
||||
## Search
|
||||
|
||||
One search path: `crates/hanzo-search` (`hanzo_search::Search::query`) answers text
|
||||
(regex), file names (glob) and symbols (word-boundary identifier, definitions first;
|
||||
textual, with a `Resolver` hook for the LSP or code graph to confirm). The agent reaches
|
||||
it as the native `grep` tool (`crates/hanzo-grep`), contributed through upstream's
|
||||
extension API and installed by one anchored edit in the app server's
|
||||
`thread_extensions`; the base instructions send the model to `grep`, and to `rg` in the
|
||||
shell only when `grep` errors. `hanzo-search` is also a CLI that prints what the tool
|
||||
returns (`-n` for `path:line:text`, `--files GLOB`, `--symbol`, `--json`, `--no-index`,
|
||||
`--index-only`, `--prepare PATH`, `--stats`; exit 0/1/2 as rg).
|
||||
|
||||
- Engine: tgrep (MIT), the single search engine, linked as `tgrep-core`: a git dependency
|
||||
pinned by exact commit in `crates/hanzo-search/Cargo.toml`, upstream v1.2.1
|
||||
(`c7b2ab827274f8a56ff95eadc005f69e0732ee92`) until the hanzoai/tgrep fork's rev replaces
|
||||
it. Its rusqlite git pin would link a second SQLite beside sqlx's, so the root
|
||||
`Cargo.toml` patches it to rusqlite 0.39, which shares libsqlite3-sys 0.37.
|
||||
- Per query path: the nearest `.tgrep/` index above it; a running tgrep server for that
|
||||
root (its `serve.json`, loopback JSON-RPC), else the on-disk index in process, else a
|
||||
scan with tgrep's walker (rg's ignore, hidden and binary rules). Nothing is spawned.
|
||||
- Freshness: paths Dev names (`Search::touched`), files changed after a tool that may
|
||||
write started (`Search::since`, called by `hanzo-grep` on every other tool and command
|
||||
start), and, for an on-disk index, files whose size or mtime differs from its record
|
||||
are read from disk instead of the index; a row whose file is gone is dropped. The walk
|
||||
that finds them is the cost of an index that lags.
|
||||
- Ranking: file score (text: matching lines; symbol: definitions x 2^32 + references;
|
||||
name: shallower first), then path, then line.
|
||||
- Licence boundary: tgrep-core is MIT and linked; its notice is in `NOTICE`. ripgrep-all
|
||||
(`rga`, AGPL-3.0) is never used: not linked, copied, called, shipped, or compared.
|
||||
|
||||
## House rules
|
||||
|
||||
- Hanzo service APIs use `https://api.hanzo.ai/v1/` exclusively.
|
||||
|
||||
@@ -53,7 +53,7 @@ v8:
|
||||
|
||||
## fmt: format the crates Hanzo owns
|
||||
fmt:
|
||||
@$(CARGO) fmt -p hanzo-dev -p hanzo-config -p hanzo-tui -p hanzo-kai -p hanzo-loop -p hanzo-version
|
||||
@$(CARGO) fmt -p hanzo-dev -p hanzo-config -p hanzo-tui -p hanzo-kai -p hanzo-loop -p hanzo-version -p hanzo-search -p hanzo-grep
|
||||
@$(CARGO) fmt --manifest-path crates/hanzo-upstream/Cargo.toml
|
||||
|
||||
## clean: discard build output
|
||||
|
||||
@@ -14,6 +14,30 @@ as required by Apache-2.0 section 4(d):
|
||||
Copyright (c) 2016-2022 Florian Dehau
|
||||
Copyright (c) 2023-2025 The Ratatui Developers
|
||||
|
||||
This product includes tgrep (https://github.com/microsoft/tgrep), linked as
|
||||
its tgrep-core library and the copy of the ignore crate it vendors, licensed
|
||||
under the MIT license:
|
||||
|
||||
Copyright (c) Microsoft Corporation.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE
|
||||
|
||||
--- Fork provenance ---
|
||||
|
||||
Hanzo Dev descends from OpenAI Codex by way of just-every/code
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
[package]
|
||||
name = "hanzo-grep"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
description = "The agent's grep tool: Dev's search engine as one native tool, kept current with Dev's writes."
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
hanzo-search = { path = "../hanzo-search", version = "0.7.0-dev.1" }
|
||||
codex-exec-server = { workspace = true }
|
||||
codex-extension-api = { workspace = true }
|
||||
codex-protocol = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
tokio = { workspace = true, features = ["rt"] }
|
||||
|
||||
[dev-dependencies]
|
||||
pretty_assertions = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
@@ -0,0 +1,457 @@
|
||||
//! The agent's `grep` tool: Dev's search engine ([`hanzo_search`]) as one native tool.
|
||||
//!
|
||||
//! The model gets a single function tool, `grep`, for file contents (regex), file
|
||||
//! names (glob) and an identifier's definitions and references, answering in
|
||||
//! `rg`'s `path:line:text` shape. The base instructions tell the model to use it
|
||||
//! and to fall back to `rg` in the shell only when it returns an error.
|
||||
//!
|
||||
//! # Where it plugs in
|
||||
//!
|
||||
//! [`install`] contributes the tool through upstream's extension API
|
||||
//! (`ToolContributor`) and observes tool starts (`ToolLifecycleContributor`). One
|
||||
//! anchored edit in `patches/upstream.json` calls it from the app server's
|
||||
//! `thread_extensions`, beside Kai, and one more per prompt rewrites its `rg`
|
||||
//! line. That is the whole integration: no tool handler in core, no MCP server
|
||||
//! (another process and protocol, and no view of Dev's writes), no wrapper around
|
||||
//! the shell's `rg`.
|
||||
//!
|
||||
//! # Freshness
|
||||
//!
|
||||
//! Every tool start other than `grep`'s, and every command, calls
|
||||
//! [`Search::since`]: the tool may write files it does not name, so files changed
|
||||
//! from then on bypass a lagging index. See [`hanzo_search`] for the rest.
|
||||
//!
|
||||
//! The tool answers only for the local environment, whose files this process
|
||||
//! reads; a remote environment gets an error that sends the model to its shell.
|
||||
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use codex_exec_server::LOCAL_ENVIRONMENT_ID;
|
||||
use codex_extension_api::CommandStartInput;
|
||||
use codex_extension_api::ExtensionData;
|
||||
use codex_extension_api::ExtensionRegistryBuilder;
|
||||
use codex_extension_api::FunctionCallError;
|
||||
use codex_extension_api::ResponsesApiTool;
|
||||
use codex_extension_api::ToolCall;
|
||||
use codex_extension_api::ToolContributor;
|
||||
use codex_extension_api::ToolExecutor;
|
||||
use codex_extension_api::ToolExecutorFuture;
|
||||
use codex_extension_api::ToolLifecycleContributor;
|
||||
use codex_extension_api::ToolLifecycleFuture;
|
||||
use codex_extension_api::ToolName;
|
||||
use codex_extension_api::ToolOutput;
|
||||
use codex_extension_api::ToolPayload;
|
||||
use codex_extension_api::ToolSpec;
|
||||
use codex_extension_api::ToolStartInput;
|
||||
use codex_extension_api::parse_tool_input_schema;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
use hanzo_search::Format;
|
||||
use hanzo_search::Kind;
|
||||
use hanzo_search::Query;
|
||||
use hanzo_search::Search;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
use serde_json::json;
|
||||
|
||||
/// The tool's model-visible name.
|
||||
pub const NAME: &str = "grep";
|
||||
|
||||
/// Hits returned when the model names no limit.
|
||||
const LIMIT: usize = 200;
|
||||
/// The most hits the model may ask for.
|
||||
const MOST: usize = 2_000;
|
||||
/// The most context lines the model may ask for.
|
||||
const CONTEXT: usize = 10;
|
||||
/// The tool's own output bound, before the host's.
|
||||
const BYTES: usize = 256 * 1024;
|
||||
|
||||
const DESCRIPTION: &str = "Search the workspace through Dev's index. \
|
||||
kind=text (default): lines matching a regex (or a literal with fixed=true), as path:line:text. \
|
||||
kind=name: files whose path matches a glob (\"*.rs\" matches by file name at any depth; \"src/**/*_test.py\" by path). \
|
||||
kind=symbol: lines that spell an identifier on word boundaries, definitions first; textual, so it can miss semantic callers. \
|
||||
Respects .gitignore and skips hidden and binary files. Always reflects your own edits. \
|
||||
Results are ranked: files with the most hits first, then path, then line. \
|
||||
Use this instead of rg, grep or find in the shell; use rg only if this tool returns an error.";
|
||||
|
||||
/// Registers the `grep` tool and the observer that keeps it current with Dev's writes.
|
||||
pub fn install<C: Sync>(builder: &mut ExtensionRegistryBuilder<C>) {
|
||||
let search = Arc::new(Search::new());
|
||||
builder.tool_contributor(Arc::new(Tools(Arc::clone(&search))));
|
||||
builder.tool_lifecycle_contributor(Arc::new(Writes(search)));
|
||||
}
|
||||
|
||||
struct Tools(Arc<Search>);
|
||||
|
||||
impl ToolContributor for Tools {
|
||||
fn tools(
|
||||
&self,
|
||||
_session_store: &ExtensionData,
|
||||
_thread_store: &ExtensionData,
|
||||
) -> Vec<Arc<dyn for<'call> ToolExecutor<ToolCall<'call>>>> {
|
||||
vec![Arc::new(Grep(Arc::clone(&self.0)))]
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks the start of every tool and command that may write files.
|
||||
struct Writes(Arc<Search>);
|
||||
|
||||
impl ToolLifecycleContributor for Writes {
|
||||
fn on_tool_start<'a>(&'a self, input: ToolStartInput<'a>) -> ToolLifecycleFuture<'a> {
|
||||
if !is_grep(input.tool_name) {
|
||||
self.0.since(SystemTime::now());
|
||||
}
|
||||
Box::pin(std::future::ready(()))
|
||||
}
|
||||
|
||||
fn on_command_start<'a>(&'a self, _input: CommandStartInput<'a>) -> ToolLifecycleFuture<'a> {
|
||||
self.0.since(SystemTime::now());
|
||||
Box::pin(std::future::ready(()))
|
||||
}
|
||||
}
|
||||
|
||||
fn is_grep(name: &ToolName) -> bool {
|
||||
name.is_default_namespace() && name.name == NAME
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
enum Output {
|
||||
#[default]
|
||||
Lines,
|
||||
Files,
|
||||
Count,
|
||||
}
|
||||
|
||||
/// The tool's arguments, as the model sends them.
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Args {
|
||||
pattern: String,
|
||||
#[serde(default)]
|
||||
kind: Kind,
|
||||
#[serde(default)]
|
||||
paths: Vec<String>,
|
||||
#[serde(default)]
|
||||
glob: Vec<String>,
|
||||
#[serde(default)]
|
||||
fixed: bool,
|
||||
#[serde(default)]
|
||||
ignore_case: bool,
|
||||
#[serde(default)]
|
||||
context: usize,
|
||||
#[serde(default)]
|
||||
output: Output,
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
fn schema() -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"pattern": {
|
||||
"type": "string",
|
||||
"description": "A regex for kind=text, a glob for kind=name, an identifier for kind=symbol."
|
||||
},
|
||||
"kind": {"type": "string", "enum": ["text", "name", "symbol"]},
|
||||
"paths": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Files or directories to search, relative to the working directory. Default: the working directory."
|
||||
},
|
||||
"glob": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "For kind=text and kind=symbol: only files matching these globs; a leading ! excludes."
|
||||
},
|
||||
"fixed": {"type": "boolean", "description": "The pattern is a literal string."},
|
||||
"ignore_case": {"type": "boolean"},
|
||||
"context": {"type": "integer", "description": "Lines of context around each hit, at most 10."},
|
||||
"output": {
|
||||
"type": "string",
|
||||
"enum": ["lines", "files", "count"],
|
||||
"description": "lines: path:line:text. files: each matching file once. count: path:N."
|
||||
},
|
||||
"limit": {"type": "integer", "description": "The most hits, best first. Default 200, at most 2000."}
|
||||
},
|
||||
"required": ["pattern"],
|
||||
"additionalProperties": false
|
||||
})
|
||||
}
|
||||
|
||||
struct Grep(Arc<Search>);
|
||||
|
||||
impl<'call> ToolExecutor<ToolCall<'call>> for Grep {
|
||||
fn tool_name(&self) -> ToolName {
|
||||
ToolName::plain(NAME)
|
||||
}
|
||||
|
||||
fn spec(&self) -> ToolSpec {
|
||||
let parameters = match parse_tool_input_schema(&schema()) {
|
||||
Ok(parameters) => parameters,
|
||||
Err(error) => panic!("the grep schema is a constant and must parse: {error}"),
|
||||
};
|
||||
ToolSpec::Function(ResponsesApiTool {
|
||||
name: NAME.to_string(),
|
||||
description: DESCRIPTION.to_string(),
|
||||
strict: false,
|
||||
defer_loading: None,
|
||||
parameters,
|
||||
output_schema: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn supports_parallel_tool_calls(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn handle<'a>(&'a self, call: ToolCall<'call>) -> ToolExecutorFuture<'a>
|
||||
where
|
||||
'call: 'a,
|
||||
{
|
||||
Box::pin(async move {
|
||||
let ToolPayload::Function { arguments } = &call.payload else {
|
||||
return Err(FunctionCallError::Fatal(format!(
|
||||
"{NAME} takes function arguments"
|
||||
)));
|
||||
};
|
||||
let args: Args = serde_json::from_str(arguments)
|
||||
.map_err(|error| FunctionCallError::RespondToModel(error.to_string()))?;
|
||||
let Some(environment) = call
|
||||
.environments
|
||||
.iter()
|
||||
.find(|environment| environment.environment_id == LOCAL_ENVIRONMENT_ID)
|
||||
else {
|
||||
return Err(FunctionCallError::RespondToModel(format!(
|
||||
"{NAME} searches the local workspace; this environment is remote, so use rg in its shell"
|
||||
)));
|
||||
};
|
||||
let cwd = environment.cwd.to_path_buf();
|
||||
let budget = call.response_byte_budget(BYTES);
|
||||
let search = Arc::clone(&self.0);
|
||||
let body = tokio::task::spawn_blocking(move || answer(&search, args, &cwd, budget))
|
||||
.await
|
||||
.map_err(|error| FunctionCallError::Fatal(error.to_string()))?
|
||||
.map_err(FunctionCallError::RespondToModel)?;
|
||||
Ok(Box::new(Text(body)) as Box<dyn ToolOutput>)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn query(args: Args, cwd: &Path) -> (Query, Format) {
|
||||
let format = match args.output {
|
||||
Output::Lines => Format::Lines,
|
||||
Output::Files => Format::Files,
|
||||
Output::Count => Format::Count,
|
||||
};
|
||||
let query = Query {
|
||||
kind: args.kind,
|
||||
pattern: args.pattern,
|
||||
root: cwd.to_path_buf(),
|
||||
paths: args.paths.into_iter().map(PathBuf::from).collect(),
|
||||
glob: args.glob,
|
||||
fixed: args.fixed,
|
||||
ignore_case: args.ignore_case,
|
||||
context: args.context.min(CONTEXT),
|
||||
limit: Some(args.limit.unwrap_or(LIMIT).min(MOST)),
|
||||
index: hanzo_search::Index::Auto,
|
||||
};
|
||||
(query, format)
|
||||
}
|
||||
|
||||
/// The tool's text: the rendered hits within `budget` bytes, or why it failed.
|
||||
fn answer(search: &Search, args: Args, cwd: &Path, budget: usize) -> Result<String, String> {
|
||||
let (query, format) = query(args, cwd);
|
||||
let hits = search
|
||||
.query(&query)
|
||||
.map_err(|error| format!("{NAME} failed: {error}. Use rg in the shell instead."))?;
|
||||
if hits.is_empty() {
|
||||
return Ok("no matches".to_string());
|
||||
}
|
||||
let text = hanzo_search::render(&hits, format);
|
||||
let limited = query.limit.is_some_and(|limit| hits.len() >= limit);
|
||||
if text.len() <= budget && !limited {
|
||||
return Ok(text);
|
||||
}
|
||||
let footer = format!(
|
||||
"[{} hits shown, best first; more may exist: narrow paths or glob, or raise limit]\n",
|
||||
hits.len()
|
||||
);
|
||||
if text.len() + footer.len() <= budget {
|
||||
return Ok(text + &footer);
|
||||
}
|
||||
let room = budget.saturating_sub(footer.len());
|
||||
let cut = text[..text.floor_char_boundary(room)]
|
||||
.rfind('\n')
|
||||
.map_or(0, |at| at + 1);
|
||||
let shown = text[..cut].lines().count();
|
||||
Ok(format!(
|
||||
"{}[output cut at {shown} lines of {}: narrow paths or glob, or lower limit]\n",
|
||||
&text[..cut],
|
||||
text.lines().count()
|
||||
))
|
||||
}
|
||||
|
||||
/// Plain text for the model.
|
||||
struct Text(String);
|
||||
|
||||
impl ToolOutput for Text {
|
||||
fn log_output(&self) -> String {
|
||||
self.0.clone()
|
||||
}
|
||||
|
||||
fn success_for_logging(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem {
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id: call_id.to_string(),
|
||||
output: FunctionCallOutputPayload {
|
||||
body: FunctionCallOutputBody::Text(self.0.clone()),
|
||||
success: Some(true),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn code_mode_result(&self, _payload: &ToolPayload) -> Value {
|
||||
Value::String(self.0.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
fn args(value: Value) -> Args {
|
||||
serde_json::from_value(value).unwrap_or_else(|error| panic!("{error}"))
|
||||
}
|
||||
|
||||
fn tree() -> tempfile::TempDir {
|
||||
let dir = tempfile::tempdir().unwrap_or_else(|error| panic!("{error}"));
|
||||
for (path, text) in [
|
||||
("src/a.rs", "fn alpha() {}\nalpha();\n"),
|
||||
("src/b.rs", "alpha();\n"),
|
||||
("notes.md", "nothing\n"),
|
||||
] {
|
||||
let path = dir.path().join(path);
|
||||
std::fs::create_dir_all(path.parent().unwrap_or(dir.path()))
|
||||
.unwrap_or_else(|error| panic!("{error}"));
|
||||
std::fs::write(path, text).unwrap_or_else(|error| panic!("{error}"));
|
||||
}
|
||||
dir
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_schema_parses_and_names_the_tool() {
|
||||
let tool = Grep(Arc::new(Search::new()));
|
||||
let ToolSpec::Function(spec) = tool.spec() else {
|
||||
panic!("grep is a function tool");
|
||||
};
|
||||
assert_eq!(spec.name, NAME);
|
||||
assert!(is_grep(&tool.tool_name()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn it_answers_text_names_and_symbols_as_rg_lines() {
|
||||
let dir = tree();
|
||||
let search = Search::new();
|
||||
let text = answer(
|
||||
&search,
|
||||
args(json!({"pattern": "alpha"})),
|
||||
dir.path(),
|
||||
BYTES,
|
||||
);
|
||||
assert_eq!(
|
||||
text,
|
||||
Ok("src/a.rs:1:fn alpha() {}\nsrc/a.rs:2:alpha();\nsrc/b.rs:1:alpha();\n".to_string())
|
||||
);
|
||||
let names = answer(
|
||||
&search,
|
||||
args(json!({"pattern": "*.rs", "kind": "name"})),
|
||||
dir.path(),
|
||||
BYTES,
|
||||
);
|
||||
assert_eq!(names, Ok("src/a.rs\nsrc/b.rs\n".to_string()));
|
||||
let symbol = answer(
|
||||
&search,
|
||||
args(json!({"pattern": "alpha", "kind": "symbol", "output": "count"})),
|
||||
dir.path(),
|
||||
BYTES,
|
||||
);
|
||||
assert_eq!(symbol, Ok("src/a.rs:2\nsrc/b.rs:1\n".to_string()));
|
||||
let none = answer(&search, args(json!({"pattern": "zzz"})), dir.path(), BYTES);
|
||||
assert_eq!(none, Ok("no matches".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bad_pattern_sends_the_model_to_rg() {
|
||||
let dir = tree();
|
||||
let Err(error) = answer(
|
||||
&Search::new(),
|
||||
args(json!({"pattern": "("})),
|
||||
dir.path(),
|
||||
BYTES,
|
||||
) else {
|
||||
panic!("a pattern that does not compile is an error");
|
||||
};
|
||||
assert!(error.contains("Use rg in the shell"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_past_the_budget_is_cut_at_a_line_and_says_so() {
|
||||
let dir = tree();
|
||||
std::fs::write(dir.path().join("src/c.rs"), "alpha();\n".repeat(30))
|
||||
.unwrap_or_else(|error| panic!("{error}"));
|
||||
let text = answer(
|
||||
&Search::new(),
|
||||
args(json!({"pattern": "alpha"})),
|
||||
dir.path(),
|
||||
200,
|
||||
)
|
||||
.unwrap_or_else(|error| panic!("{error}"));
|
||||
assert!(text.len() <= 200, "{text}");
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
assert_eq!(lines[0], "src/c.rs:1:alpha();");
|
||||
assert_eq!(
|
||||
lines.last().copied(),
|
||||
Some(
|
||||
format!(
|
||||
"[output cut at {} lines of 33: narrow paths or glob, or lower limit]",
|
||||
lines.len() - 1
|
||||
)
|
||||
.as_str()
|
||||
)
|
||||
);
|
||||
let limited = answer(
|
||||
&Search::new(),
|
||||
args(json!({"pattern": "alpha", "limit": 1})),
|
||||
dir.path(),
|
||||
BYTES,
|
||||
)
|
||||
.unwrap_or_else(|error| panic!("{error}"));
|
||||
assert_eq!(
|
||||
limited,
|
||||
"src/c.rs:1:alpha();\n[1 hits shown, best first; more may exist: narrow paths or glob, or raise limit]\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_arguments_are_refused() {
|
||||
let parsed: Result<Args, _> =
|
||||
serde_json::from_value(json!({"pattern": "x", "regex": true}));
|
||||
assert!(parsed.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_grep_itself_is_not_a_write() {
|
||||
assert!(is_grep(&ToolName::plain(NAME)));
|
||||
assert!(!is_grep(&ToolName::plain("apply_patch")));
|
||||
assert!(!is_grep(&ToolName::namespaced("web", NAME)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
[package]
|
||||
name = "hanzo-search"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
description = "Dev's one search engine: text, file names and symbols over tgrep's trigram index."
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "hanzo-search"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
# tgrep (MIT), pinned by exact commit: upstream v1.2.1. The root workspace patches its
|
||||
# rusqlite to the release that shares Dev's SQLite.
|
||||
tgrep-core = { git = "https://github.com/microsoft/tgrep", rev = "c7b2ab827274f8a56ff95eadc005f69e0732ee92" }
|
||||
clap = { workspace = true, features = ["derive"] }
|
||||
globset = { workspace = true }
|
||||
memchr = { workspace = true }
|
||||
rayon = "1"
|
||||
regex = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
pretty_assertions = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
@@ -0,0 +1,378 @@
|
||||
//! Matching: a query compiled once, then run over files read from disk.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use std::time::SystemTime;
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
use globset::Glob;
|
||||
use globset::GlobBuilder;
|
||||
use globset::GlobSet;
|
||||
use globset::GlobSetBuilder;
|
||||
use rayon::prelude::*;
|
||||
use regex::Regex;
|
||||
use regex::RegexBuilder;
|
||||
use tgrep_core::encoding::EncodingMode;
|
||||
use tgrep_core::walker;
|
||||
|
||||
use crate::Error;
|
||||
use crate::Hit;
|
||||
use crate::Kind;
|
||||
use crate::Mark;
|
||||
use crate::Query;
|
||||
|
||||
/// Declaration keywords that put the declared name right after them.
|
||||
const DECLARE: &str = "fn|def|class|struct|enum|union|trait|interface|type|mod|module|namespace|\
|
||||
const|static|let|var|val|function|typedef|macro_rules!";
|
||||
|
||||
/// A query compiled for files.
|
||||
pub(crate) struct Plan {
|
||||
pub kind: Kind,
|
||||
/// The regex as tgrep's planner and server receive it.
|
||||
pub pattern: String,
|
||||
/// Line matcher for text and symbol queries.
|
||||
pub line: Option<Regex>,
|
||||
/// Declaration matcher for symbol queries.
|
||||
pub declare: Option<Regex>,
|
||||
/// Name matcher for name queries.
|
||||
pub name: Option<Names>,
|
||||
/// File filter for text and symbol queries: the globs to keep, the `!` globs to drop.
|
||||
pub keep: Option<GlobSet>,
|
||||
pub drop: Option<GlobSet>,
|
||||
pub ignore_case: bool,
|
||||
pub context: usize,
|
||||
}
|
||||
|
||||
/// A name glob: matched against the file name when it has no `/`, else against the
|
||||
/// path relative to the searched directory.
|
||||
pub(crate) struct Names {
|
||||
set: GlobSet,
|
||||
base: bool,
|
||||
}
|
||||
|
||||
impl Names {
|
||||
pub fn matches(&self, rel: &str) -> bool {
|
||||
if self.base {
|
||||
self.set.is_match(rel.rsplit('/').next().unwrap_or(rel))
|
||||
} else {
|
||||
self.set.is_match(rel)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Plan {
|
||||
pub fn new(query: &Query) -> Result<Plan, Error> {
|
||||
let pattern = match query.kind {
|
||||
Kind::Text if query.fixed => regex::escape(&query.pattern),
|
||||
Kind::Text | Kind::Name => query.pattern.clone(),
|
||||
Kind::Symbol => format!(r"\b{}\b", regex::escape(&query.pattern)),
|
||||
};
|
||||
let compile = |pattern: &str| {
|
||||
RegexBuilder::new(pattern)
|
||||
.case_insensitive(query.ignore_case)
|
||||
.multi_line(true)
|
||||
.crlf(true)
|
||||
.build()
|
||||
.map_err(|error| Error::Pattern(error.to_string()))
|
||||
};
|
||||
let (line, declare, name) = match query.kind {
|
||||
Kind::Text => (Some(compile(&pattern)?), None, None),
|
||||
Kind::Symbol => {
|
||||
let name = regex::escape(&query.pattern);
|
||||
let declare = format!(
|
||||
r"\b(?:{DECLARE})\s+(?:mut\s+)?{name}\b|\bfunc\s+(?:\([^)]*\)\s*)?{name}\b|^\s*#\s*define\s+{name}\b|\b{name}\s*[:=]\s*(?:function\b|async\b|lambda\b|\([^)]*\)\s*=>)"
|
||||
);
|
||||
(Some(compile(&pattern)?), Some(compile(&declare)?), None)
|
||||
}
|
||||
Kind::Name => {
|
||||
let glob = GlobBuilder::new(&query.pattern)
|
||||
.case_insensitive(query.ignore_case)
|
||||
.literal_separator(true)
|
||||
.build()
|
||||
.map_err(|error| Error::Pattern(error.to_string()))?;
|
||||
let set = GlobSetBuilder::new()
|
||||
.add(glob)
|
||||
.build()
|
||||
.map_err(|error| Error::Pattern(error.to_string()))?;
|
||||
let base = !query.pattern.contains('/');
|
||||
(None, None, Some(Names { set, base }))
|
||||
}
|
||||
};
|
||||
let set = |globs: Vec<&str>| -> Result<Option<GlobSet>, Error> {
|
||||
if globs.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut set = GlobSetBuilder::new();
|
||||
for glob in globs {
|
||||
let glob = if glob.contains('/') {
|
||||
Glob::new(glob)
|
||||
} else {
|
||||
Glob::new(&format!("**/{glob}"))
|
||||
}
|
||||
.map_err(|error| Error::Pattern(error.to_string()))?;
|
||||
set.add(glob);
|
||||
}
|
||||
set.build()
|
||||
.map(Some)
|
||||
.map_err(|error| Error::Pattern(error.to_string()))
|
||||
};
|
||||
let keep = set(query
|
||||
.glob
|
||||
.iter()
|
||||
.filter(|glob| !glob.starts_with('!'))
|
||||
.map(String::as_str)
|
||||
.collect())?;
|
||||
let drop = set(query
|
||||
.glob
|
||||
.iter()
|
||||
.filter_map(|glob| glob.strip_prefix('!'))
|
||||
.collect())?;
|
||||
Ok(Plan {
|
||||
kind: query.kind,
|
||||
pattern,
|
||||
line,
|
||||
declare,
|
||||
name,
|
||||
keep,
|
||||
drop,
|
||||
ignore_case: query.ignore_case,
|
||||
context: query.context,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether a text or symbol query reads the file at `rel` (relative to the
|
||||
/// searched directory).
|
||||
pub fn admits(&self, rel: &str) -> bool {
|
||||
self.keep.as_ref().is_none_or(|set| set.is_match(rel))
|
||||
&& !self.drop.as_ref().is_some_and(|set| set.is_match(rel))
|
||||
}
|
||||
|
||||
/// The hits in one file's text. `path` is the absolute path the hits carry.
|
||||
pub fn lines(&self, path: &Path, text: &str) -> Vec<Hit> {
|
||||
let Some(regex) = &self.line else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut found: Vec<(usize, usize)> = Vec::new();
|
||||
let mut at = 0;
|
||||
while at <= text.len() {
|
||||
let Some(hit) = regex.find_at(text, at) else {
|
||||
break;
|
||||
};
|
||||
// The empty line after a final terminator is not a line.
|
||||
if hit.start() == text.len() && (text.is_empty() || text.ends_with('\n')) {
|
||||
break;
|
||||
}
|
||||
let start = text[..hit.start()].rfind('\n').map_or(0, |i| i + 1);
|
||||
let end = text[hit.start()..]
|
||||
.find('\n')
|
||||
.map_or(text.len(), |i| hit.start() + i);
|
||||
let line = text[start..end]
|
||||
.strip_suffix('\r')
|
||||
.unwrap_or(&text[start..end]);
|
||||
if regex.is_match(line) {
|
||||
found.push((start, end));
|
||||
}
|
||||
if end >= text.len() {
|
||||
break;
|
||||
}
|
||||
at = end + 1;
|
||||
}
|
||||
if found.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let lines: Vec<&str> = if self.context > 0 {
|
||||
text.split('\n')
|
||||
.map(|line| line.strip_suffix('\r').unwrap_or(line))
|
||||
.collect()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let mut number = 1u64;
|
||||
let mut counted = 0;
|
||||
found
|
||||
.into_iter()
|
||||
.map(|(start, end)| {
|
||||
number +=
|
||||
memchr::memchr_iter(b'\n', &text.as_bytes()[counted..start]).count() as u64;
|
||||
counted = start;
|
||||
let excerpt = text[start..end]
|
||||
.strip_suffix('\r')
|
||||
.unwrap_or(&text[start..end])
|
||||
.to_string();
|
||||
let mark = match &self.declare {
|
||||
None => Mark::Match,
|
||||
Some(declare) if declare.is_match(&excerpt) => Mark::Definition,
|
||||
Some(_) => Mark::Reference,
|
||||
};
|
||||
let index = (number - 1) as usize;
|
||||
let (before, after) = if self.context > 0 {
|
||||
let first = index.saturating_sub(self.context);
|
||||
let last = (index + 1 + self.context).min(lines.len());
|
||||
(
|
||||
lines[first..index]
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect(),
|
||||
lines[(index + 1).min(last)..last]
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect(),
|
||||
)
|
||||
} else {
|
||||
(Vec::new(), Vec::new())
|
||||
};
|
||||
Hit {
|
||||
path: path.to_path_buf(),
|
||||
entry: None,
|
||||
place: None,
|
||||
line: Some(number),
|
||||
excerpt,
|
||||
before,
|
||||
after,
|
||||
mark,
|
||||
score: 0,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A name hit for `path` when its relative form matches.
|
||||
pub fn file(&self, path: &Path, rel: &str) -> Option<Hit> {
|
||||
let names = self.name.as_ref()?;
|
||||
names.matches(rel).then(|| Hit {
|
||||
path: path.to_path_buf(),
|
||||
entry: None,
|
||||
place: None,
|
||||
line: None,
|
||||
excerpt: String::new(),
|
||||
before: Vec::new(),
|
||||
after: Vec::new(),
|
||||
mark: Mark::File,
|
||||
score: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The file's searchable text: decoded as tgrep decodes it, `None` when it is
|
||||
/// binary (holds a NUL) or unreadable.
|
||||
pub(crate) fn read(path: &Path) -> Option<String> {
|
||||
let bytes = std::fs::read(path).ok()?;
|
||||
let (text, _) = tgrep_core::encoding::decode_owned_with_fixups(bytes, EncodingMode::Auto);
|
||||
(!text.as_bytes().contains(&0)).then_some(text)
|
||||
}
|
||||
|
||||
/// Searches the given absolute files from disk, in parallel.
|
||||
pub(crate) fn grep(plan: &Plan, files: &[PathBuf]) -> Vec<Hit> {
|
||||
files
|
||||
.par_iter()
|
||||
.flat_map_iter(|path| match read(path) {
|
||||
Some(text) => plan.lines(path, &text),
|
||||
None => Vec::new(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// `rel` of `path` beneath `scope`, `/`-separated.
|
||||
pub(crate) fn relative(scope: &Path, path: &Path) -> String {
|
||||
path.strip_prefix(scope)
|
||||
.unwrap_or(path)
|
||||
.to_string_lossy()
|
||||
.replace('\\', "/")
|
||||
}
|
||||
|
||||
/// Answers a query by walking `scope`: tgrep's walker, with `.gitignore`, hidden
|
||||
/// files and binary extensions applied as `rg` applies them.
|
||||
pub(crate) fn scan(plan: &Plan, scope: &Path) -> Result<Vec<Hit>, Error> {
|
||||
if scope.is_file() {
|
||||
let rel = scope
|
||||
.file_name()
|
||||
.map(|name| name.to_string_lossy().into_owned())
|
||||
.unwrap_or_default();
|
||||
return Ok(match plan.kind {
|
||||
Kind::Name => plan.file(scope, &rel).into_iter().collect(),
|
||||
Kind::Text | Kind::Symbol => grep(plan, &[scope.to_path_buf()]),
|
||||
});
|
||||
}
|
||||
let walk = walker::walk_dir(scope, &walker::WalkOptions::default());
|
||||
Ok(match plan.kind {
|
||||
Kind::Name => walk
|
||||
.listed_files
|
||||
.iter()
|
||||
.filter_map(|path| plan.file(path, &relative(scope, path)))
|
||||
.collect(),
|
||||
Kind::Text | Kind::Symbol => {
|
||||
let files: Vec<PathBuf> = walk
|
||||
.files
|
||||
.into_iter()
|
||||
.filter(|path| plan.admits(&relative(scope, path)))
|
||||
.collect();
|
||||
grep(plan, &files)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// The files under `scope` an index may not hold as they are: the `touched` ones
|
||||
/// that are not hidden, and, with `since`, every file whose content or name
|
||||
/// changed at or after it (a walk: names and metadata, no contents).
|
||||
pub(crate) fn dirty(
|
||||
scope: &Path,
|
||||
touched: &[PathBuf],
|
||||
since: Option<SystemTime>,
|
||||
) -> BTreeSet<PathBuf> {
|
||||
let mut dirty: BTreeSet<PathBuf> = touched
|
||||
.iter()
|
||||
.filter(|path| path.starts_with(scope))
|
||||
.filter(|path| {
|
||||
!relative(scope, path)
|
||||
.split('/')
|
||||
.any(|part| part.starts_with('.'))
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
if let Some(since) = since {
|
||||
let walk = walker::walk_dir(scope, &walker::WalkOptions::default());
|
||||
dirty.extend(
|
||||
walk.listed_files
|
||||
.par_iter()
|
||||
.filter(|path| {
|
||||
std::fs::symlink_metadata(path)
|
||||
.is_ok_and(|metadata| changed(&metadata) >= since)
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
}
|
||||
dirty
|
||||
}
|
||||
|
||||
/// The dirty files a text or symbol query reads: present, admitted, not binary.
|
||||
pub(crate) fn fresh(plan: &Plan, scope: &Path, dirty: &BTreeSet<PathBuf>) -> Vec<PathBuf> {
|
||||
dirty
|
||||
.iter()
|
||||
.filter(|path| path.is_file() && !walker::is_binary_extension(path))
|
||||
.filter(|path| plan.admits(&relative(scope, path)))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The later of a file's content change and its status change: a rename moves the
|
||||
/// second, not the first.
|
||||
fn changed(metadata: &std::fs::Metadata) -> SystemTime {
|
||||
let modified = metadata.modified().unwrap_or(UNIX_EPOCH);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let status = UNIX_EPOCH
|
||||
+ Duration::new(
|
||||
u64::try_from(metadata.ctime()).unwrap_or(0),
|
||||
u32::try_from(metadata.ctime_nsec()).unwrap_or(0),
|
||||
);
|
||||
modified.max(status)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
modified
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
//! tgrep's on-disk index: where it is, building it, and answering from it in process.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use std::time::SystemTime;
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
use tgrep_core::builder;
|
||||
use tgrep_core::meta;
|
||||
use tgrep_core::meta::IndexMeta;
|
||||
use tgrep_core::path_index;
|
||||
use tgrep_core::query;
|
||||
use tgrep_core::reader::IndexReader;
|
||||
|
||||
use crate::Error;
|
||||
use crate::Hit;
|
||||
use crate::Kind;
|
||||
use crate::Root;
|
||||
use crate::find;
|
||||
use crate::find::Plan;
|
||||
use crate::server;
|
||||
|
||||
/// The nearest directory at or above `path` that holds a tgrep index.
|
||||
pub(crate) fn locate(path: &Path) -> Option<PathBuf> {
|
||||
let start = if path.is_file() { path.parent()? } else { path };
|
||||
start
|
||||
.ancestors()
|
||||
.find(|dir| builder::default_index_dir(dir).join("meta.json").is_file())
|
||||
.map(Path::to_path_buf)
|
||||
}
|
||||
|
||||
/// Builds the index for `root` into `root/.tgrep`, holding tgrep's writer lock, as
|
||||
/// `tgrep index` does.
|
||||
pub(crate) fn build(root: &Path) -> Result<(), Error> {
|
||||
let fail = |message: String| Error::Build(root.to_path_buf(), message);
|
||||
let dir = builder::default_index_dir(root);
|
||||
std::fs::create_dir_all(&dir).map_err(|error| fail(error.to_string()))?;
|
||||
let lock = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(dir.join("serve.lock"))
|
||||
.map_err(|error| fail(error.to_string()))?;
|
||||
lock.try_lock()
|
||||
.map_err(|_| fail("an index server or build owns this index".to_string()))?;
|
||||
builder::build_index_with_options(root, Some(&dir), &builder::BuildOptions::default())
|
||||
.map_err(|error| fail(error.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The index state of `root`.
|
||||
pub(crate) fn root(root: &Path) -> Root {
|
||||
let dir = builder::default_index_dir(root);
|
||||
let meta = IndexMeta::load(&dir).ok();
|
||||
let server = server::attach(root);
|
||||
let status = server.as_ref().and_then(server::Server::status);
|
||||
let ready = match &status {
|
||||
Some(status) => status["hidden_complete"].as_bool() == Some(true),
|
||||
None => meta.as_ref().is_some_and(usable),
|
||||
};
|
||||
let files = status
|
||||
.as_ref()
|
||||
.and_then(|status| status["num_files"].as_u64())
|
||||
.or(meta.as_ref().map(|meta| meta.num_files))
|
||||
.unwrap_or(0);
|
||||
let refreshed = match &server {
|
||||
Some(server) => server.reconciled(),
|
||||
None => meta
|
||||
.as_ref()
|
||||
.map(|meta| UNIX_EPOCH + Duration::from_secs(meta.updated_at)),
|
||||
};
|
||||
let bytes = std::fs::read_dir(&dir)
|
||||
.map(|entries| {
|
||||
entries
|
||||
.filter_map(Result::ok)
|
||||
.filter_map(|entry| entry.metadata().ok())
|
||||
.filter(std::fs::Metadata::is_file)
|
||||
.map(|metadata| metadata.len())
|
||||
.sum()
|
||||
})
|
||||
.unwrap_or(0);
|
||||
Root {
|
||||
path: root.to_path_buf(),
|
||||
server: server.map(|server| server.pid),
|
||||
ready,
|
||||
files,
|
||||
bytes,
|
||||
refreshed,
|
||||
lag: refreshed.and_then(|at| SystemTime::now().duration_since(at).ok()),
|
||||
}
|
||||
}
|
||||
|
||||
fn usable(meta: &IndexMeta) -> bool {
|
||||
meta.complete && meta.hidden_complete && meta.version == meta::INDEX_FORMAT_VERSION
|
||||
}
|
||||
|
||||
/// `/`-separated path of `path` under `root`, empty for `root` itself.
|
||||
pub(crate) fn indexed(root: &Path, path: &Path) -> String {
|
||||
find::relative(root, path)
|
||||
}
|
||||
|
||||
/// Answers from the on-disk index of `root`, or `None` when it cannot (missing,
|
||||
/// partial, or from another tgrep format).
|
||||
///
|
||||
/// Candidates come from the trigram index and are matched against their current
|
||||
/// bytes; a candidate that no longer exists is skipped. `dirty` files are searched
|
||||
/// from disk instead of the index.
|
||||
pub(crate) fn query(
|
||||
plan: &Plan,
|
||||
root: &Path,
|
||||
scope: &Path,
|
||||
dirty: &BTreeSet<PathBuf>,
|
||||
) -> Option<Result<Vec<Hit>, Error>> {
|
||||
let dir = builder::default_index_dir(root);
|
||||
let meta = IndexMeta::load(&dir).ok().filter(usable)?;
|
||||
let reader = IndexReader::open(&dir).ok()?;
|
||||
let names = path_index::read_filename_index(&dir).ok()??;
|
||||
let visibility = names.visibility?;
|
||||
if !visibility.covers_index(&meta, reader.file_table_id()) {
|
||||
return None;
|
||||
}
|
||||
let prefix = match indexed(root, scope) {
|
||||
rel if rel.is_empty() => String::new(),
|
||||
rel => format!("{rel}/"),
|
||||
};
|
||||
let stale: BTreeSet<String> = dirty.iter().map(|path| indexed(root, path)).collect();
|
||||
let current = |path: &&str| {
|
||||
path.starts_with(&prefix)
|
||||
&& visibility.paths.is_visible(path, &prefix, false)
|
||||
&& !stale.contains(*path)
|
||||
};
|
||||
|
||||
if plan.kind == Kind::Name {
|
||||
let mut hits: Vec<Hit> = reader
|
||||
.all_paths()
|
||||
.iter()
|
||||
.chain(names.paths.iter())
|
||||
.map(String::as_str)
|
||||
.filter(current)
|
||||
.filter_map(|path| plan.file(&root.join(path), &path[prefix.len()..]))
|
||||
.filter(|hit| hit.path.symlink_metadata().is_ok())
|
||||
.collect();
|
||||
hits.extend(
|
||||
dirty
|
||||
.iter()
|
||||
.filter(|path| path.is_file())
|
||||
.filter_map(|path| plan.file(path, &find::relative(scope, path))),
|
||||
);
|
||||
return Some(Ok(hits));
|
||||
}
|
||||
|
||||
let trigrams = match query::build_query_plan(&plan.pattern, plan.ignore_case) {
|
||||
Ok(trigrams) => trigrams,
|
||||
Err(error) => return Some(Err(Error::Pattern(error))),
|
||||
};
|
||||
let ids = if trigrams.is_match_all() {
|
||||
reader.all_file_ids()
|
||||
} else {
|
||||
query::execute_plan_with_masks(&trigrams, &|trigram| {
|
||||
reader.lookup_trigram_with_masks(trigram)
|
||||
})
|
||||
};
|
||||
let mut files: Vec<PathBuf> = ids
|
||||
.iter()
|
||||
.filter_map(|&id| reader.file_path(id))
|
||||
.filter(current)
|
||||
.filter(|path| plan.admits(&path[prefix.len()..]))
|
||||
.map(|path| root.join(path))
|
||||
.collect();
|
||||
files.extend(find::fresh(plan, scope, dirty));
|
||||
Some(Ok(find::grep(plan, &files)))
|
||||
}
|
||||
@@ -0,0 +1,543 @@
|
||||
//! Dev's one search engine: text, file names and symbols, over tgrep's trigram index.
|
||||
//!
|
||||
//! Every search in Dev goes through [`Search::query`]: the agent's `grep` tool
|
||||
//! (crate `hanzo-grep`), the loop, and the `hanzo-search` command. There is no
|
||||
//! second path. The engine is [tgrep](https://github.com/microsoft/tgrep) (MIT),
|
||||
//! linked as the `tgrep-core` library and pinned by exact commit in `Cargo.toml`;
|
||||
//! nothing is spawned.
|
||||
//!
|
||||
//! # API
|
||||
//!
|
||||
//! ```no_run
|
||||
//! use hanzo_search::{Query, Search};
|
||||
//!
|
||||
//! let search = Search::new();
|
||||
//! let hits = search.query(&Query::text("/repo", "fn main"))?;
|
||||
//! for hit in &hits {
|
||||
//! println!("{}:{}:{}", hit.path.display(), hit.line.unwrap_or(0), hit.excerpt);
|
||||
//! }
|
||||
//! search.touched(std::path::Path::new("/repo/src/lib.rs")); // Dev wrote it
|
||||
//! let status = search.status();
|
||||
//! # Ok::<(), hanzo_search::Error>(())
|
||||
//! ```
|
||||
//!
|
||||
//! A [`Query`] has one of three kinds:
|
||||
//!
|
||||
//! - [`Kind::Text`]: a regex (or a literal with `fixed`) matched line by line, as `rg`.
|
||||
//! - [`Kind::Name`]: a glob over file paths, as `rg --files -g`. A glob without `/`
|
||||
//! matches the file name at any depth; one with `/` matches the path relative to
|
||||
//! the searched directory.
|
||||
//! - [`Kind::Symbol`]: an identifier matched on word boundaries. Each hit is marked
|
||||
//! [`Mark::Definition`] when the line reads like a declaration of it (`fn x`,
|
||||
//! `def x`, `class x`, `const x`, `func (r) x`, `x = function`, `#define x`, ...)
|
||||
//! and [`Mark::Reference`] otherwise. This is textual: it finds the lines that
|
||||
//! spell the name, not every semantic caller, and it can call a comment a
|
||||
//! reference. A [`Resolver`] installed with [`Search::with_resolver`] (Dev's
|
||||
//! LSP or code graph) confirms or drops the candidates.
|
||||
//!
|
||||
//! # Ranking
|
||||
//!
|
||||
//! Results are ordered by file score, highest first, then path, then line. The
|
||||
//! score is per file:
|
||||
//!
|
||||
//! - Text: the number of matching lines in the file.
|
||||
//! - Symbol: definitions × 2³² + references, so every file that defines the name
|
||||
//! comes before every file that only uses it.
|
||||
//! - Name: `u64::MAX` less the path's depth, so shallower paths come first.
|
||||
//!
|
||||
//! `limit` keeps the first `limit` hits of that order.
|
||||
//!
|
||||
//! # Output
|
||||
//!
|
||||
//! [`render`] prints hits the way `rg -n --no-heading` does: `path:line:text` for
|
||||
//! a match, `path-line-text` for a context line, `--` between context groups that
|
||||
//! are not adjacent, `path` for a name hit, `path:N` for a count. A hit inside a
|
||||
//! document or archive (from the tgrep fork's document adapters) prints its
|
||||
//! archive members after the path, each behind `!`, and its place before the
|
||||
//! line: `docs/a.zip!inner/b.docx:page 2:14:text`. JSON prints one object per hit
|
||||
//! with the fields of [`Hit`].
|
||||
//!
|
||||
//! # Engines and fallback
|
||||
//!
|
||||
//! A query path belongs to the nearest enclosing directory with a tgrep index
|
||||
//! (`.tgrep/`). For that root, in order:
|
||||
//!
|
||||
//! 1. a running tgrep server (found by `.tgrep/serve.json`) answers over its
|
||||
//! loopback JSON-RPC;
|
||||
//! 2. else the on-disk index answers in process: trigram candidates, then the
|
||||
//! regex over each candidate's current bytes;
|
||||
//! 3. else, or with no index at all, the tree is scanned with tgrep's walker,
|
||||
//! which applies `.gitignore`, hidden-file and binary rules as `rg` does.
|
||||
//!
|
||||
//! [`Index::Skip`] forces the scan; [`Index::Only`] refuses it.
|
||||
//!
|
||||
//! # Freshness
|
||||
//!
|
||||
//! An index can lag the disk; a search after Dev wrote a file must still see the
|
||||
//! write. So, before a root's index answers, its dirty files are read from disk
|
||||
//! instead, and the index's rows for them are dropped:
|
||||
//!
|
||||
//! - every path given to [`Search::touched`] (Dev named the file it wrote);
|
||||
//! - with a time given to [`Search::since`] (Dev started a tool that may write
|
||||
//! files it does not name), every file whose content or name (ctime) changed at
|
||||
//! or after it, found by a walk of the searched directory: names and metadata,
|
||||
//! no contents.
|
||||
//!
|
||||
//! For a server's root, a touch or `since` older than the start of the server's
|
||||
//! last full reconcile no longer counts: its watcher has the write. An on-disk
|
||||
//! index never catches up, so there the walk repeats on every query once `since`
|
||||
//! is set; that walk is what freshness costs without a watcher. A write by another
|
||||
//! process is seen by a server's watcher, and by an on-disk index only once it is
|
||||
//! rebuilt, as tgrep documents.
|
||||
//!
|
||||
//! A row whose file no longer exists is dropped, so a deleted or renamed file is
|
||||
//! never reported from the index.
|
||||
//!
|
||||
//! # Licence boundary
|
||||
//!
|
||||
//! tgrep is MIT and linked. ripgrep-all (`rga`) is AGPL-3.0 and is never used:
|
||||
//! not linked, copied, called, shipped, or used as a comparator.
|
||||
|
||||
mod find;
|
||||
mod index;
|
||||
mod render;
|
||||
mod server;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
|
||||
pub use render::Format;
|
||||
pub use render::render;
|
||||
|
||||
/// What a query looks for.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Kind {
|
||||
/// A regex, or a literal with [`Query::fixed`], matched per line.
|
||||
#[default]
|
||||
Text,
|
||||
/// A glob over file paths.
|
||||
Name,
|
||||
/// An identifier on word boundaries, its definitions ranked first.
|
||||
Symbol,
|
||||
}
|
||||
|
||||
/// Which engine may answer.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum Index {
|
||||
/// A server, else the on-disk index, else a scan.
|
||||
#[default]
|
||||
Auto,
|
||||
/// Scan the files; never read an index.
|
||||
Skip,
|
||||
/// A server or the on-disk index; a path with neither is an error.
|
||||
Only,
|
||||
}
|
||||
|
||||
/// One search.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct Query {
|
||||
pub kind: Kind,
|
||||
/// The regex, glob or identifier.
|
||||
pub pattern: String,
|
||||
/// Where relative paths resolve and what hit paths are shown relative to.
|
||||
pub root: PathBuf,
|
||||
/// Files or directories to search; empty searches `root`.
|
||||
pub paths: Vec<PathBuf>,
|
||||
/// Text and symbol queries read only files whose path matches one of these.
|
||||
pub glob: Vec<String>,
|
||||
/// The pattern is a literal.
|
||||
pub fixed: bool,
|
||||
pub ignore_case: bool,
|
||||
/// Lines of context on each side of a text or symbol hit.
|
||||
pub context: usize,
|
||||
/// The most hits returned, best first.
|
||||
pub limit: Option<usize>,
|
||||
pub index: Index,
|
||||
}
|
||||
|
||||
impl Query {
|
||||
/// A regex over the files under `root`.
|
||||
pub fn text(root: impl Into<PathBuf>, pattern: impl Into<String>) -> Query {
|
||||
Query {
|
||||
kind: Kind::Text,
|
||||
pattern: pattern.into(),
|
||||
root: root.into(),
|
||||
..Query::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// A glob over the file paths under `root`.
|
||||
pub fn name(root: impl Into<PathBuf>, glob: impl Into<String>) -> Query {
|
||||
Query {
|
||||
kind: Kind::Name,
|
||||
pattern: glob.into(),
|
||||
root: root.into(),
|
||||
..Query::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// The textual definitions and references of an identifier under `root`.
|
||||
pub fn symbol(root: impl Into<PathBuf>, name: impl Into<String>) -> Query {
|
||||
Query {
|
||||
kind: Kind::Symbol,
|
||||
pattern: name.into(),
|
||||
root: root.into(),
|
||||
..Query::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What a hit is.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Mark {
|
||||
/// A line matching a text query.
|
||||
Match,
|
||||
/// A file matching a name query.
|
||||
File,
|
||||
/// A line that reads like a declaration of the symbol.
|
||||
Definition,
|
||||
/// Any other line spelling the symbol.
|
||||
Reference,
|
||||
}
|
||||
|
||||
/// One result.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct Hit {
|
||||
/// Relative to the query's root when beneath it, else absolute.
|
||||
pub path: PathBuf,
|
||||
/// Archive members from the outer file inward, joined by `!`
|
||||
/// (`inner/b.docx` inside `a.zip`). `None` for a plain file.
|
||||
pub entry: Option<String>,
|
||||
/// Where in a document: `page 2`, `slide 3`, `sheet Data!B4`. `None` for text.
|
||||
pub place: Option<String>,
|
||||
/// 1-based. `None` for a name hit.
|
||||
pub line: Option<u64>,
|
||||
/// The matching line without its terminator; the path for a name hit.
|
||||
pub excerpt: String,
|
||||
/// Up to `context` lines before the hit, in order.
|
||||
pub before: Vec<String>,
|
||||
/// Up to `context` lines after the hit, in order.
|
||||
pub after: Vec<String>,
|
||||
pub mark: Mark,
|
||||
/// The file's score; see the crate docs for the ranking.
|
||||
pub score: u64,
|
||||
}
|
||||
|
||||
/// Confirms textual symbol candidates semantically.
|
||||
///
|
||||
/// [`Kind::Symbol`] hits are lines that spell the name. A resolver (an LSP, a code
|
||||
/// graph) returns the ones that really define or use the symbol, and may change
|
||||
/// their [`Mark`]. Hits it drops are gone; the result is ranked again after.
|
||||
pub trait Resolver: Send + Sync {
|
||||
fn resolve(&self, symbol: &str, hits: Vec<Hit>) -> Vec<Hit>;
|
||||
}
|
||||
|
||||
/// Why a query failed.
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
/// The pattern or a glob does not compile.
|
||||
Pattern(String),
|
||||
/// A path cannot be read.
|
||||
Io(PathBuf, std::io::Error),
|
||||
/// [`Index::Only`] and no index answers for this path.
|
||||
Unindexed(PathBuf),
|
||||
/// An index cannot be built.
|
||||
Build(PathBuf, String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Error::Pattern(message) => write!(f, "{message}"),
|
||||
Error::Io(path, error) => write!(f, "{}: {error}", path.display()),
|
||||
Error::Unindexed(path) => write!(f, "{}: no index answers here", path.display()),
|
||||
Error::Build(path, message) => write!(f, "{}: {message}", path.display()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Error {}
|
||||
|
||||
/// One indexed root as the engine sees it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct Root {
|
||||
pub path: PathBuf,
|
||||
/// Pid of the tgrep server answering for this root.
|
||||
pub server: Option<u32>,
|
||||
/// A complete index can answer.
|
||||
pub ready: bool,
|
||||
pub files: u64,
|
||||
/// Index size on disk.
|
||||
pub bytes: u64,
|
||||
/// When the index last caught up with the disk: the server's last reconcile, or
|
||||
/// the on-disk index's last write.
|
||||
pub refreshed: Option<SystemTime>,
|
||||
/// How long ago that was.
|
||||
pub lag: Option<Duration>,
|
||||
}
|
||||
|
||||
/// The engine's state.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct Status {
|
||||
/// Every indexed root a query has reached, and the ones given to [`Search::status_of`].
|
||||
pub roots: Vec<Root>,
|
||||
/// Paths given to [`Search::touched`].
|
||||
pub touched: usize,
|
||||
/// The earliest time given to [`Search::since`].
|
||||
pub since: Option<SystemTime>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Dirty {
|
||||
touched: BTreeMap<PathBuf, SystemTime>,
|
||||
since: Option<SystemTime>,
|
||||
}
|
||||
|
||||
/// The engine. Cheap to call; holds the dirty set and the roots it has met.
|
||||
#[derive(Default)]
|
||||
pub struct Search {
|
||||
dirty: Mutex<Dirty>,
|
||||
roots: Mutex<BTreeSet<PathBuf>>,
|
||||
resolver: Option<Arc<dyn Resolver>>,
|
||||
}
|
||||
|
||||
impl Search {
|
||||
pub fn new() -> Search {
|
||||
Search::default()
|
||||
}
|
||||
|
||||
/// Confirms [`Kind::Symbol`] hits through `resolver`.
|
||||
pub fn with_resolver(mut self, resolver: Arc<dyn Resolver>) -> Search {
|
||||
self.resolver = Some(resolver);
|
||||
self
|
||||
}
|
||||
|
||||
/// Dev wrote, created, renamed or deleted `path` (absolute). Searches read it
|
||||
/// from disk, never from an index, unless a server reconciled after the write.
|
||||
pub fn touched(&self, path: &Path) {
|
||||
let path = canonical(path);
|
||||
if let Ok(mut dirty) = self.dirty.lock() {
|
||||
dirty.touched.insert(path, SystemTime::now());
|
||||
}
|
||||
}
|
||||
|
||||
/// A tool that may write files it does not name started at `at` (a shell
|
||||
/// command). Files changed at or after `at` bypass an index, unless a server
|
||||
/// reconciled after `at`.
|
||||
pub fn since(&self, at: SystemTime) {
|
||||
if let Ok(mut dirty) = self.dirty.lock() {
|
||||
dirty.since = Some(dirty.since.map_or(at, |since| since.min(at)));
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs a query. See the crate docs for kinds, ranking and freshness.
|
||||
pub fn query(&self, query: &Query) -> Result<Vec<Hit>, Error> {
|
||||
if query.pattern.is_empty() {
|
||||
return Err(Error::Pattern("empty pattern".to_string()));
|
||||
}
|
||||
let plan = find::Plan::new(query)?;
|
||||
let root = absolute(&query.root);
|
||||
let scopes: Vec<PathBuf> = if query.paths.is_empty() {
|
||||
vec![root.clone()]
|
||||
} else {
|
||||
query
|
||||
.paths
|
||||
.iter()
|
||||
.map(|path| absolute(&root.join(path)))
|
||||
.collect()
|
||||
};
|
||||
let mut hits = Vec::new();
|
||||
for scope in scopes {
|
||||
let scope = scope
|
||||
.canonicalize()
|
||||
.map_err(|error| Error::Io(scope.clone(), error))?;
|
||||
hits.extend(self.scope(query, &plan, &scope)?);
|
||||
}
|
||||
let mut seen = BTreeSet::new();
|
||||
hits.retain(|hit| seen.insert((hit.path.clone(), hit.entry.clone(), hit.line)));
|
||||
rank(&mut hits);
|
||||
if let (Kind::Symbol, Some(resolver)) = (query.kind, &self.resolver) {
|
||||
hits = resolver.resolve(&query.pattern, hits);
|
||||
rank(&mut hits);
|
||||
}
|
||||
if let Some(limit) = query.limit {
|
||||
hits.truncate(limit);
|
||||
}
|
||||
let shown = root.canonicalize().unwrap_or(root);
|
||||
for hit in &mut hits {
|
||||
if let Ok(rel) = hit.path.strip_prefix(&shown) {
|
||||
hit.path = if rel.as_os_str().is_empty() {
|
||||
PathBuf::from(".")
|
||||
} else {
|
||||
rel.to_path_buf()
|
||||
};
|
||||
}
|
||||
if hit.mark == Mark::File {
|
||||
hit.excerpt = hit.path.display().to_string();
|
||||
}
|
||||
}
|
||||
Ok(hits)
|
||||
}
|
||||
|
||||
/// The roots queries have reached, with their index state.
|
||||
pub fn status(&self) -> Status {
|
||||
let roots: Vec<PathBuf> = self
|
||||
.roots
|
||||
.lock()
|
||||
.map(|roots| roots.iter().cloned().collect())
|
||||
.unwrap_or_default();
|
||||
self.report(roots)
|
||||
}
|
||||
|
||||
/// [`Search::status`] plus the root that would answer for `path`.
|
||||
pub fn status_of(&self, path: &Path) -> Status {
|
||||
let mut roots: BTreeSet<PathBuf> = self
|
||||
.roots
|
||||
.lock()
|
||||
.map(|roots| roots.clone())
|
||||
.unwrap_or_default();
|
||||
if let Some(root) = index::locate(&absolute(path)) {
|
||||
roots.insert(root);
|
||||
}
|
||||
self.report(roots.into_iter().collect())
|
||||
}
|
||||
|
||||
/// Builds or rebuilds the on-disk index for `path` in `path/.tgrep`. Fails when
|
||||
/// a tgrep server owns that index.
|
||||
pub fn prepare(&self, path: &Path) -> Result<Root, Error> {
|
||||
let root = absolute(path)
|
||||
.canonicalize()
|
||||
.map_err(|error| Error::Io(path.to_path_buf(), error))?;
|
||||
index::build(&root)?;
|
||||
self.note(&root);
|
||||
Ok(index::root(&root))
|
||||
}
|
||||
|
||||
fn report(&self, roots: Vec<PathBuf>) -> Status {
|
||||
let (touched, since) = self
|
||||
.dirty
|
||||
.lock()
|
||||
.map(|dirty| (dirty.touched.len(), dirty.since))
|
||||
.unwrap_or_default();
|
||||
Status {
|
||||
roots: roots.iter().map(|root| index::root(root)).collect(),
|
||||
touched,
|
||||
since,
|
||||
}
|
||||
}
|
||||
|
||||
fn note(&self, root: &Path) {
|
||||
if let Ok(mut roots) = self.roots.lock() {
|
||||
roots.insert(root.to_path_buf());
|
||||
}
|
||||
}
|
||||
|
||||
/// What bypasses `root`'s index: the paths touched under it, and the earliest
|
||||
/// unnamed write, each only when it began at or after `after` (an index that
|
||||
/// reconciled at `after` holds what came before).
|
||||
fn dirt(&self, root: &Path, after: Option<SystemTime>) -> (Vec<PathBuf>, Option<SystemTime>) {
|
||||
let fresh = |when: &SystemTime| after.is_none_or(|at| *when >= at);
|
||||
self.dirty
|
||||
.lock()
|
||||
.map(|dirty| {
|
||||
(
|
||||
dirty
|
||||
.touched
|
||||
.iter()
|
||||
.filter(|(path, when)| path.starts_with(root) && fresh(when))
|
||||
.map(|(path, _)| path.clone())
|
||||
.collect(),
|
||||
dirty.since.filter(fresh),
|
||||
)
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn scope(&self, query: &Query, plan: &find::Plan, scope: &Path) -> Result<Vec<Hit>, Error> {
|
||||
if query.index == Index::Skip || scope.is_file() {
|
||||
return find::scan(plan, scope);
|
||||
}
|
||||
let Some(root) = index::locate(scope) else {
|
||||
return match query.index {
|
||||
Index::Only => Err(Error::Unindexed(scope.to_path_buf())),
|
||||
_ => find::scan(plan, scope),
|
||||
};
|
||||
};
|
||||
self.note(&root);
|
||||
if let Some(server) = server::attach(&root) {
|
||||
let (touched, since) = self.dirt(&root, server.reconciled());
|
||||
let dirty = find::dirty(scope, &touched, since);
|
||||
if let Ok(hits) = server.query(plan, &root, scope, &dirty) {
|
||||
return Ok(hits);
|
||||
}
|
||||
}
|
||||
let (touched, since) = self.dirt(&root, None);
|
||||
let dirty = find::dirty(scope, &touched, since);
|
||||
match index::query(plan, &root, scope, &dirty) {
|
||||
Some(hits) => hits,
|
||||
None => match query.index {
|
||||
Index::Only => Err(Error::Unindexed(scope.to_path_buf())),
|
||||
_ => find::scan(plan, scope),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Orders hits by file score, then path, then line, after scoring each file.
|
||||
fn rank(hits: &mut [Hit]) {
|
||||
let mut scores: BTreeMap<(PathBuf, Option<String>), u64> = BTreeMap::new();
|
||||
for hit in hits.iter() {
|
||||
let score = scores
|
||||
.entry((hit.path.clone(), hit.entry.clone()))
|
||||
.or_insert(0);
|
||||
*score = match hit.mark {
|
||||
Mark::Match => *score + 1,
|
||||
Mark::Definition => *score + (1 << 32),
|
||||
Mark::Reference => *score + 1,
|
||||
Mark::File => u64::MAX - hit.path.components().count() as u64,
|
||||
};
|
||||
}
|
||||
for hit in hits.iter_mut() {
|
||||
hit.score = scores
|
||||
.get(&(hit.path.clone(), hit.entry.clone()))
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
}
|
||||
hits.sort_by(|a, b| {
|
||||
b.score
|
||||
.cmp(&a.score)
|
||||
.then_with(|| a.path.cmp(&b.path))
|
||||
.then_with(|| a.entry.cmp(&b.entry))
|
||||
.then_with(|| a.line.cmp(&b.line))
|
||||
});
|
||||
}
|
||||
|
||||
fn absolute(path: &Path) -> PathBuf {
|
||||
std::path::absolute(path).unwrap_or_else(|_| path.to_path_buf())
|
||||
}
|
||||
|
||||
/// `path` with symlinks resolved, through its parent when it no longer exists.
|
||||
fn canonical(path: &Path) -> PathBuf {
|
||||
let path = absolute(path);
|
||||
if let Ok(path) = path.canonicalize() {
|
||||
return path;
|
||||
}
|
||||
match (
|
||||
path.parent().and_then(|parent| parent.canonicalize().ok()),
|
||||
path.file_name(),
|
||||
) {
|
||||
(Some(parent), Some(name)) => parent.join(name),
|
||||
_ => path,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
//! `hanzo-search`: Dev's search engine from the command line, printing what the
|
||||
//! agent's `grep` tool returns. Exit 0 with hits, 1 without, 2 on error, as `rg`.
|
||||
//!
|
||||
//! ```text
|
||||
//! hanzo-search [-i] [-F] [-n] [-C N] [-g GLOB]... [-l | --count | --json] [--limit N]
|
||||
//! [--no-index | --index-only] [--symbol] PATTERN [PATH...]
|
||||
//! hanzo-search --files GLOB [PATH...]
|
||||
//! hanzo-search --prepare PATH
|
||||
//! hanzo-search --stats [PATH...]
|
||||
//! ```
|
||||
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
|
||||
use clap::Parser;
|
||||
use hanzo_search::Format;
|
||||
use hanzo_search::Index;
|
||||
use hanzo_search::Kind;
|
||||
use hanzo_search::Query;
|
||||
use hanzo_search::Search;
|
||||
use hanzo_search::render;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "hanzo-search",
|
||||
about = "Search text, file names and symbols through Dev's engine"
|
||||
)]
|
||||
struct Args {
|
||||
/// Case-insensitive.
|
||||
#[arg(short = 'i', long = "ignore-case")]
|
||||
ignore_case: bool,
|
||||
/// The pattern is a literal.
|
||||
#[arg(short = 'F', long = "fixed-strings")]
|
||||
fixed: bool,
|
||||
/// Print line numbers.
|
||||
#[arg(short = 'n', long = "line-number")]
|
||||
numbers: bool,
|
||||
/// Lines of context around each hit.
|
||||
#[arg(short = 'C', long = "context", default_value_t = 0)]
|
||||
context: usize,
|
||||
/// Read only files matching this glob; `!GLOB` excludes. Repeatable.
|
||||
#[arg(short = 'g', long = "glob")]
|
||||
glob: Vec<String>,
|
||||
/// Print each file with a hit.
|
||||
#[arg(short = 'l', long = "files-with-matches")]
|
||||
files_only: bool,
|
||||
/// Print hits per file.
|
||||
#[arg(short = 'c', long = "count")]
|
||||
count: bool,
|
||||
/// One JSON object per hit.
|
||||
#[arg(long)]
|
||||
json: bool,
|
||||
/// The most hits, best first.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
/// Scan the files; never read an index.
|
||||
#[arg(long = "no-index", conflicts_with = "index_only")]
|
||||
no_index: bool,
|
||||
/// Answer from a server or an on-disk index only; fail where there is none.
|
||||
#[arg(long = "index-only")]
|
||||
index_only: bool,
|
||||
/// PATTERN is an identifier: its definitions, then its references.
|
||||
#[arg(long)]
|
||||
symbol: bool,
|
||||
/// List files whose path matches GLOB instead of searching contents.
|
||||
#[arg(long, value_name = "GLOB", conflicts_with = "symbol")]
|
||||
files: Option<String>,
|
||||
/// Build the index for PATH (and nothing else).
|
||||
#[arg(long, value_name = "PATH", conflicts_with_all = ["files", "stats"])]
|
||||
prepare: Option<PathBuf>,
|
||||
/// Print the engine's state for the given paths.
|
||||
#[arg(long)]
|
||||
stats: bool,
|
||||
/// PATTERN then PATHs; only PATHs with --files or --stats.
|
||||
args: Vec<String>,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args = Args::parse();
|
||||
match run(args) {
|
||||
Ok(true) => ExitCode::from(0),
|
||||
Ok(false) => ExitCode::from(1),
|
||||
Err(error) => {
|
||||
eprintln!("hanzo-search: {error}");
|
||||
ExitCode::from(2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run(args: Args) -> Result<bool, Box<dyn std::error::Error>> {
|
||||
let search = Search::new();
|
||||
let cwd = std::env::current_dir()?;
|
||||
if let Some(path) = &args.prepare {
|
||||
let root = search.prepare(path)?;
|
||||
emit(&stats(&[root], args.json))?;
|
||||
return Ok(true);
|
||||
}
|
||||
if args.stats {
|
||||
let paths = if args.args.is_empty() {
|
||||
vec![cwd]
|
||||
} else {
|
||||
args.args.iter().map(|path| cwd.join(path)).collect()
|
||||
};
|
||||
let mut roots = Vec::new();
|
||||
for path in &paths {
|
||||
for root in search.status_of(path).roots {
|
||||
if !roots.contains(&root) {
|
||||
roots.push(root);
|
||||
}
|
||||
}
|
||||
}
|
||||
emit(&stats(&roots, args.json))?;
|
||||
return Ok(!roots.is_empty());
|
||||
}
|
||||
let (kind, pattern, paths) = match args.files {
|
||||
Some(glob) => (Kind::Name, glob, args.args),
|
||||
None => {
|
||||
let mut rest = args.args.into_iter();
|
||||
let pattern = rest.next().ok_or("a PATTERN is required")?;
|
||||
let kind = if args.symbol {
|
||||
Kind::Symbol
|
||||
} else {
|
||||
Kind::Text
|
||||
};
|
||||
(kind, pattern, rest.collect())
|
||||
}
|
||||
};
|
||||
let query = Query {
|
||||
kind,
|
||||
pattern,
|
||||
root: cwd,
|
||||
paths: paths.into_iter().map(PathBuf::from).collect(),
|
||||
glob: args.glob,
|
||||
fixed: args.fixed,
|
||||
ignore_case: args.ignore_case,
|
||||
context: args.context,
|
||||
limit: args.limit,
|
||||
index: if args.no_index {
|
||||
Index::Skip
|
||||
} else if args.index_only {
|
||||
Index::Only
|
||||
} else {
|
||||
Index::Auto
|
||||
},
|
||||
};
|
||||
let hits = search.query(&query)?;
|
||||
let format = if args.json {
|
||||
Format::Json
|
||||
} else if args.files_only {
|
||||
Format::Files
|
||||
} else if args.count {
|
||||
Format::Count
|
||||
} else if args.numbers {
|
||||
Format::Lines
|
||||
} else {
|
||||
Format::Bare
|
||||
};
|
||||
emit(&render(&hits, format))?;
|
||||
Ok(!hits.is_empty())
|
||||
}
|
||||
|
||||
/// Writes `text` to stdout; a reader that hung up is not an error.
|
||||
fn emit(text: &str) -> std::io::Result<()> {
|
||||
let mut stdout = std::io::stdout().lock();
|
||||
match stdout
|
||||
.write_all(text.as_bytes())
|
||||
.and_then(|()| stdout.flush())
|
||||
{
|
||||
Err(error) if error.kind() == std::io::ErrorKind::BrokenPipe => Ok(()),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
/// One line per root: `path files=N bytes=N ready=yes|no server=PID|none lag=Ns|none`.
|
||||
fn stats(roots: &[hanzo_search::Root], json: bool) -> String {
|
||||
let mut out = String::new();
|
||||
for root in roots {
|
||||
if json {
|
||||
if let Ok(line) = serde_json::to_string(root) {
|
||||
out.push_str(&line);
|
||||
out.push('\n');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
out.push_str(&format!(
|
||||
"{} files={} bytes={} ready={} server={} lag={}\n",
|
||||
root.path.display(),
|
||||
root.files,
|
||||
root.bytes,
|
||||
if root.ready { "yes" } else { "no" },
|
||||
root.server
|
||||
.map_or("none".to_string(), |pid| pid.to_string()),
|
||||
root.lag
|
||||
.map_or("none".to_string(), |lag| format!("{}s", lag.as_secs())),
|
||||
));
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
//! Hits as text, the way `rg -n --no-heading` prints them, or as JSON lines.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt::Write;
|
||||
|
||||
use crate::Hit;
|
||||
|
||||
/// How [`render`] prints hits.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum Format {
|
||||
/// `path:line:text`, context as `path-line-text`, `--` between groups.
|
||||
#[default]
|
||||
Lines,
|
||||
/// `path:text`, without line numbers.
|
||||
Bare,
|
||||
/// Each file with a hit, once, in rank order.
|
||||
Files,
|
||||
/// `path:N`, the hits in each file, in rank order.
|
||||
Count,
|
||||
/// One JSON object per hit.
|
||||
Json,
|
||||
}
|
||||
|
||||
/// The printed form of a hit's location before the line: the path, then each
|
||||
/// archive member behind `!`, then the place in the document behind `:`.
|
||||
pub fn locate(hit: &Hit) -> String {
|
||||
let mut out = hit.path.display().to_string();
|
||||
if let Some(entry) = &hit.entry {
|
||||
out.push('!');
|
||||
out.push_str(entry);
|
||||
}
|
||||
if let Some(place) = &hit.place {
|
||||
out.push(':');
|
||||
out.push_str(place);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Prints `hits` in `format`. Every line ends with `\n`.
|
||||
pub fn render(hits: &[Hit], format: Format) -> String {
|
||||
let mut out = String::new();
|
||||
match format {
|
||||
Format::Json => {
|
||||
for hit in hits {
|
||||
if let Ok(line) = serde_json::to_string(hit) {
|
||||
out.push_str(&line);
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
Format::Files | Format::Count => {
|
||||
let mut files: Vec<(String, usize)> = Vec::new();
|
||||
for hit in hits {
|
||||
let file = locate_file(hit);
|
||||
match files.iter_mut().find(|(name, _)| *name == file) {
|
||||
Some((_, count)) => *count += 1,
|
||||
None => files.push((file, 1)),
|
||||
}
|
||||
}
|
||||
for (file, count) in files {
|
||||
if format == Format::Count {
|
||||
let _ = writeln!(out, "{file}:{count}");
|
||||
} else {
|
||||
let _ = writeln!(out, "{file}");
|
||||
}
|
||||
}
|
||||
}
|
||||
Format::Lines | Format::Bare => {
|
||||
// Hits of one location are adjacent in rank order. Each run prints its
|
||||
// lines once, in order, a match as `:` over a context line as `-`; `--`
|
||||
// separates runs and gaps when there is context, as rg does.
|
||||
let context = hits
|
||||
.iter()
|
||||
.any(|hit| !hit.before.is_empty() || !hit.after.is_empty());
|
||||
let mut started = false;
|
||||
let mut i = 0;
|
||||
while i < hits.len() {
|
||||
let at = locate(&hits[i]);
|
||||
let mut j = i;
|
||||
let mut lines: BTreeMap<u64, (bool, &str)> = BTreeMap::new();
|
||||
while j < hits.len() && locate(&hits[j]) == at {
|
||||
let hit = &hits[j];
|
||||
match hit.line {
|
||||
None => {
|
||||
let _ = writeln!(out, "{at}");
|
||||
}
|
||||
Some(line) => {
|
||||
let first = line.saturating_sub(hit.before.len() as u64);
|
||||
for (k, text) in hit.before.iter().enumerate() {
|
||||
lines.entry(first + k as u64).or_insert((false, text));
|
||||
}
|
||||
lines.insert(line, (true, &hit.excerpt));
|
||||
for (k, text) in hit.after.iter().enumerate() {
|
||||
lines.entry(line + 1 + k as u64).or_insert((false, text));
|
||||
}
|
||||
}
|
||||
}
|
||||
j += 1;
|
||||
}
|
||||
let mut previous: Option<u64> = None;
|
||||
for (number, (matched, text)) in lines {
|
||||
if context && started && previous.is_none_or(|p| number > p + 1) {
|
||||
out.push_str("--\n");
|
||||
}
|
||||
started = true;
|
||||
previous = Some(number);
|
||||
let separator = if matched { ':' } else { '-' };
|
||||
if format == Format::Bare {
|
||||
let _ = writeln!(out, "{at}{separator}{text}");
|
||||
} else {
|
||||
let _ = writeln!(out, "{at}{separator}{number}{separator}{text}");
|
||||
}
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The file a hit is in, with its archive members: what `-l` and `-c` print.
|
||||
fn locate_file(hit: &Hit) -> String {
|
||||
let mut out = hit.path.display().to_string();
|
||||
if let Some(entry) = &hit.entry {
|
||||
out.push('!');
|
||||
out.push_str(entry);
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
//! A running tgrep server: found by its `serve.json`, asked over loopback JSON-RPC.
|
||||
//!
|
||||
//! The server is tgrep's own (`search`, `files` and `status` methods, one JSON
|
||||
//! object per line). Its index updates asynchronously from a file watcher, so the
|
||||
//! caller's dirty files are searched from disk and the server's rows for them
|
||||
//! are dropped.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::BTreeSet;
|
||||
use std::io::BufRead;
|
||||
use std::io::BufReader;
|
||||
use std::io::Write;
|
||||
use std::net::SocketAddr;
|
||||
use std::net::TcpStream;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use std::time::SystemTime;
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
use serde_json::Value;
|
||||
use serde_json::json;
|
||||
use tgrep_core::builder;
|
||||
|
||||
use crate::Hit;
|
||||
use crate::Kind;
|
||||
use crate::Mark;
|
||||
use crate::find;
|
||||
use crate::find::Plan;
|
||||
use crate::index;
|
||||
|
||||
/// How long a server has to accept a connection before it counts as absent.
|
||||
const CONNECT: Duration = Duration::from_millis(250);
|
||||
/// How long a query may take.
|
||||
const ANSWER: Duration = Duration::from_secs(120);
|
||||
|
||||
pub(crate) struct Server {
|
||||
pub pid: u32,
|
||||
address: SocketAddr,
|
||||
}
|
||||
|
||||
/// The server serving `root`'s index, when one accepts a connection.
|
||||
pub(crate) fn attach(root: &Path) -> Option<Server> {
|
||||
let info: Value = serde_json::from_slice(
|
||||
&std::fs::read(builder::default_index_dir(root).join("serve.json")).ok()?,
|
||||
)
|
||||
.ok()?;
|
||||
let port = u16::try_from(info["port"].as_u64()?).ok()?;
|
||||
let pid = u32::try_from(info["pid"].as_u64()?).ok()?;
|
||||
let address = SocketAddr::from(([127, 0, 0, 1], port));
|
||||
TcpStream::connect_timeout(&address, CONNECT).ok()?;
|
||||
Some(Server { pid, address })
|
||||
}
|
||||
|
||||
impl Server {
|
||||
fn call(&self, method: &str, params: Value) -> Result<Value, String> {
|
||||
let stream = TcpStream::connect_timeout(&self.address, CONNECT)
|
||||
.map_err(|error| error.to_string())?;
|
||||
stream
|
||||
.set_read_timeout(Some(ANSWER))
|
||||
.map_err(|error| error.to_string())?;
|
||||
let request = json!({"jsonrpc": "2.0", "method": method, "params": params, "id": 1});
|
||||
(&stream)
|
||||
.write_all(format!("{request}\n").as_bytes())
|
||||
.map_err(|error| error.to_string())?;
|
||||
let mut line = String::new();
|
||||
BufReader::new(&stream)
|
||||
.read_line(&mut line)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let mut response: Value = serde_json::from_str(&line).map_err(|error| error.to_string())?;
|
||||
if let Some(error) = response.get("error") {
|
||||
return Err(error["message"]
|
||||
.as_str()
|
||||
.unwrap_or("server error")
|
||||
.to_string());
|
||||
}
|
||||
Ok(response["result"].take())
|
||||
}
|
||||
|
||||
pub fn status(&self) -> Option<Value> {
|
||||
self.call("status", json!({})).ok()
|
||||
}
|
||||
|
||||
/// When the server's last full reconcile began: every write before it is in the index.
|
||||
pub fn reconciled(&self) -> Option<SystemTime> {
|
||||
let status = self.status()?;
|
||||
let finished = status["last_reconcile_at"].as_u64()?;
|
||||
let took = status["last_reconcile_duration_ms"].as_u64().unwrap_or(0);
|
||||
// Second resolution: step back a second so the start is never late.
|
||||
Some(UNIX_EPOCH + Duration::from_secs(finished.saturating_sub(took / 1000 + 1)))
|
||||
}
|
||||
|
||||
/// Answers `plan` for `scope` under `root`, reading `dirty` files from disk.
|
||||
pub fn query(
|
||||
&self,
|
||||
plan: &Plan,
|
||||
root: &Path,
|
||||
scope: &Path,
|
||||
dirty: &BTreeSet<PathBuf>,
|
||||
) -> Result<Vec<Hit>, String> {
|
||||
let prefix = index::indexed(root, scope);
|
||||
let keep = |path: &Path| !dirty.contains(path) && path.is_file();
|
||||
|
||||
let mut hits = Vec::new();
|
||||
if plan.kind == Kind::Name {
|
||||
let result = self.call("files", json!({"hidden": false, "scope": prefix}))?;
|
||||
covered(&result)?;
|
||||
for file in result["files"].as_array().ok_or("no files in reply")? {
|
||||
let file = file.as_str().ok_or("a file that is not a string")?;
|
||||
let path = root.join(file);
|
||||
if let Some(hit) = plan.file(&path, &find::relative(scope, &path))
|
||||
&& keep(&path)
|
||||
{
|
||||
hits.push(hit);
|
||||
}
|
||||
}
|
||||
hits.extend(
|
||||
dirty
|
||||
.iter()
|
||||
.filter(|path| path.is_file())
|
||||
.filter_map(|path| plan.file(path, &find::relative(scope, path))),
|
||||
);
|
||||
return Ok(hits);
|
||||
}
|
||||
|
||||
let result = self.call(
|
||||
"search",
|
||||
json!({
|
||||
"pattern": plan.pattern,
|
||||
"case_insensitive": plan.ignore_case,
|
||||
"before_context": plan.context,
|
||||
"after_context": plan.context,
|
||||
"scope": prefix,
|
||||
"hidden": false,
|
||||
}),
|
||||
)?;
|
||||
covered(&result)?;
|
||||
// Rows by file: every line the server sent, and which of them matched.
|
||||
let mut files: BTreeMap<String, (BTreeMap<u64, String>, Vec<u64>)> = BTreeMap::new();
|
||||
for row in result["matches"].as_array().ok_or("no matches in reply")? {
|
||||
let kind = row["type"].as_str().unwrap_or("");
|
||||
let (Some(file), Some(line), Some(content)) = (
|
||||
row["file"].as_str(),
|
||||
row["line"].as_u64(),
|
||||
row["content"].as_str(),
|
||||
) else {
|
||||
continue;
|
||||
};
|
||||
if kind != "match" && kind != "context" {
|
||||
continue;
|
||||
}
|
||||
let entry = files.entry(file.to_string()).or_default();
|
||||
entry
|
||||
.0
|
||||
.insert(line, content.trim_end_matches(['\n', '\r']).to_string());
|
||||
if kind == "match" {
|
||||
entry.1.push(line);
|
||||
}
|
||||
}
|
||||
for (file, (lines, matched)) in files {
|
||||
let path = root.join(&file);
|
||||
if !plan.admits(&find::relative(scope, &path)) || !keep(&path) {
|
||||
continue;
|
||||
}
|
||||
for line in matched {
|
||||
let excerpt = lines.get(&line).cloned().unwrap_or_default();
|
||||
let run = |step: i64| -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
let mut at = line as i64 + step;
|
||||
while out.len() < plan.context && at > 0 {
|
||||
match lines.get(&(at as u64)) {
|
||||
Some(text) => out.push(text.clone()),
|
||||
None => break,
|
||||
}
|
||||
at += step;
|
||||
}
|
||||
out
|
||||
};
|
||||
let mut before = run(-1);
|
||||
before.reverse();
|
||||
let mark = match &plan.declare {
|
||||
None => Mark::Match,
|
||||
Some(declare) if declare.is_match(&excerpt) => Mark::Definition,
|
||||
Some(_) => Mark::Reference,
|
||||
};
|
||||
hits.push(Hit {
|
||||
path: path.clone(),
|
||||
entry: None,
|
||||
place: None,
|
||||
line: Some(line),
|
||||
excerpt,
|
||||
before,
|
||||
after: run(1),
|
||||
mark,
|
||||
score: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
hits.extend(find::grep(plan, &find::fresh(plan, scope, dirty)));
|
||||
Ok(hits)
|
||||
}
|
||||
}
|
||||
|
||||
/// The server's reply covers hidden-file visibility, as tgrep's own client requires.
|
||||
fn covered(result: &Value) -> Result<(), String> {
|
||||
match result["hidden_complete"].as_bool() {
|
||||
Some(true) => Ok(()),
|
||||
_ => Err("the server's index is not complete".to_string()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,862 @@
|
||||
//! The engine against a naive scan, rg, its own index, a tgrep server, and edits.
|
||||
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used)]
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Child;
|
||||
use std::process::Command;
|
||||
use std::process::Stdio;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use hanzo_search::Format;
|
||||
use hanzo_search::Hit;
|
||||
use hanzo_search::Index;
|
||||
use hanzo_search::Mark;
|
||||
use hanzo_search::Query;
|
||||
use hanzo_search::Resolver;
|
||||
use hanzo_search::Search;
|
||||
use hanzo_search::render;
|
||||
use pretty_assertions::assert_eq;
|
||||
use regex::Regex;
|
||||
|
||||
fn write(root: &Path, rel: &str, text: &str) {
|
||||
let path = root.join(rel);
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(path, text).unwrap();
|
||||
}
|
||||
|
||||
/// A small repository: code, an ignored directory and file, a hidden file, a binary.
|
||||
fn tree() -> tempfile::TempDir {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
std::fs::create_dir(root.join(".git")).unwrap();
|
||||
write(root, ".gitignore", "ignored/\n*.log\n");
|
||||
write(
|
||||
root,
|
||||
"src/main.rs",
|
||||
"fn main() {\n let needle = 1;\n println!(\"{needle}\");\n}\n",
|
||||
);
|
||||
write(
|
||||
root,
|
||||
"src/lib.rs",
|
||||
"pub fn needle() -> u32 {\n 7\n}\n// needle again\n",
|
||||
);
|
||||
write(root, "src/deep/mod.rs", "mod x;\nconst NEEDLE: u8 = 1;\n");
|
||||
write(
|
||||
root,
|
||||
"docs/notes.txt",
|
||||
"a needle in a haystack\nnothing here\r\nneedle at crlf\r\n",
|
||||
);
|
||||
write(root, "ignored/skip.rs", "needle ignored\n");
|
||||
write(root, "debug.log", "needle in a log\n");
|
||||
write(root, ".hidden.rs", "needle hidden\n");
|
||||
std::fs::write(root.join("blob.bin"), b"needle\0binary").unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
/// What rg reports, written out by hand: every visible, non-ignored text file,
|
||||
/// line by line.
|
||||
fn naive(root: &Path, pattern: &str) -> BTreeSet<(String, u64, String)> {
|
||||
let regex = Regex::new(pattern).unwrap();
|
||||
let mut out = BTreeSet::new();
|
||||
let mut stack = vec![root.to_path_buf()];
|
||||
while let Some(dir) = stack.pop() {
|
||||
for entry in std::fs::read_dir(&dir).unwrap() {
|
||||
let path = entry.unwrap().path();
|
||||
let rel = path
|
||||
.strip_prefix(root)
|
||||
.unwrap()
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
let name = path.file_name().unwrap().to_string_lossy().into_owned();
|
||||
if name.starts_with('.') || rel == "ignored" || name.ends_with(".log") {
|
||||
continue;
|
||||
}
|
||||
if path.is_dir() {
|
||||
stack.push(path);
|
||||
continue;
|
||||
}
|
||||
let bytes = std::fs::read(&path).unwrap();
|
||||
if bytes.contains(&0) {
|
||||
continue;
|
||||
}
|
||||
let text = String::from_utf8(bytes).unwrap();
|
||||
for (i, line) in text.lines().enumerate() {
|
||||
if regex.is_match(line) {
|
||||
out.insert((rel.clone(), i as u64 + 1, line.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn lines(hits: &[Hit]) -> BTreeSet<(String, u64, String)> {
|
||||
hits.iter()
|
||||
.map(|hit| {
|
||||
(
|
||||
hit.path.to_string_lossy().into_owned(),
|
||||
hit.line.unwrap(),
|
||||
hit.excerpt.clone(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn names(hits: &[Hit]) -> Vec<String> {
|
||||
hits.iter()
|
||||
.map(|hit| hit.path.to_string_lossy().into_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn text(root: &Path, pattern: &str, index: Index) -> Vec<Hit> {
|
||||
Search::new()
|
||||
.query(&Query {
|
||||
index,
|
||||
..Query::text(root, pattern)
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_scan_finds_what_a_naive_scan_finds() {
|
||||
let dir = tree();
|
||||
for pattern in ["needle", "(?i)NEEDLE", r"fn \w+\(", "crlf$", "^nothing"] {
|
||||
assert_eq!(
|
||||
lines(&text(dir.path(), pattern, Index::Skip)),
|
||||
naive(dir.path(), pattern),
|
||||
"{pattern}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_scan_finds_what_rg_finds() {
|
||||
let Ok(output) = Command::new("rg").arg("--version").output() else {
|
||||
return;
|
||||
};
|
||||
assert!(output.status.success());
|
||||
let dir = tree();
|
||||
// Not `crlf$`: tgrep strips a line's `\r` as `rg --crlf` does, and plain rg does not.
|
||||
for pattern in ["needle", r"fn \w+\(", "^nothing"] {
|
||||
let output = Command::new("rg")
|
||||
.args(["-n", "--no-heading", "--", pattern, "."])
|
||||
.current_dir(dir.path())
|
||||
.output()
|
||||
.unwrap();
|
||||
let rg: BTreeSet<(String, u64, String)> = String::from_utf8(output.stdout)
|
||||
.unwrap()
|
||||
.lines()
|
||||
.map(|line| {
|
||||
let mut parts = line.splitn(3, ':');
|
||||
let path = parts.next().unwrap().trim_start_matches("./").to_string();
|
||||
let number = parts.next().unwrap().parse().unwrap();
|
||||
(path, number, parts.next().unwrap().to_string())
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
lines(&text(dir.path(), pattern, Index::Skip)),
|
||||
rg,
|
||||
"{pattern}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_no_index_auto_scans_and_only_refuses() {
|
||||
let dir = tree();
|
||||
assert_eq!(
|
||||
lines(&text(dir.path(), "needle", Index::Auto)),
|
||||
naive(dir.path(), "needle")
|
||||
);
|
||||
let error = Search::new()
|
||||
.query(&Query {
|
||||
index: Index::Only,
|
||||
..Query::text(dir.path(), "needle")
|
||||
})
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(error, hanzo_search::Error::Unindexed(_)),
|
||||
"{error}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_index_answers_as_the_scan_does() {
|
||||
let dir = tree();
|
||||
let search = Search::new();
|
||||
let root = search.prepare(dir.path()).unwrap();
|
||||
assert!(root.ready);
|
||||
assert_eq!(root.server, None);
|
||||
for pattern in ["needle", r"fn \w+\(", "crlf$", "NEEDLE|haystack", "x"] {
|
||||
let indexed = search
|
||||
.query(&Query {
|
||||
index: Index::Only,
|
||||
..Query::text(dir.path(), pattern)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(lines(&indexed), naive(dir.path(), pattern), "{pattern}");
|
||||
}
|
||||
let status = search.status();
|
||||
assert_eq!(status.roots.len(), 1);
|
||||
assert!(status.roots[0].files > 0 && status.roots[0].bytes > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_edit_is_found_by_the_next_search() {
|
||||
let dir = tree();
|
||||
let search = Search::new();
|
||||
search.prepare(dir.path()).unwrap();
|
||||
let path = dir.path().join("src/lib.rs");
|
||||
let mut source = std::fs::read_to_string(&path).unwrap();
|
||||
source.push_str("// zq_unique_8213\n");
|
||||
std::fs::write(&path, &source).unwrap();
|
||||
search.touched(&path);
|
||||
let hits = search
|
||||
.query(&Query {
|
||||
index: Index::Only,
|
||||
..Query::text(dir.path(), "zq_unique_8213")
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(names(&hits), vec!["src/lib.rs"]);
|
||||
assert_eq!(hits[0].line, Some(5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_write_after_since_is_found_and_an_unnamed_one_waits_for_the_index() {
|
||||
let dir = tree();
|
||||
let search = Search::new();
|
||||
search.prepare(dir.path()).unwrap();
|
||||
// Another process wrote this: an on-disk index has no watcher, so only a scan
|
||||
// sees it until the index is rebuilt.
|
||||
write(dir.path(), "src/new.rs", "mod y_fresh;\n");
|
||||
assert_eq!(text(dir.path(), "y_fresh", Index::Only), Vec::new());
|
||||
assert_eq!(
|
||||
names(&text(dir.path(), "y_fresh", Index::Skip)),
|
||||
vec!["src/new.rs"]
|
||||
);
|
||||
// Dev started a command that may write anything: what changed from then on is
|
||||
// read from disk, a same-size edit within the same second included.
|
||||
search.since(SystemTime::now() - Duration::from_millis(5));
|
||||
write(
|
||||
dir.path(),
|
||||
"src/main.rs",
|
||||
"fn main() { grown_token_31(); }\n",
|
||||
);
|
||||
write(
|
||||
dir.path(),
|
||||
"src/deep/mod.rs",
|
||||
"mod y;\nconst NEEDLE: u8 = 1;\n",
|
||||
);
|
||||
let only = |pattern: &str| {
|
||||
names(
|
||||
&search
|
||||
.query(&Query {
|
||||
index: Index::Only,
|
||||
..Query::text(dir.path(), pattern)
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
};
|
||||
assert_eq!(only("grown_token_31"), vec!["src/main.rs"]);
|
||||
assert_eq!(only(r"mod y\b"), vec!["src/deep/mod.rs"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_deleted_or_renamed_file_is_not_reported_stale() {
|
||||
let dir = tree();
|
||||
let search = Search::new();
|
||||
search.prepare(dir.path()).unwrap();
|
||||
std::fs::remove_file(dir.path().join("src/main.rs")).unwrap();
|
||||
search.touched(&dir.path().join("src/main.rs"));
|
||||
std::fs::rename(
|
||||
dir.path().join("src/lib.rs"),
|
||||
dir.path().join("src/moved.rs"),
|
||||
)
|
||||
.unwrap();
|
||||
search.touched(&dir.path().join("src/lib.rs"));
|
||||
search.touched(&dir.path().join("src/moved.rs"));
|
||||
let hits = search
|
||||
.query(&Query {
|
||||
index: Index::Only,
|
||||
..Query::text(dir.path(), "needle")
|
||||
})
|
||||
.unwrap();
|
||||
let found: BTreeSet<String> = names(&hits).into_iter().collect();
|
||||
assert!(!found.contains("src/main.rs"), "{found:?}");
|
||||
assert!(!found.contains("src/lib.rs"), "{found:?}");
|
||||
assert!(found.contains("src/moved.rs"), "{found:?}");
|
||||
assert_eq!(lines(&hits), naive(dir.path(), "needle"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_root_outside_the_query_root_answers_for_its_paths() {
|
||||
let here = tree();
|
||||
let there = tree();
|
||||
let search = Search::new();
|
||||
search.prepare(there.path()).unwrap();
|
||||
write(there.path(), "src/far.rs", "far_token_55\n");
|
||||
search.touched(&there.path().join("src/far.rs"));
|
||||
let hits = search
|
||||
.query(&Query {
|
||||
paths: vec![there.path().join("src")],
|
||||
index: Index::Only,
|
||||
..Query::text(here.path(), "far_token_55")
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(hits.len(), 1);
|
||||
assert_eq!(
|
||||
hits[0].path,
|
||||
there.path().canonicalize().unwrap().join("src/far.rs")
|
||||
);
|
||||
let status = search.status();
|
||||
assert_eq!(
|
||||
status
|
||||
.roots
|
||||
.iter()
|
||||
.map(|root| root.path.clone())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![there.path().canonicalize().unwrap()]
|
||||
);
|
||||
assert_eq!(status.touched, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn names_match_by_file_name_or_path_and_stay_fresh() {
|
||||
let dir = tree();
|
||||
let search = Search::new();
|
||||
let name = |glob: &str, index: Index| {
|
||||
names(
|
||||
&search
|
||||
.query(&Query {
|
||||
index,
|
||||
..Query::name(dir.path(), glob)
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
};
|
||||
assert_eq!(
|
||||
name("*.rs", Index::Skip),
|
||||
vec!["src/lib.rs", "src/main.rs", "src/deep/mod.rs"]
|
||||
);
|
||||
assert_eq!(
|
||||
name("src/*.rs", Index::Skip),
|
||||
vec!["src/lib.rs", "src/main.rs"]
|
||||
);
|
||||
assert_eq!(name("notes.*", Index::Skip), vec!["docs/notes.txt"]);
|
||||
assert_eq!(name("*.bin", Index::Skip), vec!["blob.bin"]);
|
||||
search.prepare(dir.path()).unwrap();
|
||||
write(dir.path(), "src/added.rs", "\n");
|
||||
search.touched(&dir.path().join("src/added.rs"));
|
||||
std::fs::remove_file(dir.path().join("src/main.rs")).unwrap();
|
||||
assert_eq!(
|
||||
name("src/*.rs", Index::Only),
|
||||
vec!["src/added.rs", "src/lib.rs"]
|
||||
);
|
||||
}
|
||||
|
||||
struct Drop1;
|
||||
|
||||
impl Resolver for Drop1 {
|
||||
fn resolve(&self, _symbol: &str, hits: Vec<Hit>) -> Vec<Hit> {
|
||||
hits.into_iter()
|
||||
.filter(|hit| !hit.excerpt.contains("not really"))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_symbol_ranks_its_definitions_first_and_a_resolver_confirms() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
write(
|
||||
dir.path(),
|
||||
"a_use.rs",
|
||||
"widget();\nwidget(); // twice\nwidgets();\n",
|
||||
);
|
||||
write(dir.path(), "b_def.rs", "pub fn widget() {}\n");
|
||||
write(
|
||||
dir.path(),
|
||||
"c_py.py",
|
||||
"def widget(x):\n return widget # not really\n",
|
||||
);
|
||||
let query = Query::symbol(dir.path(), "widget");
|
||||
let hits = Search::new().query(&query).unwrap();
|
||||
let got: Vec<(String, u64, Mark)> = hits
|
||||
.iter()
|
||||
.map(|hit| {
|
||||
(
|
||||
hit.path.to_string_lossy().into_owned(),
|
||||
hit.line.unwrap(),
|
||||
hit.mark,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
got,
|
||||
vec![
|
||||
("c_py.py".to_string(), 1, Mark::Definition),
|
||||
("c_py.py".to_string(), 2, Mark::Reference),
|
||||
("b_def.rs".to_string(), 1, Mark::Definition),
|
||||
("a_use.rs".to_string(), 1, Mark::Reference),
|
||||
("a_use.rs".to_string(), 2, Mark::Reference),
|
||||
]
|
||||
);
|
||||
let resolved = Search::new()
|
||||
.with_resolver(Arc::new(Drop1))
|
||||
.query(&query)
|
||||
.unwrap();
|
||||
assert_eq!(resolved.len(), 4);
|
||||
assert!(
|
||||
resolved
|
||||
.iter()
|
||||
.all(|hit| !hit.excerpt.contains("not really"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_ranks_files_by_hits_then_path_then_line() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
write(dir.path(), "a.txt", "k\n");
|
||||
write(dir.path(), "b.txt", "k\nk\nk\n");
|
||||
write(dir.path(), "c.txt", "k\n");
|
||||
let hits = Search::new()
|
||||
.query(&Query {
|
||||
limit: Some(4),
|
||||
..Query::text(dir.path(), "k")
|
||||
})
|
||||
.unwrap();
|
||||
let got: Vec<(String, u64, u64)> = hits
|
||||
.iter()
|
||||
.map(|hit| {
|
||||
(
|
||||
hit.path.to_string_lossy().into_owned(),
|
||||
hit.line.unwrap(),
|
||||
hit.score,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
got,
|
||||
vec![
|
||||
("b.txt".to_string(), 1, 3),
|
||||
("b.txt".to_string(), 2, 3),
|
||||
("b.txt".to_string(), 3, 3),
|
||||
("a.txt".to_string(), 1, 1),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_prints_rg_lines_with_merged_context() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
write(
|
||||
dir.path(),
|
||||
"f.txt",
|
||||
"1\n2 hit\n3\n4 hit\n5\n6\n7\n8 hit\n9\n",
|
||||
);
|
||||
let hits = Search::new()
|
||||
.query(&Query {
|
||||
context: 1,
|
||||
..Query::text(dir.path(), "hit")
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
render(&hits, Format::Lines),
|
||||
"f.txt-1-1\nf.txt:2:2 hit\nf.txt-3-3\nf.txt:4:4 hit\nf.txt-5-5\n--\nf.txt-7-7\nf.txt:8:8 hit\nf.txt-9-9\n"
|
||||
);
|
||||
assert_eq!(render(&hits, Format::Count), "f.txt:3\n");
|
||||
assert_eq!(render(&hits, Format::Files), "f.txt\n");
|
||||
let json = render(&hits[..1], Format::Json);
|
||||
let value: serde_json::Value = serde_json::from_str(json.trim()).unwrap();
|
||||
assert_eq!(value["path"], "f.txt");
|
||||
assert_eq!(value["line"], 2);
|
||||
assert_eq!(value["mark"], "match");
|
||||
assert_eq!(value["entry"], serde_json::Value::Null);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_document_location_prints_before_the_line() {
|
||||
let hit = Hit {
|
||||
path: PathBuf::from("docs/a.zip"),
|
||||
entry: Some("inner/b.docx".to_string()),
|
||||
place: Some("page 2".to_string()),
|
||||
line: Some(14),
|
||||
excerpt: "text".to_string(),
|
||||
before: Vec::new(),
|
||||
after: Vec::new(),
|
||||
mark: Mark::Match,
|
||||
score: 1,
|
||||
};
|
||||
assert_eq!(
|
||||
render(&[hit], Format::Lines),
|
||||
"docs/a.zip!inner/b.docx:page 2:14:text\n"
|
||||
);
|
||||
}
|
||||
|
||||
/// A `tgrep serve` from PATH, stopped on drop. It stands in for the per-user
|
||||
/// service; Dev's code never starts it.
|
||||
struct Served(Child);
|
||||
|
||||
impl Drop for Served {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
|
||||
fn serve(root: &Path, search: &Search) -> Option<Served> {
|
||||
let child = Command::new("tgrep")
|
||||
.args(["serve", "--no-watch", "."])
|
||||
.current_dir(root)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.ok()?;
|
||||
let served = Served(child);
|
||||
let deadline = Instant::now() + Duration::from_secs(120);
|
||||
while Instant::now() < deadline {
|
||||
let status = search.status_of(root);
|
||||
if status
|
||||
.roots
|
||||
.iter()
|
||||
.any(|root| root.server.is_some() && root.ready)
|
||||
{
|
||||
return Some(served);
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
panic!("tgrep serve did not become ready");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_running_server_answers_and_its_lag_is_bypassed() {
|
||||
let dir = tree();
|
||||
let search = Search::new();
|
||||
let Some(_served) = serve(dir.path(), &search) else {
|
||||
return;
|
||||
};
|
||||
let status = search.status_of(dir.path());
|
||||
assert_eq!(status.roots.len(), 1);
|
||||
assert!(status.roots[0].server.is_some());
|
||||
assert_eq!(
|
||||
lines(&text(dir.path(), "needle", Index::Only)),
|
||||
naive(dir.path(), "needle")
|
||||
);
|
||||
|
||||
// The server watches nothing: it cannot see this edit until told.
|
||||
let path = dir.path().join("src/lib.rs");
|
||||
let mut source = std::fs::read_to_string(&path).unwrap();
|
||||
source.push_str("// served_token_771\n");
|
||||
std::fs::write(&path, &source).unwrap();
|
||||
let query = Query {
|
||||
index: Index::Only,
|
||||
..Query::text(dir.path(), "served_token_771")
|
||||
};
|
||||
assert_eq!(search.query(&query).unwrap(), Vec::new(), "the server lags");
|
||||
search.touched(&path);
|
||||
assert_eq!(names(&search.query(&query).unwrap()), vec!["src/lib.rs"]);
|
||||
|
||||
// A shell command wrote a file nobody named.
|
||||
search.since(SystemTime::now() - Duration::from_millis(5));
|
||||
write(dir.path(), "src/shell.rs", "served_token_771\n");
|
||||
let found: BTreeSet<String> = names(&search.query(&query).unwrap()).into_iter().collect();
|
||||
assert_eq!(
|
||||
found,
|
||||
BTreeSet::from(["src/lib.rs".to_string(), "src/shell.rs".to_string()])
|
||||
);
|
||||
|
||||
// Deleted under the server: its rows are dropped.
|
||||
std::fs::remove_file(dir.path().join("src/main.rs")).unwrap();
|
||||
let hits = text(dir.path(), "needle", Index::Only);
|
||||
assert!(!names(&hits).contains(&"src/main.rs".to_string()));
|
||||
|
||||
let named = search
|
||||
.query(&Query {
|
||||
index: Index::Only,
|
||||
..Query::name(dir.path(), "src/*.rs")
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(names(&named), vec!["src/lib.rs", "src/shell.rs"]);
|
||||
}
|
||||
|
||||
/// Document and archive fixtures, written by hand: stored zip, tar, gzip with a
|
||||
/// stored deflate block, and a two-page PDF.
|
||||
mod docs {
|
||||
fn crc32(data: &[u8]) -> u32 {
|
||||
let mut crc = 0xFFFF_FFFFu32;
|
||||
for &byte in data {
|
||||
crc ^= u32::from(byte);
|
||||
for _ in 0..8 {
|
||||
crc = if crc & 1 == 1 {
|
||||
(crc >> 1) ^ 0xEDB8_8320
|
||||
} else {
|
||||
crc >> 1
|
||||
};
|
||||
}
|
||||
}
|
||||
!crc
|
||||
}
|
||||
|
||||
pub fn zip(files: &[(&str, &[u8])]) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
let mut central = Vec::new();
|
||||
for (name, data) in files {
|
||||
let offset = out.len() as u32;
|
||||
let crc = crc32(data);
|
||||
let head = |signature: u32, central: bool| {
|
||||
let mut h = Vec::new();
|
||||
h.extend(signature.to_le_bytes());
|
||||
if central {
|
||||
h.extend(20u16.to_le_bytes());
|
||||
}
|
||||
h.extend(20u16.to_le_bytes());
|
||||
h.extend([0u8; 4]); // flags, stored
|
||||
h.extend([0u8; 4]); // time, date
|
||||
h.extend(crc.to_le_bytes());
|
||||
h.extend((data.len() as u32).to_le_bytes());
|
||||
h.extend((data.len() as u32).to_le_bytes());
|
||||
h.extend((name.len() as u16).to_le_bytes());
|
||||
h.extend(0u16.to_le_bytes());
|
||||
if central {
|
||||
h.extend([0u8; 8]); // comment, disk, attributes
|
||||
h.extend([0u8; 4]);
|
||||
h.extend(offset.to_le_bytes());
|
||||
}
|
||||
h.extend(name.as_bytes());
|
||||
h
|
||||
};
|
||||
out.extend(head(0x0403_4b50, false));
|
||||
out.extend(*data);
|
||||
central.extend(head(0x0201_4b50, true));
|
||||
}
|
||||
let start = out.len() as u32;
|
||||
out.extend(¢ral);
|
||||
out.extend(0x0605_4b50u32.to_le_bytes());
|
||||
out.extend([0u8; 4]);
|
||||
out.extend((files.len() as u16).to_le_bytes());
|
||||
out.extend((files.len() as u16).to_le_bytes());
|
||||
out.extend((central.len() as u32).to_le_bytes());
|
||||
out.extend(start.to_le_bytes());
|
||||
out.extend(0u16.to_le_bytes());
|
||||
out
|
||||
}
|
||||
|
||||
pub fn tar(files: &[(&str, &[u8])]) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
for (name, data) in files {
|
||||
let mut header = [0u8; 512];
|
||||
header[..name.len()].copy_from_slice(name.as_bytes());
|
||||
header[100..108].copy_from_slice(b"0000644\0");
|
||||
header[108..116].copy_from_slice(b"0000000\0");
|
||||
header[116..124].copy_from_slice(b"0000000\0");
|
||||
header[124..136].copy_from_slice(format!("{:011o}\0", data.len()).as_bytes());
|
||||
header[136..148].copy_from_slice(b"00000000000\0");
|
||||
header[156] = b'0';
|
||||
header[257..263].copy_from_slice(b"ustar\0");
|
||||
header[263..265].copy_from_slice(b"00");
|
||||
header[148..156].copy_from_slice(b" ");
|
||||
let sum: u32 = header.iter().map(|&b| u32::from(b)).sum();
|
||||
header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes());
|
||||
out.extend(header);
|
||||
out.extend(*data);
|
||||
out.extend(std::iter::repeat_n(0u8, (512 - data.len() % 512) % 512));
|
||||
}
|
||||
out.extend([0u8; 1024]);
|
||||
out
|
||||
}
|
||||
|
||||
pub fn gzip(data: &[u8]) -> Vec<u8> {
|
||||
let mut out = vec![0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 255];
|
||||
let mut chunks = data.chunks(65_535).peekable();
|
||||
if chunks.peek().is_none() {
|
||||
out.extend([1, 0, 0, 0xff, 0xff]);
|
||||
}
|
||||
while let Some(chunk) = chunks.next() {
|
||||
out.push(u8::from(chunks.peek().is_none()));
|
||||
let len = chunk.len() as u16;
|
||||
out.extend(len.to_le_bytes());
|
||||
out.extend((!len).to_le_bytes());
|
||||
out.extend(chunk);
|
||||
}
|
||||
out.extend(crc32(data).to_le_bytes());
|
||||
out.extend((data.len() as u32).to_le_bytes());
|
||||
out
|
||||
}
|
||||
|
||||
pub fn docx(paragraphs: &[&str]) -> Vec<u8> {
|
||||
let body: String = paragraphs
|
||||
.iter()
|
||||
.map(|text| format!("<w:p><w:r><w:t>{text}</w:t></w:r></w:p>"))
|
||||
.collect();
|
||||
let document = format!(
|
||||
"<?xml version=\"1.0\"?><w:document xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"><w:body>{body}</w:body></w:document>"
|
||||
);
|
||||
zip(&[
|
||||
("[Content_Types].xml", b"<?xml version=\"1.0\"?><Types xmlns=\"http://schemas.openxmlformats.org/package/2006/content-types\"/>"),
|
||||
("word/document.xml", document.as_bytes()),
|
||||
])
|
||||
}
|
||||
|
||||
pub fn pptx(slides: &[&str]) -> Vec<u8> {
|
||||
let parts: Vec<(String, String)> = slides
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, text)| {
|
||||
(
|
||||
format!("ppt/slides/slide{}.xml", i + 1),
|
||||
format!("<?xml version=\"1.0\"?><p:sld xmlns:p=\"http://schemas.openxmlformats.org/presentationml/2006/main\" xmlns:a=\"http://schemas.openxmlformats.org/drawingml/2006/main\"><p:cSld><p:spTree><p:sp><p:txBody><a:p><a:r><a:t>{text}</a:t></a:r></a:p></p:txBody></p:sp></p:spTree></p:cSld></p:sld>"),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let files: Vec<(&str, &[u8])> = parts
|
||||
.iter()
|
||||
.map(|(name, xml)| (name.as_str(), xml.as_bytes()))
|
||||
.collect();
|
||||
zip(&files)
|
||||
}
|
||||
|
||||
pub fn xlsx(sheet: &str, cell: &str, text: &str) -> Vec<u8> {
|
||||
let workbook = format!(
|
||||
"<?xml version=\"1.0\"?><workbook xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\" xmlns:r=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships\"><sheets><sheet name=\"{sheet}\" sheetId=\"1\" r:id=\"rId1\"/></sheets></workbook>"
|
||||
);
|
||||
let rels = "<?xml version=\"1.0\"?><Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\"><Relationship Id=\"rId1\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet\" Target=\"worksheets/sheet1.xml\"/></Relationships>";
|
||||
let data = format!(
|
||||
"<?xml version=\"1.0\"?><worksheet xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\"><sheetData><row r=\"{row}\"><c r=\"{cell}\" t=\"inlineStr\"><is><t>{text}</t></is></c></row></sheetData></worksheet>",
|
||||
row = cell.trim_start_matches(char::is_alphabetic)
|
||||
);
|
||||
zip(&[
|
||||
("xl/workbook.xml", workbook.as_bytes()),
|
||||
("xl/_rels/workbook.xml.rels", rels.as_bytes()),
|
||||
("xl/worksheets/sheet1.xml", data.as_bytes()),
|
||||
])
|
||||
}
|
||||
|
||||
pub fn pdf(pages: &[&str]) -> Vec<u8> {
|
||||
let mut objects: Vec<String> = Vec::new();
|
||||
let kids: Vec<String> = (0..pages.len())
|
||||
.map(|i| format!("{} 0 R", 4 + 2 * i))
|
||||
.collect();
|
||||
objects.push("<< /Type /Catalog /Pages 2 0 R >>".to_string());
|
||||
objects.push(format!(
|
||||
"<< /Type /Pages /Kids [{}] /Count {} >>",
|
||||
kids.join(" "),
|
||||
pages.len()
|
||||
));
|
||||
objects.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string());
|
||||
for (i, text) in pages.iter().enumerate() {
|
||||
let stream = format!("BT /F1 12 Tf 72 720 Td ({text}) Tj ET");
|
||||
objects.push(format!(
|
||||
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents {} 0 R >>",
|
||||
5 + 2 * i
|
||||
));
|
||||
objects.push(format!(
|
||||
"<< /Length {} >>\nstream\n{stream}\nendstream",
|
||||
stream.len()
|
||||
));
|
||||
}
|
||||
let mut out = b"%PDF-1.4\n".to_vec();
|
||||
let mut offsets = Vec::new();
|
||||
for (i, object) in objects.iter().enumerate() {
|
||||
offsets.push(out.len());
|
||||
out.extend(format!("{} 0 obj\n{object}\nendobj\n", i + 1).as_bytes());
|
||||
}
|
||||
let xref = out.len();
|
||||
out.extend(format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes());
|
||||
for offset in offsets {
|
||||
out.extend(format!("{offset:010} 00000 n \n").as_bytes());
|
||||
}
|
||||
out.extend(
|
||||
format!(
|
||||
"trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",
|
||||
objects.len() + 1
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// What a document or archive hit must say: the file, the members inside it, and the
|
||||
/// place in the document.
|
||||
fn located(hits: &[Hit]) -> BTreeSet<(String, Option<String>, Option<String>)> {
|
||||
hits.iter()
|
||||
.map(|hit| {
|
||||
(
|
||||
hit.path.to_string_lossy().into_owned(),
|
||||
hit.entry.clone(),
|
||||
hit.place.clone(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the document adapters of the tgrep fork (hanzoai/tgrep) in tgrep-core"]
|
||||
fn documents_and_archives_report_where_the_hit_is() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
std::fs::write(
|
||||
root.join("a.pdf"),
|
||||
docs::pdf(&["first page", "second doctoken"]),
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(root.join("b.docx"), docs::docx(&["intro", "body doctoken"])).unwrap();
|
||||
std::fs::write(
|
||||
root.join("c.pptx"),
|
||||
docs::pptx(&["title", "slide doctoken"]),
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
root.join("d.xlsx"),
|
||||
docs::xlsx("Data", "B4", "cell doctoken"),
|
||||
)
|
||||
.unwrap();
|
||||
let docx = docs::docx(&["zipped doctoken"]);
|
||||
std::fs::write(root.join("e.zip"), docs::zip(&[("inner/b.docx", &docx)])).unwrap();
|
||||
let zipped = docs::zip(&[("deep.txt", b"nested doctoken\n")]);
|
||||
let tar = docs::tar(&[
|
||||
("x/inner.zip", &zipped),
|
||||
("plain.txt", b"tarred doctoken\n"),
|
||||
]);
|
||||
std::fs::write(root.join("f.tar.gz"), docs::gzip(&tar)).unwrap();
|
||||
|
||||
let search = Search::new();
|
||||
let expect = BTreeSet::from([
|
||||
("a.pdf".to_string(), None, Some("page 2".to_string())),
|
||||
("b.docx".to_string(), None, Some("paragraph 2".to_string())),
|
||||
("c.pptx".to_string(), None, Some("slide 2".to_string())),
|
||||
(
|
||||
"d.xlsx".to_string(),
|
||||
None,
|
||||
Some("sheet Data!B4".to_string()),
|
||||
),
|
||||
(
|
||||
"e.zip".to_string(),
|
||||
Some("inner/b.docx".to_string()),
|
||||
Some("paragraph 1".to_string()),
|
||||
),
|
||||
(
|
||||
"f.tar.gz".to_string(),
|
||||
Some("x/inner.zip!deep.txt".to_string()),
|
||||
None,
|
||||
),
|
||||
("f.tar.gz".to_string(), Some("plain.txt".to_string()), None),
|
||||
]);
|
||||
for index in [Index::Skip, Index::Auto] {
|
||||
if index == Index::Auto {
|
||||
search.prepare(root).unwrap();
|
||||
}
|
||||
let hits = search
|
||||
.query(&Query {
|
||||
index,
|
||||
..Query::text(root, "doctoken")
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(located(&hits), expect, "{index:?}");
|
||||
}
|
||||
}
|
||||
@@ -634,5 +634,35 @@
|
||||
"old": " let available_models = session\n .services\n .models_manager\n .list_models(\n codex_models_manager::manager::RefreshStrategy::Offline,\n turn.config.http_client_factory(),\n )\n .await;\n let default_review_model_id = turn.provider.approval_review_preferred_model();\n",
|
||||
"new": " let mut available_models = session\n .services\n .models_manager\n .list_models(\n codex_models_manager::manager::RefreshStrategy::Offline,\n turn.config.http_client_factory(),\n )\n .await;\n // Hanzo: off OpenAI a provider serves the models its catalog lists. The bundled\n // OpenAI entries ride along unlisted, for lookups, and no other provider serves them.\n if !turn.provider.info().is_openai() {\n available_models.retain(|preset| preset.show_in_picker);\n }\n let default_review_model_id = turn.provider.approval_review_preferred_model();\n",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"file": "app-server/Cargo.toml",
|
||||
"old": "codex-memories-extension = { workspace = true }",
|
||||
"new": "codex-memories-extension = { workspace = true }\nhanzo-grep = { path = \"../../../../crates/hanzo-grep\" }",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"file": "app-server/src/extensions.rs",
|
||||
"old": " codex_memories_extension::install(&mut builder, codex_otel::global());",
|
||||
"new": " // Hanzo: the grep tool, Dev's one search path (crates/hanzo-search).\n hanzo_grep::install(&mut builder);\n codex_memories_extension::install(&mut builder, codex_otel::global());",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"file": "protocol/src/prompts/base_instructions/default.md",
|
||||
"old": "- When searching for text or files, prefer using `rg` or `rg --files` respectively because `rg` is much faster than alternatives like `grep`. (If the `rg` command is not found, then use alternatives.)",
|
||||
"new": "- When searching for text or files, use the `grep` tool: it searches file contents (`kind: \"text\"`), file names (`kind: \"name\"`) and an identifier's definitions and references (`kind: \"symbol\"`) through an index that reflects your edits. Run `rg` in the shell only if `grep` returns an error.",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"file": "models-manager/prompt.md",
|
||||
"old": "- When searching for text or files, prefer using `rg` or `rg --files` respectively because `rg` is much faster than alternatives like `grep`. (If the `rg` command is not found, then use alternatives.)",
|
||||
"new": "- When searching for text or files, use the `grep` tool: it searches file contents (`kind: \"text\"`), file names (`kind: \"name\"`) and an identifier's definitions and references (`kind: \"symbol\"`) through an index that reflects your edits. Run `rg` in the shell only if `grep` returns an error.",
|
||||
"count": 1
|
||||
},
|
||||
{
|
||||
"file": "core/templates/agents/orchestrator.md",
|
||||
"old": "- Unless you are otherwise instructed, prefer using `rg` or `rg --files` respectively when searching because `rg` is much faster than alternatives like `grep`. If the `rg` command is not found, then use alternatives.",
|
||||
"new": "- Unless you are otherwise instructed, use the `grep` tool when searching for text or files: it is indexed and reflects your edits. Run `rg` in the shell only if `grep` returns an error.",
|
||||
"count": 1
|
||||
}
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user