The description comes from the binary, not from a file a macro left behind

Writing the manifest fragments during macro expansion had a failure nobody
would see: rustc reuses a cached expansion, so a macro that did not run left
its file saying what the source used to say. A rebuild after deleting one
fragment recompiled the crate and did not restore it — measured — which means a
service could publish a document that quietly disagreed with its own code, the
one failure this whole design exists to prevent.

So the description is compiled IN. `#[derive(zip::Wire)]` gives a type
`stated()`, its manifest entry as a `&'static str`, and `reach()`, which claims
its entry and then asks the types of its fields — the graph is walked by the
types themselves, because a macro sees only the item it is on. `#[zip::ops]`
joins the ops half to whatever the ops reach and hands back `manifest()`. The
service prints it on `--manifest`, so what is projected is what this build
declared and there is nothing to go stale.

That leaves one shape for a manifest instead of two, so zipc reads a document
and no longer merges a directory of fragments; what it refused about a
directory it now refuses about the document, in Manifest.Check beside the
version rule. The example's build script tolerates the documents not existing
yet, which is the state of the one build that makes them.
This commit is contained in:
2026-09-15 11:41:57 -07:00
parent fb2b7f6d5c
commit f728e8fdce
13 changed files with 243 additions and 188 deletions
+8 -122
View File
@@ -22,8 +22,8 @@
//
// zipc [-out DIR] [-package NAME] MANIFEST
//
// MANIFEST is a .json file, or a directory holding ops/*.json and types/*.json
// fragments.
// MANIFEST is the document a front end wrote: `App.Manifest()` in Go, what
// `Service::manifest()` prints in Rust, what the libclang pass emits in C++.
package main
import (
@@ -55,8 +55,7 @@ func main() {
func usage() {
fmt.Fprintln(os.Stderr, "usage: zipc [-out DIR] [-package NAME] MANIFEST")
fmt.Fprintln(os.Stderr)
fmt.Fprintln(os.Stderr, "MANIFEST is a manifest .json file, or a directory of ops/*.json")
fmt.Fprintln(os.Stderr, "and types/*.json fragments written by a front end.")
fmt.Fprintln(os.Stderr, "MANIFEST is the description a front end wrote of one app.")
}
func run(in, out, pkg string) error {
@@ -64,15 +63,10 @@ func run(in, out, pkg string) error {
if err != nil {
return err
}
if err := m.Check(); err != nil {
return fmt.Errorf("%s: %w", in, err)
}
m.Settle()
if m.Manifest != zip.ManifestVersion {
return fmt.Errorf("%s: manifest version %d, this zipc reads %d", in, m.Manifest, zip.ManifestVersion)
}
if len(m.Ops) == 0 {
// An empty registry projects an empty document, which is worse than no
// document: it publishes that the service has nothing to offer.
return fmt.Errorf("%s: no ops; there is nothing to project", in)
}
if pkg == "" {
pkg = m.App
}
@@ -106,123 +100,15 @@ func run(in, out, pkg string) error {
return nil
}
// read is one manifest, from a file or from the fragments a compiler wrote.
// read is one manifest.
func read(at string) (zip.Manifest, error) {
info, err := os.Stat(at)
if err != nil {
return zip.Manifest{}, err
}
if !info.IsDir() {
return one(at)
}
return assemble(at)
}
func one(at string) (zip.Manifest, error) {
b, err := os.ReadFile(at)
if err != nil {
return zip.Manifest{}, err
}
var m zip.Manifest
if err := json.Unmarshal(b, &m); err != nil {
return zip.Manifest{}, fmt.Errorf("%s: %w", at, err)
}
return m, nil
}
// assemble is one manifest out of the fragments a front end wrote as it
// compiled: one per impl block, one per described type.
//
// It is a merge and not a build. Nothing here decides anything about an op or a
// type — a fragment is already the finished description — so the order the
// files arrive in changes nothing, which is what lets a compiler write them as
// it reaches them.
func assemble(dir string) (zip.Manifest, error) {
m := zip.Manifest{Manifest: zip.ManifestVersion}
seen := map[string]bool{}
ops, err := filepath.Glob(filepath.Join(dir, "ops", "*.json"))
if err != nil {
if err := load(at, &m); err != nil {
return m, err
}
sort.Strings(ops)
for _, at := range ops {
var part zip.Manifest
if err := load(at, &part); err != nil {
return m, err
}
if part.App != "" {
m.App = part.App
}
if part.Title != "" {
m.Title = part.Title
}
if part.Version != "" {
m.Version = part.Version
}
if part.Description != "" {
m.Description = part.Description
}
m.Ops = append(m.Ops, part.Ops...)
}
types, err := filepath.Glob(filepath.Join(dir, "types", "*.json"))
if err != nil {
return m, err
}
sort.Strings(types)
for _, at := range types {
var td zip.TypeDesc
if err := load(at, &td); err != nil {
return m, err
}
if seen[td.ID] {
return m, fmt.Errorf("%s: two types described under %q; a name is how a field refers to a type, so it can only mean one", at, td.ID)
}
seen[td.ID] = true
m.Types = append(m.Types, td)
}
sort.Slice(m.Types, func(i, j int) bool { return m.Types[i].ID < m.Types[j].ID })
// A field naming a type nobody described is a dangling reference, and it is
// found HERE rather than in the document, where it would have become a
// schema of `{}` that every generated client reads as "any value".
for _, td := range m.Types {
for _, f := range td.Fields {
if err := resolves(seen, td.ID, f.Name, f.Type); err != nil {
return m, err
}
}
if td.Elem != nil {
if err := resolves(seen, td.ID, "element", *td.Elem); err != nil {
return m, err
}
}
}
for _, op := range m.Ops {
for what, id := range map[string]string{"input": op.In, "output": op.Out} {
if id != "" && !seen[id] {
return m, fmt.Errorf("op %s names %s %q, which nothing described", op.ID, what, id)
}
}
}
return m, nil
}
func resolves(seen map[string]bool, in, field string, r zip.TypeRef) error {
switch {
case r.Ref != "":
if !seen[r.Ref] {
return fmt.Errorf("%s.%s is %q, which nothing described", in, field, r.Ref)
}
case r.List != nil:
return resolves(seen, in, field, *r.List)
case r.Map != nil:
return resolves(seen, in, field, *r.Map)
}
return nil
}
func load(at string, into any) error {
b, err := os.ReadFile(at)
if err != nil {
+57
View File
@@ -216,6 +216,63 @@ func (m *Manifest) Settle() {
}
}
// Check refuses a description a projection would silently mis-read.
//
// A field naming a type nobody described is a dangling reference, and it is
// caught HERE rather than in the document, where it would have become a schema
// of `{}` that every generated client reads as "any value".
func (m Manifest) Check() error {
if m.Manifest != ManifestVersion {
return fmt.Errorf("manifest version %d, this zip reads %d", m.Manifest, ManifestVersion)
}
if len(m.Ops) == 0 {
// An empty registry projects an empty document, which is worse than no
// document: it publishes that the service has nothing to offer.
return fmt.Errorf("no ops; there is nothing to project")
}
known := make(map[string]bool, len(m.Types))
for _, td := range m.Types {
if known[td.ID] {
return fmt.Errorf("two types described under %q; a name is how a field refers to a type, so it can only mean one", td.ID)
}
known[td.ID] = true
}
for _, td := range m.Types {
for _, f := range td.Fields {
if err := resolves(known, td.ID+"."+f.Name, f.Type); err != nil {
return err
}
}
if td.Elem != nil {
if err := resolves(known, td.ID+" element", *td.Elem); err != nil {
return err
}
}
}
for _, op := range m.Ops {
for what, id := range map[string]string{"input": op.In, "output": op.Out} {
if id != "" && !known[id] {
return fmt.Errorf("op %s names %s %q, which nothing described", op.ID, what, id)
}
}
}
return nil
}
func resolves(known map[string]bool, at string, r TypeRef) error {
switch {
case r.Ref != "":
if !known[r.Ref] {
return fmt.Errorf("%s is %q, which nothing described", at, r.Ref)
}
case r.List != nil:
return resolves(known, at, *r.List)
case r.Map != nil:
return resolves(known, at, *r.Map)
}
return nil
}
// Manifest describes this app the way any front end would, so the projections
// read the same document whichever language declared the ops.
func (a *App) Manifest() Manifest {
+15 -11
View File
@@ -1,27 +1,30 @@
# The build, in the order the facts become available.
#
# A proc-macro writes its fragment of the manifest as the crate compiles, so the
# description exists only after a compile — which is why projecting it is a
# second step and not a build script. What zipc writes is checked in beside the
# source, the way a generated file is anywhere else, so a fresh clone builds
# without either tool.
# The macro compiles what it read into the crate, and the binary prints it. So
# the manifest describes THIS build — there is no file beside the source to fall
# out of step, and no way to project a service that no longer exists.
#
# It is two steps and not a build script because the description exists only
# after a compile. What zipc writes is checked in beside the source, the way a
# generated file is anywhere else, so a fresh clone builds without either tool.
#
# make gen compile, project, and compile again against the projections
# make run serve on the default addresses
# make check the doors answer, and the wire is ZAP
# make wire the ZAP accessors for these ops, for a client that wants them
# gen names a directory as well as a step, and the step is the one meant.
.PHONY: gen run check wire
ZIPC ?= zipc
ZAPGEN ?= zapgen
ROOT := ../..
OUT ?= wire
gen:
cd $(ROOT) && cargo run -q -p info -- --manifest > examples/info/gen/manifest.json
$(ZIPC) -out gen -package info gen/manifest.json
cd $(ROOT) && cargo build -p info
$(ZIPC) -out gen -package info target/zip
cd $(ROOT) && cargo build -p info
wire:
$(ZAPGEN) -lang rust -out $(OUT) gen/info.zap
run:
cd $(ROOT) && cargo run -p info
@@ -29,4 +32,5 @@ run:
check:
cd $(ROOT) && cargo test -p info
OUT ?= wire
wire:
$(ZAPGEN) -lang rust -out $(OUT) gen/info.zap
+26
View File
@@ -0,0 +1,26 @@
// SPDX-License-Identifier: BSD-3-Clause-Eco
// Copyright (C) 2026, Lux Industries Inc. All rights reserved.
// See the file LICENSE for licensing terms.
//! The documents this service publishes, if they have been projected yet.
//!
//! They are checked in, so a clone builds with them in hand. But they are made
//! FROM this crate — zipc reads the manifest the compiled binary prints — so
//! the first build of a new service necessarily happens before they exist. This
//! says so instead of failing: no documents yet, and `make gen` makes them.
use std::path::PathBuf;
fn main() {
println!("cargo:rerun-if-changed=gen");
let out = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR"));
let mut body = String::new();
for (name, file) in [("OPENAPI", "gen/openapi.json"), ("MCP", "gen/mcp.json")] {
let text = std::fs::read_to_string(file).unwrap_or_default();
body.push_str(&format!(
"pub const {name}: &str = {:?};\n",
if text.is_empty() { "" } else { text.as_str() }
));
}
std::fs::write(out.join("documents.rs"), body).expect("write documents.rs");
}
File diff suppressed because one or more lines are too long
+10 -6
View File
@@ -7,10 +7,10 @@
pub mod ops;
pub mod types;
/// The projections, as zipc wrote them from what the compiler read. A service
/// publishes the document it implements.
pub const OPENAPI: &str = include_str!("../gen/openapi.json");
pub const MCP: &str = include_str!("../gen/mcp.json");
// The projections, as zipc wrote them from what this crate declared. A service
// publishes the document it implements. Empty on the build that MAKES them,
// which is the one build where they cannot exist yet.
include!(concat!(env!("OUT_DIR"), "/documents.rs"));
/// service is this node's info app, with its documents on it.
pub fn service(release: &str, network: u32) -> zip::App {
@@ -19,7 +19,11 @@ pub fn service(release: &str, network: u32) -> zip::App {
network,
}
.ops();
app.serve("/openapi.json", "application/json", OPENAPI);
app.serve("/mcp.json", "application/json", MCP);
if !OPENAPI.is_empty() {
app.serve("/openapi.json", "application/json", OPENAPI);
}
if !MCP.is_empty() {
app.serve("/mcp.json", "application/json", MCP);
}
app
}
+6
View File
@@ -23,6 +23,12 @@ fn main() {
zap = args.get(i + 1).cloned();
i += 2;
}
// What this service declared, for zipc to project. It comes from
// the binary, so it describes THIS build and cannot be stale.
"--manifest" => {
println!("{}", info::ops::Info::manifest());
return;
}
other => {
eprintln!("info: unknown argument {other}");
std::process::exit(2);
+9 -31
View File
@@ -2,39 +2,17 @@
// Copyright (C) 2026, Lux Industries Inc. All rights reserved.
// See the file LICENSE for licensing terms.
//! Where the description goes.
//! Writing the description down.
//!
//! A macro runs inside the compiler and writes its fragment of the manifest as
//! it expands, so the description of an op exists the moment the op does. There
//! is no second pass over the source, no generated file to check in, and no
//! staleness to gate: what the compiler read is what the projector reads.
//! A macro runs inside the compiler, and what it learns has to reach the
//! projector somehow. It reaches it as a `&'static str` COMPILED INTO the
//! crate, which the service prints on request — not as a file written during
//! expansion, which was the first shape and the wrong one: rustc reuses a
//! cached expansion, so a macro that did not run left its file saying what the
//! source used to say, and a document that quietly disagrees with the code is
//! the one failure this whole design exists to prevent.
//!
//! The fragments land under `ZIP_MANIFEST_DIR`, or `target/zip` beside the
//! crate. `zipc` merges them.
use std::fs;
use std::path::PathBuf;
/// dir is where this crate's fragments go, creating it if it is not there.
pub fn dir(kind: &str) -> Option<PathBuf> {
let base = match std::env::var("ZIP_MANIFEST_DIR") {
Ok(v) if !v.is_empty() => PathBuf::from(v),
_ => PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").ok()?)
.join("target")
.join("zip"),
};
let at = base.join(kind);
fs::create_dir_all(&at).ok()?;
Some(at)
}
/// write files one fragment, named so a second expansion of the same item
/// replaces the first rather than adding to it.
pub fn write(kind: &str, name: &str, body: &str) {
if let Some(at) = dir(kind) {
let _ = fs::write(at.join(format!("{name}.json")), body);
}
}
//! So: JSON, built from tokens, small enough to have no machinery.
/// quote is a JSON string literal.
pub fn quote(s: &str) -> String {
+23 -2
View File
@@ -50,7 +50,13 @@ pub fn expand(args: TokenStream, mut block: ItemImpl) -> Result<TokenStream, syn
));
}
frag::write("ops", &holder.to_string(), &fragment(&about, &ops));
let stated = fragment(&about, &ops);
let reaches: Vec<TokenStream> = ops
.iter()
.flat_map(|o| [o.input.clone(), o.output.clone()])
.flatten()
.map(|t| quote!(<#t as ::zip::Wire>::reach(&mut types);))
.collect();
let app = about.app;
let title = about.title;
@@ -61,6 +67,18 @@ pub fn expand(args: TokenStream, mut block: ItemImpl) -> Result<TokenStream, syn
#block
impl #holder {
/// Manifest is what this service declared, said neutrally: the ops
/// the impl block carries and every type they reach.
///
/// It is built from what the compiler read, so it cannot describe a
/// service other than this one. `zipc` projects it into the OpenAPI
/// document, the MCP tool list, the CLI and the ZAP schema.
pub fn manifest() -> ::std::string::String {
let mut types: ::std::vec::Vec<(&'static str, &'static str)> = ::std::vec::Vec::new();
#(#reaches)*
::zip::manifest(#stated, &types)
}
/// Ops is this service's typed operations: the REST routes, the ZAP
/// methods, and the document that describes them, from the one
/// registration each op already carries.
@@ -243,6 +261,8 @@ impl Op {
}
}
/// fragment is the ops half of the manifest — the object's MEMBERS, without its
/// braces, because [zip::manifest] closes it around the types half.
fn fragment(about: &About, ops: &[Op]) -> String {
let items: Vec<String> = ops.iter().map(Op::json).collect();
let mut o = frag::Object::new();
@@ -252,7 +272,8 @@ fn fragment(about: &About, ops: &[Op]) -> String {
.text("version", &about.version)
.text("description", &about.describe)
.raw("ops", &frag::list(&items));
o.finish()
let whole = o.finish();
whole[1..whole.len() - 1].to_string()
}
/// route reads `#[get("/path")]` and its siblings off a method.
+47 -11
View File
@@ -42,13 +42,16 @@ pub fn derive(input: DeriveInput) -> Result<TokenStream, syn::Error> {
.iter()
.map(Field::read)
.collect::<Result<_, _>>()?;
frag::write("types", &id, &record(&id, &prose.text, &fields));
Ok(shape(&name, &id, &fields))
Ok(shape(
&name,
&id,
&record(&id, &prose.text, &fields),
&fields,
))
}
Fields::Unnamed(un) if un.unnamed.len() == 1 => {
let inner = &un.unnamed[0].ty;
frag::write("types", &id, &value(&id, &own, inner));
Ok(newtype(&name, &id, &own, inner))
Ok(newtype(&name, &id, &value(&id, &own, inner), inner, &own))
}
_ => Err(syn::Error::new_spanned(
&input.ident,
@@ -137,9 +140,9 @@ fn value(id: &str, a: &Attrs, inner: &Type) -> String {
o.finish()
}
/// shape is the impl for a struct: the description a binder reads, and the JSON
/// the wire carries.
fn shape(name: &syn::Ident, id: &str, fields: &[Field]) -> TokenStream {
/// shape is the impl for a struct: the description a binder reads, the entry a
/// manifest carries, and the JSON the wire binds.
fn shape(name: &syn::Ident, id: &str, stated: &str, fields: &[Field]) -> TokenStream {
let descs = fields.iter().map(|f| {
let n = f.name.to_string();
let j = &f.json;
@@ -176,6 +179,11 @@ fn shape(name: &syn::Ident, id: &str, fields: &[Field]) -> TokenStream {
quote! { #at: #r }
});
let reaches = fields
.iter()
.filter_map(|f| named_in(&f.ty))
.map(|path| quote!(<#path as ::zip::Wire>::reach(into);));
quote! {
impl ::zip::Wire for #name {
fn describe() -> &'static ::zip::TypeDesc {
@@ -186,6 +194,14 @@ fn shape(name: &syn::Ident, id: &str, fields: &[Field]) -> TokenStream {
};
&DESC
}
fn stated() -> &'static str { #stated }
fn reach(into: &mut ::std::vec::Vec<(&'static str, &'static str)>) {
if into.iter().any(|(k, _)| *k == #id) {
return; // claimed before the fields are walked: the cycle guard.
}
into.push((#id, #stated));
#(#reaches)*
}
fn write_json(&self, out: &mut ::std::string::String) {
out.push('{');
#(#writes)*
@@ -200,7 +216,15 @@ fn shape(name: &syn::Ident, id: &str, fields: &[Field]) -> TokenStream {
/// newtype is the impl for a value type. `text` carries it as one word, which
/// is what a quoted decimal is and what a URL can hold.
fn newtype(name: &syn::Ident, id: &str, a: &Attrs, inner: &Type) -> TokenStream {
fn newtype(name: &syn::Ident, id: &str, stated: &str, inner: &Type, a: &Attrs) -> TokenStream {
let says = quote! {
fn stated() -> &'static str { #stated }
fn reach(into: &mut ::std::vec::Vec<(&'static str, &'static str)>) {
if into.iter().all(|(k, _)| *k != #id) {
into.push((#id, #stated));
}
}
};
if a.text {
return quote! {
impl ::zip::Wire for #name {
@@ -208,6 +232,7 @@ fn newtype(name: &syn::Ident, id: &str, a: &Attrs, inner: &Type) -> TokenStream
static DESC: ::zip::TypeDesc = ::zip::TypeDesc { id: #id, text: true, fields: &[] };
&DESC
}
#says
fn write_json(&self, out: &mut ::std::string::String) {
::zip::json::write_str(out, &::std::string::ToString::to_string(&self.0));
}
@@ -229,6 +254,7 @@ fn newtype(name: &syn::Ident, id: &str, a: &Attrs, inner: &Type) -> TokenStream
static DESC: ::zip::TypeDesc = ::zip::TypeDesc { id: #id, text: false, fields: &[] };
&DESC
}
#says
fn write_json(&self, out: &mut ::std::string::String) { #w; }
fn read_json(v: &::zip::Json) -> ::std::result::Result<Self, ::zip::Error> {
Ok(#name(#r))
@@ -253,18 +279,28 @@ fn scalar_of(t: &Type) -> TokenStream {
let e = scalar_of(&inner);
quote!(::zip::Scalar::List(&#e))
}
Shape::Named => quote!(::zip::Scalar::Named),
Shape::Named(_) => quote!(::zip::Scalar::Named),
_ => quote!(::zip::Scalar::Body),
}
}
/// named_in is the path of the described type a field reaches, through whatever
/// container holds it, or None for a field made only of primitives.
fn named_in(t: &Type) -> Option<syn::Path> {
match shape_of(t) {
Shape::Named(p) => Some(p),
Shape::Opt(inner) | Shape::List(inner) | Shape::Map(inner) => named_in(&inner),
_ => None,
}
}
/// Shape is the structure of a written type, which is all a macro can see.
enum Shape {
Prim(&'static str),
Opt(Type),
List(Type),
Map(Type),
Named,
Named(syn::Path),
Other,
}
@@ -301,7 +337,7 @@ fn shape_of(t: &Type) -> Shape {
("HashMap", 2) | ("BTreeMap", 2) => Shape::Map(args[1].clone()),
_ => match ty::prim(t) {
Some(p) => Shape::Prim(p),
None => Shape::Named,
None => Shape::Named(p.path.clone()),
},
}
}
-4
View File
@@ -10,7 +10,6 @@
//! spellings of one.
use std::collections::HashMap;
use std::sync::Arc;
use crate::{Error, FieldDesc, Json, Scalar, TypeDesc, Wire};
@@ -381,6 +380,3 @@ fn scalar(text: &str, want: &Scalar) -> Json {
_ => Json::Text(text.to_string()),
}
}
/// Shared is an app several doors serve at once.
pub type Shared = Arc<App>;
+14
View File
@@ -74,6 +74,20 @@ pub enum Scalar {
/// another name.
pub trait Wire: Sized {
fn describe() -> &'static TypeDesc;
/// stated is this type's whole description — its fields, their wire names,
/// their prose — as one manifest entry.
///
/// It is a `&'static str` built by the macro from the declaration, so what
/// the service publishes is what the compiler read, and there is no file
/// beside the source to fall out of step with it.
fn stated() -> &'static str;
/// reach adds this type and everything it holds to a manifest, once each.
/// A macro sees only the item it is on, so the graph is walked by the types
/// themselves: each one claims its entry, then asks the types of its fields.
fn reach(into: &mut Vec<(&'static str, &'static str)>);
fn write_json(&self, out: &mut String);
fn read_json(v: &crate::Json) -> Result<Self, crate::Error>;
}
+27 -1
View File
@@ -49,7 +49,7 @@ pub mod zaphttp;
// written by hand.
include!(concat!(env!("OUT_DIR"), "/generated.rs"));
pub use app::{bind, Answer, App, Call, Input, Op, Shared};
pub use app::{bind, Answer, App, Call, Input, Op};
pub use desc::{FieldDesc, Scalar, TypeDesc, Wire};
pub use error::Error;
pub use json::Json;
@@ -57,6 +57,32 @@ pub use json::Json;
/// The two macros. `ops` declares a service; `Wire` makes a type say what it is.
pub use zip_macros::{ops, Wire};
/// manifest is this service's whole description, as one document.
///
/// The ops half comes from the macro that read the impl block; the types half
/// is every type those ops reach, each stating itself. Nothing here decides
/// anything — it is a join — so the document a service prints is the
/// declaration it compiled from and cannot be anything else.
///
/// `zipc` reads it and writes the OpenAPI document, the MCP tool list, the CLI
/// and the ZAP schema. A Go service's manifest is the same document, filled in
/// the same way by the front end that language has.
pub fn manifest(ops: &str, types: &[(&str, &str)]) -> String {
let mut sorted: Vec<&(&str, &str)> = types.iter().collect();
sorted.sort_by_key(|(id, _)| *id);
let mut out = String::from("{");
out.push_str(ops);
out.push_str(",\"types\":[");
for (i, (_, stated)) in sorted.iter().enumerate() {
if i > 0 {
out.push(',');
}
out.push_str(stated);
}
out.push_str("]}");
out
}
/// listen serves an app on every address given, and does not return.
///
/// One verb, whatever the transport: an address decides which door it is, so a