node/conformance/runner/main.go

// SPDX-License-Identifier: BSD-3-Clause-Eco

// The cross-language differential runner.
//
// It hands ONE corpus to every evaluator it is given, collects their verdict
// lines, and compares them field by field. A field two implementations answer
// differently is a disagreement: the run fails, and the row names the pair and
// prints what each of them said.
//
// It knows nothing about any chain. It parses two tab-separated formats and
// compares strings — which is the point. A runner that understood the rules
// would be a fourth implementation, and the day it was wrong it would hide the
// disagreement instead of reporting it.
//
// Nothing here reaches luxfi/node: the reference lives behind the Go evaluator,
// in its own module.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"sort"
	"strings"
)

// The fields that are compared, in the order they appear on a result line.
// Which fields those are is not knowledge this program has: the caller names
// them with -fields, because the caller is what chose the evaluators, and a
// runner that knew the chain's five field names would have to learn the next
// differential's as well.
var compared []string

// notEvaluated marks a field an implementation declined to answer. It is never
// a pass: it is counted, reported, excluded from comparison, and it fails the
// run, because comparing an answer against a non-answer would let a chain that
// evaluates nothing agree with everyone.
const notEvaluated = "SKIPPED"

type result struct {
	fields []string
	note   string
}

type evaluator struct {
	name string
	cmd  []string
	rows map[string]result

	// A recording is what one implementation said when the corpus was built,
	// read back from a file. It is compared like any other voice, so a
	// reference that has changed its mind since is a disagreement. It does
	// NOT count toward the two answers a field needs, because it is not a
	// second opinion: it is the first one, written down.
	recorded bool
}

func main() {
	var vectors string
	var evalFlags stringList
	flag.StringVar(&vectors, "vectors", "conformance/corpus/vectors.tsv", "the corpus")
	fields := flag.String("fields", "", "comma-separated names of the compared fields, in result-line order")
	subject := flag.String("subject", "", "what is being compared, for the banner")
	flag.Var(&evalFlags, "eval", "name=command to run (repeatable); the corpus path is appended")
	expected := flag.String("expected", "", "the recorded reference answers, compared as one more implementation")
	verbose := flag.Bool("v", false, "print every row, not only the disagreements")
	flag.Parse()

	for _, f := range strings.Split(*fields, ",") {
		if f = strings.TrimSpace(f); f != "" {
			compared = append(compared, f)
		}
	}
	if len(compared) == 0 {
		fmt.Fprintln(os.Stderr, "-fields names the columns to compare; without it nothing is compared")
		os.Exit(2)
	}

	if len(evalFlags) < 2 {
		fmt.Fprintln(os.Stderr, "a differential needs at least two implementations to differ")
		os.Exit(2)
	}

	vecs, err := readVectors(vectors)
	if err != nil {
		fmt.Fprintln(os.Stderr, "runner:", err)
		os.Exit(1)
	}

	var evals []*evaluator
	byName := map[string]*evaluator{}
	for _, spec := range evalFlags {
		name, command, ok := strings.Cut(spec, "=")
		if !ok {
			fmt.Fprintf(os.Stderr, "runner: -eval wants name=command, got %q\n", spec)
			os.Exit(2)
		}
		// One implementation may answer through more than one binary — a chain
		// per program is how two of the three are built. Repeating a name
		// merges the rows, and a vector answered twice under one name is an
		// error rather than a silent last-one-wins.
		e := byName[name]
		if e == nil {
			e = &evaluator{name: name, rows: map[string]result{}}
			byName[name] = e
			evals = append(evals, e)
		}
		e.cmd = append(strings.Fields(command), vectors)
		if err := e.run(); err != nil {
			// An evaluator that cannot run is reported and the run fails. A
			// harness that quietly dropped an implementation would report
			// agreement among whoever was left.
			fmt.Fprintf(os.Stderr, "runner: %s did not run: %v\n", name, err)
			os.Exit(1)
		}
	}

	// The corpus's recorded answers join the comparison so that a reference
	// that changed its mind since the corpus was generated is a disagreement
	// rather than a silent new normal. They are the Go evaluator's own output,
	// written down, so they are marked as a recording: they can disagree with
	// anyone, and they cannot stand in for the second implementation a field
	// needs before it counts as compared.
	if *expected != "" {
		e := &evaluator{name: "corpus", rows: map[string]result{}, recorded: true}
		if err := e.load(*expected); err != nil {
			fmt.Fprintln(os.Stderr, "runner:", err)
			os.Exit(1)
		}
		evals = append(evals, e)
	}

	report(*subject, vecs, evals, *verbose)
}

func (e *evaluator) run() error {
	cmd := exec.Command(e.cmd[0], e.cmd[1:]...)
	cmd.Stderr = os.Stderr
	out, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("%s: %w", strings.Join(e.cmd, " "), err)
	}
	s := bufio.NewScanner(strings.NewReader(string(out)))
	s.Buffer(make([]byte, 1<<20), 1<<26)
	for s.Scan() {
		line := s.Text()
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		f := strings.Split(line, "\t")
		if len(f) != resultWidth() || f[0] != "R" {
			return fmt.Errorf("not a result line, want %d columns for %s, got %d: %q",
				resultWidth(), strings.Join(compared, "/"), len(f), line)
		}
		if _, dup := e.rows[f[1]]; dup {
			return fmt.Errorf("%s answered %s twice", e.name, f[1])
		}
		if i := blank(f[2 : 2+len(compared)]); i >= 0 {
			return fmt.Errorf("%s left %s of %s empty; a field it cannot answer is %s",
				e.name, compared[i], f[1], notEvaluated)
		}
		e.rows[f[1]] = result{
			fields: append([]string(nil), f[2:2+len(compared)]...),
			note:   f[len(f)-1],
		}
	}
	return s.Err()
}

// load reads recorded answers from a file instead of running a program.
func (e *evaluator) load(path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	s := bufio.NewScanner(f)
	s.Buffer(make([]byte, 1<<20), 1<<26)
	for s.Scan() {
		line := s.Text()
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		fs := strings.Split(line, "\t")
		if len(fs) != resultWidth() || fs[0] != "R" {
			return fmt.Errorf("%s: not a result line, want %d columns, got %d: %q",
				path, resultWidth(), len(fs), line)
		}
		if i := blank(fs[2 : 2+len(compared)]); i >= 0 {
			return fmt.Errorf("%s: %s of %s is empty; a field the reference cannot answer is %s",
				path, compared[i], fs[1], notEvaluated)
		}
		e.rows[fs[1]] = result{
			fields: append([]string(nil), fs[2:2+len(compared)]...),
			note:   fs[len(fs)-1],
		}
	}
	return s.Err()
}

// blank reports the first compared field left empty, or -1. An evaluator that
// cannot answer says so with SKIPPED; an empty column is a row it got wrong,
// and reading it as a polite non-answer is how a broken evaluator disappears.
func blank(fields []string) int {
	for i, f := range fields {
		if f == "" {
			return i
		}
	}
	return -1
}

// A vector is its id and the label the corpus files it under. Every corpus
// here writes that label in the column after the id — the chain for the chain
// corpus, the precompile address for the precompile one — and it is the only
// thing past the id this program reads. It reads it so a report can say that
// twelve declines are one chain declining one field rather than twelve gaps.
// The payload after it stays the evaluators' business: the file is handed over
// whole and never opened for its bytes.
type vector struct {
	id    string
	label string
}

func readVectors(path string) ([]vector, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var vs []vector
	s := bufio.NewScanner(f)
	s.Buffer(make([]byte, 1<<20), 1<<26)
	for s.Scan() {
		line := s.Text()
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		f := strings.Split(line, "\t")
		if len(f) < 3 || f[0] != "V" {
			return nil, fmt.Errorf("not a vector line: %q", line)
		}
		vs = append(vs, vector{id: f[1], label: f[2]})
	}
	return vs, s.Err()
}

// A decline is one implementation refusing one field of one vector, and what
// it said about it. Counting them says how big the gap is; this says where it
// is, which is the only form of it anyone can act on.
//
// The note is the row's note, not the field's: an evaluator prints one line per
// vector and one note on it. Where a row declines a field and its note is about
// a different one, that shows up here as a reason that does not mention the
// declined field — which is a finding about the port, and better read than
// hidden behind a count.
type decline struct {
	impl, vector, label, field, note string
}

type disagreement struct {
	vector string
	field  string
	a, b   string // evaluator names
	av, bv string // their answers
	an, bn string // their notes
}

// A result line is R, the vector id, one value per compared field, and a note.
func resultWidth() int { return len(compared) + 3 }

func report(subject string, vecs []vector, evals []*evaluator, verbose bool) {
	ids := make([]string, len(vecs))
	label := make(map[string]string, len(vecs))
	for i, v := range vecs {
		ids[i] = v.id
		label[v.id] = v.label
	}

	names := make([]string, len(evals))
	for i, e := range evals {
		names[i] = e.name
	}
	fmt.Printf("%s differential: %d vectors, %d implementations (%s)\n\n",
		subject, len(ids), len(evals), strings.Join(names, ", "))

	var bad []disagreement
	uncovered := map[string][]string{} // vector -> fields nobody could compare
	silent := map[string][]string{}    // implementation -> vectors it never answered
	var declined []decline             // every field an implementation printed SKIPPED for
	answered := map[string]int{}

	// An implementation that prints no row for a vector has not passed it. The
	// field comparison cannot see that on its own: three of four answering is
	// still a comparison, and it agrees, and the fourth's silence reads as
	// success. Every implementation is asked for every vector, and one that
	// skips a chain it does not serve is expected to be paired with the
	// evaluator that does, under the same name.
	for _, e := range evals {
		for _, id := range ids {
			if _, ok := e.rows[id]; !ok {
				silent[e.name] = append(silent[e.name], id)
			}
		}
	}

	for _, id := range ids {
		rowDisagrees := false
		for fi, field := range compared {
			// Collect the implementations that actually answered this field.
			type said struct {
				name, value, note string
				recorded          bool
			}
			var says []said
			for _, e := range evals {
				r, ok := e.rows[id]
				if !ok {
					continue // this implementation printed no row: absent
				}
				v := r.fields[fi]
				if v == notEvaluated {
					declined = append(declined, decline{
						impl: e.name, vector: id, label: label[id],
						field: field, note: r.note,
					})
					continue
				}
				says = append(says, said{e.name, v, r.note, e.recorded})
			}
			running := 0
			for _, s := range says {
				if !s.recorded {
					running++
				}
			}
			if running < 2 {
				uncovered[id] = append(uncovered[id], field)
				continue
			}
			for i := 0; i < len(says); i++ {
				for j := i + 1; j < len(says); j++ {
					if says[i].value != says[j].value {
						rowDisagrees = true
						bad = append(bad, disagreement{
							vector: id, field: field,
							a: says[i].name, av: says[i].value, an: says[i].note,
							b: says[j].name, bv: says[j].value, bn: says[j].note,
						})
					}
				}
			}
		}
		if !rowDisagrees {
			answered[id]++
		}
		if verbose {
			printRow(id, evals)
		}
	}

	// What nobody could compare. Printed loudly, never counted as a pass.
	if len(uncovered) > 0 {
		fmt.Println("NOT COMPARED — fewer than two running implementations answered:")
		keys := make([]string, 0, len(uncovered))
		for k := range uncovered {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, k := range keys {
			fmt.Printf("  %-34s %s\n", k, strings.Join(uncovered[k], " "))
		}
		fmt.Println()
	}

	// What each implementation declined to answer. A field two others still
	// answered is compared and the run can pass, but a column quietly shrinking
	// is the thing a differential is built to notice, so it is always printed.
	if len(declined) > 0 {
		printDeclined(declined, len(ids)*len(compared))
	}

	if len(silent) > 0 {
		fmt.Println("NOT ANSWERED — an implementation printed no row at all:")
		keys := make([]string, 0, len(silent))
		for k := range silent {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, k := range keys {
			fmt.Printf("  %-6s %d of %d: %s\n", k, len(silent[k]), len(ids), strings.Join(clip(silent[k], 6), " "))
		}
		fmt.Println()
	}

	if len(bad) == 0 && len(silent) == 0 && len(uncovered) == 0 {
		fmt.Printf("AGREED on every field of %d vectors, each answered by at least two\n", len(ids))
		fmt.Printf("running implementations.\n")
		return
	}

	if len(bad) == 0 {
		fmt.Printf("Every compared field of %d vectors agreed, but the run fails:\n", len(ids))
		fmt.Println("a field nothing compared, and an implementation that said nothing,")
		fmt.Println("are not agreement.")
		os.Exit(1)
	}

	fmt.Printf("DISAGREEMENTS: %d\n\n", len(bad))
	for _, d := range bad {
		fmt.Printf("  %s . %s\n", d.vector, d.field)
		fmt.Printf("      %-5s %-12s %s\n", d.a, d.av, trim(d.an))
		fmt.Printf("      %-5s %-12s %s\n", d.b, d.bv, trim(d.bn))
		fmt.Println()
	}

	// The pairs, so the summary line names who disagrees with whom.
	pairs := map[string]int{}
	for _, d := range bad {
		pairs[d.a+" vs "+d.b]++
	}
	keys := make([]string, 0, len(pairs))
	for k := range pairs {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	fmt.Println("by pair:")
	for _, k := range keys {
		fmt.Printf("  %-16s %d\n", k, pairs[k])
	}
	os.Exit(1)
}

// printDeclined names every declined field, gathered by the implementation, the
// label the corpus files the vector under, the field, and the reason given.
//
// The count comes first because it is the size of the gap, and then the whole
// list, because a count is not something anyone can go and fix. Twelve declines
// under one heading are one gap; twelve headings are twelve. Nothing is elided:
// a list that stopped at six would be the count again, wearing examples.
func printDeclined(declined []decline, total int) {
	fmt.Println("DECLINED — printed SKIPPED rather than an answer:")

	perImpl := map[string]int{}
	for _, d := range declined {
		perImpl[d.impl]++
	}
	impls := make([]string, 0, len(perImpl))
	for k := range perImpl {
		impls = append(impls, k)
	}
	sort.Strings(impls)
	for _, k := range impls {
		fmt.Printf("  %-6s %d of %d fields\n", k, perImpl[k], total)
	}
	fmt.Println()

	type group struct {
		impl, label, field, note string
	}
	members := map[group][]string{}
	var order []group
	for _, d := range declined {
		g := group{d.impl, d.label, d.field, d.note}
		if _, seen := members[g]; !seen {
			order = append(order, g)
		}
		members[g] = append(members[g], d.vector)
	}
	sort.Slice(order, func(i, j int) bool {
		a, b := order[i], order[j]
		if a.impl != b.impl {
			return a.impl < b.impl
		}
		if a.label != b.label {
			return a.label < b.label
		}
		if a.field != b.field {
			return a.field < b.field
		}
		return a.note < b.note
	})
	for _, g := range order {
		v := members[g]
		fmt.Printf("  %s . %s . %s — %d, each saying:\n", g.impl, g.label, g.field, len(v))
		fmt.Printf("      %s\n", g.note)
		for _, id := range v {
			fmt.Printf("      %s\n", id)
		}
		fmt.Println()
	}
}

func printRow(id string, evals []*evaluator) {
	fmt.Printf("  %-34s", id)
	for _, e := range evals {
		r, ok := e.rows[id]
		if !ok {
			fmt.Printf(" %s=%-12s", e.name, "-absent-")
			continue
		}
		fmt.Printf(" %s=%-12s", e.name, r.fields[len(r.fields)-1])
	}
	fmt.Println()
}

// clip shortens a list for printing, saying how much it left out.
func clip(v []string, n int) []string {
	if len(v) <= n {
		return v
	}
	return append(append([]string{}, v[:n]...), fmt.Sprintf("... and %d more", len(v)-n))
}

func trim(s string) string {
	if len(s) > 96 {
		return s[:96]
	}
	return s
}

type stringList []string

func (s *stringList) String() string { return strings.Join(*s, ",") }

func (s *stringList) Set(v string) error {
	*s = append(*s, v)
	return nil
}