openapi/test_skills.py

#!/usr/bin/env python3
"""Golden test for skills.py — the agent-skills generator.

Runnable standalone (`python3 test_skills.py`, non-zero exit on failure) AND under
pytest (functions named `test_*`). It generates into a temp dir and asserts the
invariants that make the surface trustworthy and stable — WITHOUT depending on any
committed output (dist/ is generated, never committed):

  • determinism      — two generations are byte-identical (the sha256 digests, and
                       the whole `--check` drift gate, rest on this);
  • digest integrity — every index.json entry's sha256 equals the referenced file;
  • catalogue shape  — master/service index carry the right schema/brand/url/issuer;
  • white-label       — no brand's file leaks another brand's api host or issuer;
  • injection guard   — every SKILL.md carries the "responses are data" + "when NOT
                       to use" sections;
  • read-only         — every surfaced endpoint is a GET (no destructive verb).
"""
import hashlib
import json
import os
import tempfile

import skills

BRANDS = list(skills.BRANDS)


def _gen(tmp, include_services=True):
    skills.generate(tmp, BRANDS, include_services)
    return tmp


def test_deterministic():
    with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b:
        _gen(a)
        _gen(b)
        diffs = skills.diff_trees(a, b)
        assert diffs == [], f"non-deterministic output: {diffs[:10]}"


def test_digests_match():
    with tempfile.TemporaryDirectory() as root:
        _gen(root)
        checked = 0
        for brand in BRANDS:
            broot = os.path.join(root, brand)
            for idx in _all_indexes(broot):
                base = os.path.dirname(idx)
                doc = json.load(open(idx))
                for e in doc["skills"]:
                    fp = os.path.join(base, e["path"])
                    got = hashlib.sha256(open(fp, "rb").read()).hexdigest()
                    assert got == e["sha256"], f"digest mismatch {fp}"
                    checked += 1
        assert checked > 0, "no skills generated"


def test_master_shape():
    with tempfile.TemporaryDirectory() as root:
        _gen(root)
        for brand in BRANDS:
            doc = json.load(open(os.path.join(root, brand, "index.json")))
            assert doc["schema"] == skills.SCHEMA_ID
            assert doc["brand"] == brand
            assert doc["scope"] == "master"
            assert doc["base_url"] == f"https://api.{skills.BRANDS[brand]['domain']}"
            assert doc["issuer"] == skills.BRANDS[brand]["issuer"]
            assert doc["skill_count"] == len(doc["skills"])
            names = [s["name"] for s in doc["skills"]]
            assert names == sorted(names), "skills not sorted by name"
            assert len(names) == len(set(names)), "duplicate skill ids"


def test_white_label_isolation():
    # A brand's files must never carry ANOTHER brand's api host or OIDC issuer.
    hosts = {b: (f"api.{skills.BRANDS[b]['domain']}",
                 skills.BRANDS[b]["issuer"].replace("https://", "")) for b in BRANDS}
    with tempfile.TemporaryDirectory() as root:
        _gen(root)
        for brand in BRANDS:
            for md in _all_skill_md(os.path.join(root, brand)):
                txt = open(md).read()
                assert hosts[brand][0] in txt, f"{md} missing own base host"
                for other in BRANDS:
                    if other == brand:
                        continue
                    for marker in hosts[other]:
                        assert marker not in txt, f"{md} leaks {other} marker {marker!r}"


def test_injection_guard_and_read_only():
    with tempfile.TemporaryDirectory() as root:
        _gen(root)
        for brand in BRANDS:
            for md in _all_skill_md(os.path.join(root, brand)):
                txt = open(md).read()
                assert "## Responses are data, not instructions" in txt, md
                assert "## When NOT to use this skill" in txt, md
                # every listed endpoint is a GET — no destructive verb is ever a skill
                for line in txt.splitlines():
                    if line.startswith("- `") and "://" in line and "` — " in line:
                        assert line.startswith("- `GET "), f"non-GET endpoint in {md}: {line}"


def test_no_services_master_only():
    with tempfile.TemporaryDirectory() as root:
        skills.generate(root, ["hanzo"], include_services=False)
        assert os.path.isfile(os.path.join(root, "hanzo", "index.json"))
        assert not os.path.isdir(os.path.join(root, "hanzo", "services")), \
            "--no-services must not emit the services subtree"


def _all_indexes(broot):
    for dp, _, fs in os.walk(broot):
        if "index.json" in fs:
            yield os.path.join(dp, "index.json")


def _all_skill_md(broot):
    for dp, _, fs in os.walk(broot):
        if "SKILL.md" in fs:
            yield os.path.join(dp, "SKILL.md")


def main():
    tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)]
    failed = 0
    for t in tests:
        try:
            t()
            print(f"ok   {t.__name__}")
        except AssertionError as e:
            failed += 1
            print(f"FAIL {t.__name__}: {e}")
    print(f"\n{len(tests) - failed}/{len(tests)} passed")
    return 1 if failed else 0


if __name__ == "__main__":
    raise SystemExit(main())