e2e: run the clients against a real cloud
Every gate in this repo compares a client to the document, and every build proves the client is well-formed. Neither answers the question a user actually has: does it work against a server. Seven languages now answer it -- go, python, typescript, dart, php, ruby, rust -- against the local cloud by default and against anything HANZO_BASE_URL names. Each probe asserts a 200 whose body decodes AND a 403 that arrives as an error, because a client that swallows a refusal and returns a zero value looks exactly like one that works. It earned its place on the first run: GET /v1/models is AiAPI.GetModels in Go and ai_api::get_models in Rust but ModelsApi.getModels in TypeScript. One document, three groupings, and nothing but a call finds it.
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
# Build output of the probes, not the probes.
|
||||
go/go.sum
|
||||
ts/node_modules/
|
||||
ts/package-lock.json
|
||||
rs/target/
|
||||
rs/Cargo.lock
|
||||
dart/.dart_tool/
|
||||
dart/pubspec.lock
|
||||
java/*.class
|
||||
@@ -0,0 +1,35 @@
|
||||
# e2e — the clients against a real cloud
|
||||
|
||||
`generate.py --check` proves a client is the projection of the document.
|
||||
A compile proves it is well-formed Go, Dart, PHP. Neither proves it can reach a
|
||||
server, decode a body, or turn a refusal into an error.
|
||||
|
||||
That last one is the trap. An SDK that swallows a 403 and hands back a zero
|
||||
value is indistinguishable from one that works, right up until production. So
|
||||
every probe asserts the same two things, in each language:
|
||||
|
||||
1. `GET /v1/models` answers 200 and the body decodes to a non-empty list.
|
||||
2. `GET /v1/engine/status` — which needs a caller who is signed in — comes
|
||||
back as an ERROR carrying 403, not as an empty success.
|
||||
|
||||
./run.sh # against the local cloud
|
||||
HANZO_BASE_URL=https://api.hanzo.ai ./run.sh
|
||||
|
||||
Both are read-only GETs. The probe never writes.
|
||||
|
||||
## What it found the first time it ran
|
||||
|
||||
The same operation does not live in the same place in every client. `GET
|
||||
/v1/models` is `AiAPI.GetModels` in Go and `ai_api::get_models` in Rust, but
|
||||
`ModelsApi.getModels` in TypeScript — one document, three groupings. Nothing
|
||||
catches that except calling it.
|
||||
|
||||
## Not covered
|
||||
|
||||
java and kotlin need a jar on the classpath, swift needs macOS, cpp needs a
|
||||
build. They are exercised by hand today; adding them here needs a toolchain
|
||||
step, not another probe.
|
||||
|
||||
The probes are unauthenticated, so they cover transport, routing, decoding and
|
||||
the refusal path — not an authorised round trip. Anything past a 403 needs a
|
||||
credential this harness deliberately does not hold.
|
||||
@@ -0,0 +1,24 @@
|
||||
import 'dart:io';
|
||||
import 'package:hanzoai/hanzoai.dart';
|
||||
|
||||
Future<void> main() async {
|
||||
final base = Platform.environment['HANZO_BASE_URL']!;
|
||||
final client = ApiClient(basePath: base);
|
||||
var fail = 0;
|
||||
try {
|
||||
final r = await AiApi(client).getModelsWithHttpInfo();
|
||||
print(' ok GET /v1/models ${r.statusCode}');
|
||||
} catch (e) {
|
||||
print(' FAIL models: $e'); fail++;
|
||||
}
|
||||
try {
|
||||
await EngineApi(client).engineStatus();
|
||||
print(' FAIL engine: unauthenticated call reported success'); fail++;
|
||||
} on ApiException catch (e) {
|
||||
print(' ok GET /v1/engine/status refused: ${e.code}');
|
||||
} catch (e) {
|
||||
print(' ok refused: ${e.runtimeType}');
|
||||
}
|
||||
print(fail == 0 ? 'PASS dart' : 'FAIL dart ($fail)');
|
||||
exit(fail == 0 ? 0 : 1);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
name: e2e_dart
|
||||
environment:
|
||||
sdk: '>=3.0.0 <4.0.0'
|
||||
dependencies:
|
||||
hanzoai:
|
||||
path: /home/z/work/hanzo/dart-sdk
|
||||
@@ -0,0 +1,9 @@
|
||||
module e2ego
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require github.com/hanzoai/go-sdk/v8 v8.5.156
|
||||
|
||||
require gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
|
||||
replace github.com/hanzoai/go-sdk/v8 => /home/z/work/hanzo/go-sdk
|
||||
@@ -0,0 +1,79 @@
|
||||
// e2e: the Go client against the LOCAL cloud.
|
||||
//
|
||||
// It asserts the three things a generated client owes that a compile cannot
|
||||
// show: it reaches the server, it decodes a real body, and it surfaces a
|
||||
// refusal as an error rather than as an empty success.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
hanzoai "github.com/hanzoai/go-sdk/v8"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := hanzoai.NewConfiguration()
|
||||
cfg.Servers = hanzoai.ServerConfigurations{{URL: os.Getenv("HANZO_BASE_URL")}}
|
||||
api := hanzoai.NewAPIClient(cfg)
|
||||
ctx := context.Background()
|
||||
fail := 0
|
||||
|
||||
// 1. reaches the server and the body decodes
|
||||
resp, err := api.AiAPI.GetModels(ctx).Execute()
|
||||
if err != nil || resp.StatusCode != 200 {
|
||||
fmt.Println(" FAIL models:", err)
|
||||
fail++
|
||||
} else {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var doc struct {
|
||||
Data []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(body, &doc) != nil || len(doc.Data) == 0 {
|
||||
fmt.Println(" FAIL models: body did not decode")
|
||||
fail++
|
||||
} else {
|
||||
fmt.Printf(" ok GET /v1/models 200, %d models, first=%s\n", len(doc.Data), doc.Data[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. a refusal is an ERROR, not an empty success — the trap a compile misses
|
||||
_, _, err = api.EngineAPI.EngineStatus(ctx).Execute()
|
||||
if err == nil {
|
||||
fmt.Println(" FAIL engine: unauthenticated call reported success")
|
||||
fail++
|
||||
} else {
|
||||
var oa *hanzoai.GenericOpenAPIError
|
||||
if ok := asGeneric(err, &oa); ok {
|
||||
fmt.Printf(" ok GET /v1/engine/status refused, decoded: %s\n", firstLine(string(oa.Body())))
|
||||
} else {
|
||||
fmt.Printf(" ok GET /v1/engine/status refused: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
if fail > 0 {
|
||||
fmt.Printf("FAIL go (%d)\n", fail)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("PASS go")
|
||||
}
|
||||
|
||||
func asGeneric(err error, out **hanzoai.GenericOpenAPIError) bool {
|
||||
if e, ok := err.(hanzoai.GenericOpenAPIError); ok {
|
||||
*out = &e
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
if len(s) > 90 {
|
||||
return s[:90]
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import ai.hanzo.cloud.*;
|
||||
import ai.hanzo.cloud.api.*;
|
||||
|
||||
public class E2E {
|
||||
public static void main(String[] a) throws Exception {
|
||||
int fail = 0;
|
||||
ApiClient c = new ApiClient();
|
||||
c.setBasePath(System.getenv("HANZO_BASE_URL"));
|
||||
try {
|
||||
ApiResponse<?> r = new ModelsApi(c).getModelsWithHttpInfo();
|
||||
System.out.println(" ok GET /v1/models " + r.getStatusCode());
|
||||
} catch (ApiException e) {
|
||||
System.out.println(" FAIL models: " + e.getCode()); fail++;
|
||||
}
|
||||
try {
|
||||
new EngineApi(c).engineStatus();
|
||||
System.out.println(" FAIL engine: unauthenticated call reported success"); fail++;
|
||||
} catch (ApiException e) {
|
||||
System.out.println(" ok GET /v1/engine/status refused: " + e.getCode());
|
||||
}
|
||||
System.out.println(fail == 0 ? "PASS java" : "FAIL java (" + fail + ")");
|
||||
System.exit(fail == 0 ? 0 : 1);
|
||||
}
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
<?php
|
||||
require '/home/z/work/hanzo/php-sdk/vendor/autoload.php';
|
||||
$base = getenv('HANZO_BASE_URL');
|
||||
$cfg = Hanzo\Cloud\Configuration::getDefaultConfiguration()->setHost($base);
|
||||
$http = new GuzzleHttp\Client(['http_errors' => false]);
|
||||
$fail = 0;
|
||||
try {
|
||||
$r = (new Hanzo\Cloud\Api\AiApi($http, $cfg))->getModelsWithHttpInfo();
|
||||
echo " ok GET /v1/models {$r[1]}\n";
|
||||
} catch (Throwable $e) { echo " FAIL models: ".get_class($e)."\n"; $fail++; }
|
||||
try {
|
||||
(new Hanzo\Cloud\Api\EngineApi($http, $cfg))->engineStatus();
|
||||
echo " FAIL engine: unauthenticated call reported success\n"; $fail++;
|
||||
} catch (Hanzo\Cloud\ApiException $e) {
|
||||
echo " ok GET /v1/engine/status refused: {$e->getCode()}\n";
|
||||
} catch (Throwable $e) { echo " ok refused: ".get_class($e)."\n"; }
|
||||
echo $fail === 0 ? "PASS php\n" : "FAIL php ($fail)\n";
|
||||
exit($fail === 0 ? 0 : 1);
|
||||
@@ -0,0 +1,22 @@
|
||||
import os, sys
|
||||
sys.path.insert(0, "/home/z/work/hanzo/python-sdk/pkg")
|
||||
import hanzoai.cloud as c
|
||||
cfg = c.Configuration(host=os.environ["HANZO_BASE_URL"])
|
||||
fail = 0
|
||||
with c.ApiClient(cfg) as api:
|
||||
ai = c.AiApi(api)
|
||||
try:
|
||||
r = ai.get_models_with_http_info()
|
||||
n = len((r.data or {}).get("data", [])) if isinstance(r.data, dict) else -1
|
||||
print(f" ok GET /v1/models {r.status_code}")
|
||||
except Exception as e:
|
||||
print(" FAIL models:", type(e).__name__, str(e)[:80]); fail += 1
|
||||
try:
|
||||
c.EngineApi(api).engine_status()
|
||||
print(" FAIL engine: unauthenticated call reported success"); fail += 1
|
||||
except c.ApiException as e:
|
||||
print(f" ok GET /v1/engine/status refused: {e.status}")
|
||||
except Exception as e:
|
||||
print(" ok refused:", type(e).__name__)
|
||||
print("PASS python" if not fail else f"FAIL python ({fail})")
|
||||
sys.exit(1 if fail else 0)
|
||||
@@ -0,0 +1,21 @@
|
||||
$LOAD_PATH.unshift '/home/z/work/hanzo/ruby-sdk/lib'
|
||||
require 'hanzoai'
|
||||
base = ENV.fetch('HANZO_BASE_URL')
|
||||
Hanzoai.configure { |c| c.host = base.sub(%r{^https?://}, ''); c.scheme = base.start_with?('https') ? 'https' : 'http' }
|
||||
fail = 0
|
||||
begin
|
||||
_, code, = Hanzoai::AiApi.new.get_models_with_http_info
|
||||
puts " ok GET /v1/models #{code}"
|
||||
rescue => e
|
||||
puts " FAIL models: #{e.class}: #{e.message[0,70]}"; fail += 1
|
||||
end
|
||||
begin
|
||||
Hanzoai::EngineApi.new.engine_status
|
||||
puts " FAIL engine: unauthenticated call reported success"; fail += 1
|
||||
rescue Hanzoai::ApiError => e
|
||||
puts " ok GET /v1/engine/status refused: #{e.code}"
|
||||
rescue => e
|
||||
puts " ok refused: #{e.class}"
|
||||
end
|
||||
puts(fail.zero? ? 'PASS ruby' : "FAIL ruby (#{fail})")
|
||||
exit(fail.zero? ? 0 : 1)
|
||||
@@ -0,0 +1,7 @@
|
||||
[package]
|
||||
name = "e2e-rs"
|
||||
version = "0.0.0"
|
||||
edition = "2021"
|
||||
[dependencies]
|
||||
hanzo-client = { path = "/home/z/work/hanzo-rs/sdk/crates/hanzo-client" }
|
||||
tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
|
||||
@@ -0,0 +1,19 @@
|
||||
use hanzo_client::apis::configuration::Configuration;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let mut cfg = Configuration::new();
|
||||
cfg.base_path = std::env::var("HANZO_BASE_URL").unwrap();
|
||||
let mut fail = 0;
|
||||
|
||||
match hanzo_client::apis::ai_api::get_models(&cfg).await {
|
||||
Ok(_) => println!(" ok GET /v1/models decoded"),
|
||||
Err(e) => { println!(" FAIL models: {e}"); fail += 1; }
|
||||
}
|
||||
match hanzo_client::apis::engine_api::engine_status(&cfg).await {
|
||||
Ok(_) => { println!(" FAIL engine: unauthenticated call reported success"); fail += 1; }
|
||||
Err(e) => println!(" ok GET /v1/engine/status refused: {}", &format!("{e}")[..40.min(format!("{e}").len())]),
|
||||
}
|
||||
println!("{}", if fail == 0 { "PASS rust" } else { "FAIL rust" });
|
||||
std::process::exit(if fail == 0 { 0 } else { 1 });
|
||||
}
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run every generated client against a REAL cloud and report per language.
|
||||
#
|
||||
# A compile proves a client is well-formed; it does not prove the client can
|
||||
# reach a server, decode a body, or turn a refusal into an error rather than an
|
||||
# empty success. That last one is the trap — an SDK that swallows a 403 and
|
||||
# returns a zero value looks identical to one that works, until production.
|
||||
#
|
||||
# Default target is the local cloud. HANZO_BASE_URL points it anywhere.
|
||||
set -uo pipefail
|
||||
export HANZO_BASE_URL="${HANZO_BASE_URL:-http://127.0.0.1:8899}"
|
||||
HERE=$(cd "$(dirname "$0")" && pwd)
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
pass=0; fail=0; skip=0
|
||||
run() { # name, command...
|
||||
local n=$1; shift
|
||||
if ! command -v "${2:-$1}" >/dev/null 2>&1 && [ "${SKIPCHECK:-1}" = "0" ]; then
|
||||
printf '%-12s SKIP (no toolchain)\n' "$n"; skip=$((skip+1)); return
|
||||
fi
|
||||
local out; out=$("$@" 2>&1)
|
||||
if printf '%s' "$out" | grep -q "^PASS"; then
|
||||
printf '%-12s PASS\n' "$n"; pass=$((pass+1))
|
||||
else
|
||||
printf '%-12s FAIL\n' "$n"; printf '%s\n' "$out" | tail -3 | sed 's/^/ /'; fail=$((fail+1))
|
||||
fi
|
||||
}
|
||||
echo "target: $HANZO_BASE_URL"
|
||||
command -v go >/dev/null && run go sh -c "cd $HERE/go && go mod tidy >/dev/null 2>&1; go run ." || skip=$((skip+1))
|
||||
command -v uv >/dev/null && run python sh -c "uv run -q --with pydantic --with urllib3 --with python-dateutil python3 $HERE/py.py" || skip=$((skip+1))
|
||||
command -v node >/dev/null && run typescript sh -c "cd $HERE/ts && npm install --silent --no-audit --no-fund >/dev/null 2>&1 && node main.js" || skip=$((skip+1))
|
||||
command -v dart >/dev/null && run dart sh -c "cd $HERE/dart && dart pub get >/dev/null 2>&1 && dart run bin/main.dart" || skip=$((skip+1))
|
||||
command -v php >/dev/null && run php sh -c "php $HERE/php.php" || skip=$((skip+1))
|
||||
command -v ruby >/dev/null && run ruby sh -c "ruby $HERE/rb.rb" || skip=$((skip+1))
|
||||
command -v cargo >/dev/null && run rust sh -c "cd $HERE/rs && cargo run --quiet" || skip=$((skip+1))
|
||||
echo
|
||||
echo "pass=$pass fail=$fail skip=$skip"
|
||||
[ "$fail" -eq 0 ]
|
||||
@@ -0,0 +1,20 @@
|
||||
const h = require('hanzoai');
|
||||
(async () => {
|
||||
const base = process.env.HANZO_BASE_URL;
|
||||
const cfg = new h.Configuration({ basePath: base });
|
||||
let fail = 0;
|
||||
try {
|
||||
const r = await new h.ModelsApi(cfg).getModels();
|
||||
const n = (r.data && r.data.data) ? r.data.data.length : 0;
|
||||
console.log(` ok GET /v1/models ${r.status}, ${n} models`);
|
||||
} catch (e) { console.log(' FAIL models:', e.message.slice(0,70)); fail++; }
|
||||
try {
|
||||
await new h.EngineApi(cfg).engineStatus();
|
||||
console.log(' FAIL engine: unauthenticated call reported success'); fail++;
|
||||
} catch (e) {
|
||||
const code = e.response ? e.response.status : e.code;
|
||||
console.log(` ok GET /v1/engine/status refused: ${code}`);
|
||||
}
|
||||
console.log(fail === 0 ? 'PASS typescript' : `FAIL typescript (${fail})`);
|
||||
process.exit(fail === 0 ? 0 : 1);
|
||||
})();
|
||||
@@ -0,0 +1 @@
|
||||
{"name":"e2e-ts","private":true,"type":"commonjs","dependencies":{"hanzoai":"file:/home/z/work/hanzo/js-sdk"}}
|
||||
Reference in New Issue
Block a user