mirror of
https://github.com/hanzo-go/sdk.git
synced 2026-11-20 20:13:42 +00:00
1294 lines
47 KiB
Go
1294 lines
47 KiB
Go
/*
|
|
Hanzo Cloud API
|
|
|
|
The Hanzo Cloud API as a customer calls it: every operation under /v1/ except the operator's admin product, relay routes, legacy spellings and capabilities still reached by flag. Tagged by product: the first path segment after /v1/.
|
|
|
|
API version: v1
|
|
*/
|
|
|
|
// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT.
|
|
|
|
package hanzoai
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
)
|
|
|
|
// BenchmarkAPIService BenchmarkAPI service
|
|
type BenchmarkAPIService service
|
|
|
|
type BenchmarkAPIGetBenchmarkCatalogRequest struct {
|
|
ctx context.Context
|
|
ApiService *BenchmarkAPIService
|
|
}
|
|
|
|
func (r BenchmarkAPIGetBenchmarkCatalogRequest) Execute() (*BenchmarkBenchmarkCatalog, *http.Response, error) {
|
|
return r.ApiService.GetBenchmarkCatalogExecute(r)
|
|
}
|
|
|
|
/*
|
|
GetBenchmarkCatalog Is the canonical benchmarks this arena reads — the id, title, axis, item count, upstream source and direction of each, with native marking the ones the standardized harness runs today.
|
|
|
|
Is the canonical benchmarks this arena reads — the id, title, axis, item
|
|
count, upstream source and direction of each, with native marking the ones the
|
|
standardized harness runs today. The first 14 are the public set every major
|
|
provider reports; the next 4 are the tests coding agents are judged on —
|
|
DeepSWE, Terminal-Bench 4.0, SWE-Atlas QnA and SWE-Bench Pro v2 — published as
|
|
claims; the rest are the Decision Suite, scored by Kai's own harness and
|
|
published as claims citing the research run each number came from.
|
|
|
|
direction is "higher" or "lower": which way a score improves, and so which way
|
|
the leaderboard orders its rows. Every score is a percentage; a latency is a
|
|
fact about serving and lives in the research plane's runs, not here.
|
|
|
|
These ids are the vocabulary the rest of the surface takes: a run and a claim
|
|
name them, and the leaderboard and compare read them from ?benchmark=. The
|
|
catalog is deployment-wide and identical for every caller — there is no tenant
|
|
in it.
|
|
|
|
@param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
|
|
@return BenchmarkAPIGetBenchmarkCatalogRequest
|
|
*/
|
|
func (a *BenchmarkAPIService) GetBenchmarkCatalog(ctx context.Context) BenchmarkAPIGetBenchmarkCatalogRequest {
|
|
return BenchmarkAPIGetBenchmarkCatalogRequest{
|
|
ApiService: a,
|
|
ctx: ctx,
|
|
}
|
|
}
|
|
|
|
// Execute executes the request
|
|
//
|
|
// @return BenchmarkBenchmarkCatalog
|
|
func (a *BenchmarkAPIService) GetBenchmarkCatalogExecute(r BenchmarkAPIGetBenchmarkCatalogRequest) (*BenchmarkBenchmarkCatalog, *http.Response, error) {
|
|
var (
|
|
localVarHTTPMethod = http.MethodGet
|
|
localVarPostBody interface{}
|
|
formFiles []formFile
|
|
localVarReturnValue *BenchmarkBenchmarkCatalog
|
|
)
|
|
|
|
localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "BenchmarkAPIService.GetBenchmarkCatalog")
|
|
if err != nil {
|
|
return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
|
|
}
|
|
|
|
localVarPath := localBasePath + "/v1/benchmark/catalog"
|
|
|
|
localVarHeaderParams := make(map[string]string)
|
|
localVarQueryParams := url.Values{}
|
|
localVarFormParams := url.Values{}
|
|
|
|
// to determine the Content-Type header
|
|
localVarHTTPContentTypes := []string{}
|
|
|
|
// set Content-Type header
|
|
localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
|
|
if localVarHTTPContentType != "" {
|
|
localVarHeaderParams["Content-Type"] = localVarHTTPContentType
|
|
}
|
|
|
|
// to determine the Accept header
|
|
localVarHTTPHeaderAccepts := []string{"application/json", "application/problem+json"}
|
|
|
|
// set Accept header
|
|
localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
|
|
if localVarHTTPHeaderAccept != "" {
|
|
localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
|
|
}
|
|
req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
|
|
if err != nil {
|
|
return localVarReturnValue, nil, err
|
|
}
|
|
|
|
localVarHTTPResponse, err := a.client.callAPI(req)
|
|
if err != nil || localVarHTTPResponse == nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
|
|
localVarHTTPResponse.Body.Close()
|
|
localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
|
|
if err != nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
if localVarHTTPResponse.StatusCode >= 300 {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: localVarHTTPResponse.Status,
|
|
}
|
|
var v ProblemDetails
|
|
err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr.error = err.Error()
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
|
|
newErr.model = v
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: err.Error(),
|
|
}
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
return localVarReturnValue, localVarHTTPResponse, nil
|
|
}
|
|
|
|
type BenchmarkAPIGetBenchmarkClaimsRequest struct {
|
|
ctx context.Context
|
|
ApiService *BenchmarkAPIService
|
|
benchmark *string
|
|
model *string
|
|
provider *string
|
|
source *string
|
|
protocol *string
|
|
org *string
|
|
}
|
|
|
|
// Benchmark filters to one benchmark id. Empty returns every benchmark.
|
|
func (r BenchmarkAPIGetBenchmarkClaimsRequest) Benchmark(benchmark string) BenchmarkAPIGetBenchmarkClaimsRequest {
|
|
r.benchmark = &benchmark
|
|
return r
|
|
}
|
|
|
|
// Model filters to one model. Empty returns every model.
|
|
func (r BenchmarkAPIGetBenchmarkClaimsRequest) Model(model string) BenchmarkAPIGetBenchmarkClaimsRequest {
|
|
r.model = &model
|
|
return r
|
|
}
|
|
|
|
// Provider filters to one lab or leaderboard — the way to read what a single source claims across every model it covers.
|
|
func (r BenchmarkAPIGetBenchmarkClaimsRequest) Provider(provider string) BenchmarkAPIGetBenchmarkClaimsRequest {
|
|
r.provider = &provider
|
|
return r
|
|
}
|
|
|
|
// Source filters to one citation, which is the finest grain there is: a source is what makes two claims about one model independent rather than a restatement of each other.
|
|
func (r BenchmarkAPIGetBenchmarkClaimsRequest) Source(source string) BenchmarkAPIGetBenchmarkClaimsRequest {
|
|
r.source = &source
|
|
return r
|
|
}
|
|
|
|
// Protocol filters by HOW a claim was scored, so provider cards can be read apart from third parties running their own harness.
|
|
func (r BenchmarkAPIGetBenchmarkClaimsRequest) Protocol(protocol string) BenchmarkAPIGetBenchmarkClaimsRequest {
|
|
r.protocol = &protocol
|
|
return r
|
|
}
|
|
|
|
// Org filters to the claims one org made; \"admin\" reads the platform's own. It narrows what the caller may already read and never widens it.
|
|
func (r BenchmarkAPIGetBenchmarkClaimsRequest) Org(org string) BenchmarkAPIGetBenchmarkClaimsRequest {
|
|
r.org = &org
|
|
return r
|
|
}
|
|
|
|
func (r BenchmarkAPIGetBenchmarkClaimsRequest) Execute() (*BenchmarkClaimsOut, *http.Response, error) {
|
|
return r.ApiService.GetBenchmarkClaimsExecute(r)
|
|
}
|
|
|
|
/*
|
|
GetBenchmarkClaims Lists the effective claims the caller may read, each labelled with the org that made it and the user who recorded it.
|
|
|
|
Lists the effective claims the caller may read, each labelled with the
|
|
org that made it and the user who recorded it.
|
|
|
|
Anyone reads the public claims. A signed-in caller also reads its own org's
|
|
private ones. No caller reads another org's private claims — a SuperAdmin
|
|
acting in that org included, since acting in an org is not membership of it.
|
|
Rows under org "admin" are the platform's own curated readings: the compiled
|
|
seed and any correction a SuperAdmin wrote.
|
|
|
|
Effective values only. The history of a key lives in the append-only file and
|
|
is not what this op is for; a list that returned every superseded row would
|
|
make the common question the hard one.
|
|
|
|
@param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
|
|
@return BenchmarkAPIGetBenchmarkClaimsRequest
|
|
*/
|
|
func (a *BenchmarkAPIService) GetBenchmarkClaims(ctx context.Context) BenchmarkAPIGetBenchmarkClaimsRequest {
|
|
return BenchmarkAPIGetBenchmarkClaimsRequest{
|
|
ApiService: a,
|
|
ctx: ctx,
|
|
}
|
|
}
|
|
|
|
// Execute executes the request
|
|
//
|
|
// @return BenchmarkClaimsOut
|
|
func (a *BenchmarkAPIService) GetBenchmarkClaimsExecute(r BenchmarkAPIGetBenchmarkClaimsRequest) (*BenchmarkClaimsOut, *http.Response, error) {
|
|
var (
|
|
localVarHTTPMethod = http.MethodGet
|
|
localVarPostBody interface{}
|
|
formFiles []formFile
|
|
localVarReturnValue *BenchmarkClaimsOut
|
|
)
|
|
|
|
localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "BenchmarkAPIService.GetBenchmarkClaims")
|
|
if err != nil {
|
|
return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
|
|
}
|
|
|
|
localVarPath := localBasePath + "/v1/benchmark/claims"
|
|
|
|
localVarHeaderParams := make(map[string]string)
|
|
localVarQueryParams := url.Values{}
|
|
localVarFormParams := url.Values{}
|
|
|
|
if r.benchmark != nil {
|
|
parameterAddToHeaderOrQuery(localVarQueryParams, "Benchmark", r.benchmark, "form", "")
|
|
}
|
|
if r.model != nil {
|
|
parameterAddToHeaderOrQuery(localVarQueryParams, "Model", r.model, "form", "")
|
|
}
|
|
if r.provider != nil {
|
|
parameterAddToHeaderOrQuery(localVarQueryParams, "Provider", r.provider, "form", "")
|
|
}
|
|
if r.source != nil {
|
|
parameterAddToHeaderOrQuery(localVarQueryParams, "Source", r.source, "form", "")
|
|
}
|
|
if r.protocol != nil {
|
|
parameterAddToHeaderOrQuery(localVarQueryParams, "Protocol", r.protocol, "form", "")
|
|
}
|
|
if r.org != nil {
|
|
parameterAddToHeaderOrQuery(localVarQueryParams, "Org", r.org, "form", "")
|
|
}
|
|
// to determine the Content-Type header
|
|
localVarHTTPContentTypes := []string{}
|
|
|
|
// set Content-Type header
|
|
localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
|
|
if localVarHTTPContentType != "" {
|
|
localVarHeaderParams["Content-Type"] = localVarHTTPContentType
|
|
}
|
|
|
|
// to determine the Accept header
|
|
localVarHTTPHeaderAccepts := []string{"application/json", "application/problem+json"}
|
|
|
|
// set Accept header
|
|
localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
|
|
if localVarHTTPHeaderAccept != "" {
|
|
localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
|
|
}
|
|
req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
|
|
if err != nil {
|
|
return localVarReturnValue, nil, err
|
|
}
|
|
|
|
localVarHTTPResponse, err := a.client.callAPI(req)
|
|
if err != nil || localVarHTTPResponse == nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
|
|
localVarHTTPResponse.Body.Close()
|
|
localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
|
|
if err != nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
if localVarHTTPResponse.StatusCode >= 300 {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: localVarHTTPResponse.Status,
|
|
}
|
|
var v ProblemDetails
|
|
err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr.error = err.Error()
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
|
|
newErr.model = v
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: err.Error(),
|
|
}
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
return localVarReturnValue, localVarHTTPResponse, nil
|
|
}
|
|
|
|
type BenchmarkAPIGetBenchmarkCompareRequest struct {
|
|
ctx context.Context
|
|
ApiService *BenchmarkAPIService
|
|
a *string
|
|
b *string
|
|
benchmark *string
|
|
}
|
|
|
|
// A is the first model id. It is required.
|
|
func (r BenchmarkAPIGetBenchmarkCompareRequest) A(a string) BenchmarkAPIGetBenchmarkCompareRequest {
|
|
r.a = &a
|
|
return r
|
|
}
|
|
|
|
// B is the second model id. It is required.
|
|
func (r BenchmarkAPIGetBenchmarkCompareRequest) B(b string) BenchmarkAPIGetBenchmarkCompareRequest {
|
|
r.b = &b
|
|
return r
|
|
}
|
|
|
|
// Benchmark is the catalog id to compare on, defaulting to gpqa_diamond.
|
|
func (r BenchmarkAPIGetBenchmarkCompareRequest) Benchmark(benchmark string) BenchmarkAPIGetBenchmarkCompareRequest {
|
|
r.benchmark = &benchmark
|
|
return r
|
|
}
|
|
|
|
func (r BenchmarkAPIGetBenchmarkCompareRequest) Execute() (*BenchmarkPairing, *http.Response, error) {
|
|
return r.ApiService.GetBenchmarkCompareExecute(r)
|
|
}
|
|
|
|
/*
|
|
GetBenchmarkCompare Is the ONLY valid arm-vs-arm test: it pairs the two models on the items BOTH completed, and answers rescue and damage counts with an exact-McNemar p.
|
|
|
|
Is the ONLY valid arm-vs-arm test: it pairs the two models on the items
|
|
BOTH completed, and answers rescue and damage counts with an exact-McNemar p.
|
|
|
|
Pairing is what prevents the subset artifact — comparing one model's easy subset
|
|
against another's full run — so n_common, not either arm's own coverage, is the
|
|
number to read this by.
|
|
|
|
Both a and b are required. The benchmark defaults to gpqa_diamond.
|
|
|
|
@param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
|
|
@return BenchmarkAPIGetBenchmarkCompareRequest
|
|
*/
|
|
func (a *BenchmarkAPIService) GetBenchmarkCompare(ctx context.Context) BenchmarkAPIGetBenchmarkCompareRequest {
|
|
return BenchmarkAPIGetBenchmarkCompareRequest{
|
|
ApiService: a,
|
|
ctx: ctx,
|
|
}
|
|
}
|
|
|
|
// Execute executes the request
|
|
//
|
|
// @return BenchmarkPairing
|
|
func (a *BenchmarkAPIService) GetBenchmarkCompareExecute(r BenchmarkAPIGetBenchmarkCompareRequest) (*BenchmarkPairing, *http.Response, error) {
|
|
var (
|
|
localVarHTTPMethod = http.MethodGet
|
|
localVarPostBody interface{}
|
|
formFiles []formFile
|
|
localVarReturnValue *BenchmarkPairing
|
|
)
|
|
|
|
localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "BenchmarkAPIService.GetBenchmarkCompare")
|
|
if err != nil {
|
|
return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
|
|
}
|
|
|
|
localVarPath := localBasePath + "/v1/benchmark/compare"
|
|
|
|
localVarHeaderParams := make(map[string]string)
|
|
localVarQueryParams := url.Values{}
|
|
localVarFormParams := url.Values{}
|
|
if r.a == nil {
|
|
return localVarReturnValue, nil, reportError("a is required and must be specified")
|
|
}
|
|
if r.b == nil {
|
|
return localVarReturnValue, nil, reportError("b is required and must be specified")
|
|
}
|
|
|
|
if r.benchmark != nil {
|
|
parameterAddToHeaderOrQuery(localVarQueryParams, "benchmark", r.benchmark, "form", "")
|
|
}
|
|
parameterAddToHeaderOrQuery(localVarQueryParams, "a", r.a, "form", "")
|
|
parameterAddToHeaderOrQuery(localVarQueryParams, "b", r.b, "form", "")
|
|
// to determine the Content-Type header
|
|
localVarHTTPContentTypes := []string{}
|
|
|
|
// set Content-Type header
|
|
localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
|
|
if localVarHTTPContentType != "" {
|
|
localVarHeaderParams["Content-Type"] = localVarHTTPContentType
|
|
}
|
|
|
|
// to determine the Accept header
|
|
localVarHTTPHeaderAccepts := []string{"application/json", "application/problem+json"}
|
|
|
|
// set Accept header
|
|
localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
|
|
if localVarHTTPHeaderAccept != "" {
|
|
localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
|
|
}
|
|
req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
|
|
if err != nil {
|
|
return localVarReturnValue, nil, err
|
|
}
|
|
|
|
localVarHTTPResponse, err := a.client.callAPI(req)
|
|
if err != nil || localVarHTTPResponse == nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
|
|
localVarHTTPResponse.Body.Close()
|
|
localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
|
|
if err != nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
if localVarHTTPResponse.StatusCode >= 300 {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: localVarHTTPResponse.Status,
|
|
}
|
|
var v ProblemDetails
|
|
err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr.error = err.Error()
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
|
|
newErr.model = v
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: err.Error(),
|
|
}
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
return localVarReturnValue, localVarHTTPResponse, nil
|
|
}
|
|
|
|
type BenchmarkAPIGetBenchmarkHistoryRequest struct {
|
|
ctx context.Context
|
|
ApiService *BenchmarkAPIService
|
|
benchmark *string
|
|
model *string
|
|
}
|
|
|
|
// Benchmark is the catalog id to read, defaulting to gpqa_diamond.
|
|
func (r BenchmarkAPIGetBenchmarkHistoryRequest) Benchmark(benchmark string) BenchmarkAPIGetBenchmarkHistoryRequest {
|
|
r.benchmark = &benchmark
|
|
return r
|
|
}
|
|
|
|
// Model filters to one model. Empty returns every model measured.
|
|
func (r BenchmarkAPIGetBenchmarkHistoryRequest) Model(model string) BenchmarkAPIGetBenchmarkHistoryRequest {
|
|
r.model = &model
|
|
return r
|
|
}
|
|
|
|
func (r BenchmarkAPIGetBenchmarkHistoryRequest) Execute() (*BenchmarkHistoryOut, *http.Response, error) {
|
|
return r.ApiService.GetBenchmarkHistoryExecute(r)
|
|
}
|
|
|
|
/*
|
|
GetBenchmarkHistory Returns each model's measured score per run over time, oldest first, with the change between runs.
|
|
|
|
Returns each model's measured score per run over time, oldest first,
|
|
with the change between runs.
|
|
|
|
This is the counterweight to a leaderboard: the board shows the latest run
|
|
because that is what "how good is it" means, and a single latest number cannot
|
|
distinguish a model that has always been strong from one that just improved,
|
|
or from one that regressed after a provider changed something. Both matter for
|
|
routing, and only one of them is visible on a board.
|
|
|
|
Runs with no id — attempts recorded before runs existed — group under the
|
|
empty run, which is honestly what they are: one undated measurement.
|
|
|
|
@param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
|
|
@return BenchmarkAPIGetBenchmarkHistoryRequest
|
|
*/
|
|
func (a *BenchmarkAPIService) GetBenchmarkHistory(ctx context.Context) BenchmarkAPIGetBenchmarkHistoryRequest {
|
|
return BenchmarkAPIGetBenchmarkHistoryRequest{
|
|
ApiService: a,
|
|
ctx: ctx,
|
|
}
|
|
}
|
|
|
|
// Execute executes the request
|
|
//
|
|
// @return BenchmarkHistoryOut
|
|
func (a *BenchmarkAPIService) GetBenchmarkHistoryExecute(r BenchmarkAPIGetBenchmarkHistoryRequest) (*BenchmarkHistoryOut, *http.Response, error) {
|
|
var (
|
|
localVarHTTPMethod = http.MethodGet
|
|
localVarPostBody interface{}
|
|
formFiles []formFile
|
|
localVarReturnValue *BenchmarkHistoryOut
|
|
)
|
|
|
|
localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "BenchmarkAPIService.GetBenchmarkHistory")
|
|
if err != nil {
|
|
return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
|
|
}
|
|
|
|
localVarPath := localBasePath + "/v1/benchmark/history"
|
|
|
|
localVarHeaderParams := make(map[string]string)
|
|
localVarQueryParams := url.Values{}
|
|
localVarFormParams := url.Values{}
|
|
|
|
if r.benchmark != nil {
|
|
parameterAddToHeaderOrQuery(localVarQueryParams, "Benchmark", r.benchmark, "form", "")
|
|
}
|
|
if r.model != nil {
|
|
parameterAddToHeaderOrQuery(localVarQueryParams, "Model", r.model, "form", "")
|
|
}
|
|
// to determine the Content-Type header
|
|
localVarHTTPContentTypes := []string{}
|
|
|
|
// set Content-Type header
|
|
localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
|
|
if localVarHTTPContentType != "" {
|
|
localVarHeaderParams["Content-Type"] = localVarHTTPContentType
|
|
}
|
|
|
|
// to determine the Accept header
|
|
localVarHTTPHeaderAccepts := []string{"application/json", "application/problem+json"}
|
|
|
|
// set Accept header
|
|
localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
|
|
if localVarHTTPHeaderAccept != "" {
|
|
localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
|
|
}
|
|
req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
|
|
if err != nil {
|
|
return localVarReturnValue, nil, err
|
|
}
|
|
|
|
localVarHTTPResponse, err := a.client.callAPI(req)
|
|
if err != nil || localVarHTTPResponse == nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
|
|
localVarHTTPResponse.Body.Close()
|
|
localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
|
|
if err != nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
if localVarHTTPResponse.StatusCode >= 300 {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: localVarHTTPResponse.Status,
|
|
}
|
|
var v ProblemDetails
|
|
err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr.error = err.Error()
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
|
|
newErr.model = v
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: err.Error(),
|
|
}
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
return localVarReturnValue, localVarHTTPResponse, nil
|
|
}
|
|
|
|
type BenchmarkAPIGetBenchmarkLeaderboardRequest struct {
|
|
ctx context.Context
|
|
ApiService *BenchmarkAPIService
|
|
benchmark *string
|
|
}
|
|
|
|
// Benchmark is the catalog id to read, defaulting to gpqa_diamond.
|
|
func (r BenchmarkAPIGetBenchmarkLeaderboardRequest) Benchmark(benchmark string) BenchmarkAPIGetBenchmarkLeaderboardRequest {
|
|
r.benchmark = &benchmark
|
|
return r
|
|
}
|
|
|
|
func (r BenchmarkAPIGetBenchmarkLeaderboardRequest) Execute() (*BenchmarkLeaderboard, *http.Response, error) {
|
|
return r.ApiService.GetBenchmarkLeaderboardExecute(r)
|
|
}
|
|
|
|
/*
|
|
GetBenchmarkLeaderboard Answers one row per model for the benchmark named — what our own harness measured, beside what the vendor claims, and the gap between them.
|
|
|
|
Answers one row per model for the benchmark named — what our own
|
|
harness measured, beside what the vendor claims, and the gap between them.
|
|
|
|
The gap is the point of the arena; provider-reported claims have run materially
|
|
hot against one standardized harness.
|
|
|
|
The two planes are NEVER blended, and that is the rule to read the rows by: a
|
|
model we have measured but no vendor has claimed for shows published null, a
|
|
model with only a claim shows measured null, and gap exists only where both do.
|
|
|
|
Published, Claims, Spread and Mean read the platform's own public claims (org
|
|
"admin") and nothing else, whoever asks. Other orgs' claims are at
|
|
/v1/benchmark/claims, labelled, and reach this board only when a SuperAdmin
|
|
publishes them: writing under org admin, or promoting an org's public claims
|
|
with POST /v1/benchmark/claims/promote, which keeps the org they came from in
|
|
the row's from.
|
|
|
|
Rows are best first. A benchmark whose score improves downward — a false-alarm
|
|
rate, a share of answers that moved — is ordered lowest first, and the answer
|
|
says which with direction.
|
|
|
|
n is coverage and is not decoration: two measured numbers taken over different
|
|
item counts are not comparable, so read the row's n before reading its accuracy.
|
|
|
|
@param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
|
|
@return BenchmarkAPIGetBenchmarkLeaderboardRequest
|
|
*/
|
|
func (a *BenchmarkAPIService) GetBenchmarkLeaderboard(ctx context.Context) BenchmarkAPIGetBenchmarkLeaderboardRequest {
|
|
return BenchmarkAPIGetBenchmarkLeaderboardRequest{
|
|
ApiService: a,
|
|
ctx: ctx,
|
|
}
|
|
}
|
|
|
|
// Execute executes the request
|
|
//
|
|
// @return BenchmarkLeaderboard
|
|
func (a *BenchmarkAPIService) GetBenchmarkLeaderboardExecute(r BenchmarkAPIGetBenchmarkLeaderboardRequest) (*BenchmarkLeaderboard, *http.Response, error) {
|
|
var (
|
|
localVarHTTPMethod = http.MethodGet
|
|
localVarPostBody interface{}
|
|
formFiles []formFile
|
|
localVarReturnValue *BenchmarkLeaderboard
|
|
)
|
|
|
|
localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "BenchmarkAPIService.GetBenchmarkLeaderboard")
|
|
if err != nil {
|
|
return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
|
|
}
|
|
|
|
localVarPath := localBasePath + "/v1/benchmark/leaderboard"
|
|
|
|
localVarHeaderParams := make(map[string]string)
|
|
localVarQueryParams := url.Values{}
|
|
localVarFormParams := url.Values{}
|
|
|
|
if r.benchmark != nil {
|
|
parameterAddToHeaderOrQuery(localVarQueryParams, "benchmark", r.benchmark, "form", "")
|
|
}
|
|
// to determine the Content-Type header
|
|
localVarHTTPContentTypes := []string{}
|
|
|
|
// set Content-Type header
|
|
localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
|
|
if localVarHTTPContentType != "" {
|
|
localVarHeaderParams["Content-Type"] = localVarHTTPContentType
|
|
}
|
|
|
|
// to determine the Accept header
|
|
localVarHTTPHeaderAccepts := []string{"application/json", "application/problem+json"}
|
|
|
|
// set Accept header
|
|
localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
|
|
if localVarHTTPHeaderAccept != "" {
|
|
localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
|
|
}
|
|
req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
|
|
if err != nil {
|
|
return localVarReturnValue, nil, err
|
|
}
|
|
|
|
localVarHTTPResponse, err := a.client.callAPI(req)
|
|
if err != nil || localVarHTTPResponse == nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
|
|
localVarHTTPResponse.Body.Close()
|
|
localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
|
|
if err != nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
if localVarHTTPResponse.StatusCode >= 300 {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: localVarHTTPResponse.Status,
|
|
}
|
|
var v ProblemDetails
|
|
err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr.error = err.Error()
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
|
|
newErr.model = v
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: err.Error(),
|
|
}
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
return localVarReturnValue, localVarHTTPResponse, nil
|
|
}
|
|
|
|
type BenchmarkAPIGetBenchmarkPresetsRequest struct {
|
|
ctx context.Context
|
|
ApiService *BenchmarkAPIService
|
|
}
|
|
|
|
func (r BenchmarkAPIGetBenchmarkPresetsRequest) Execute() (*BenchmarkPresetList, *http.Response, error) {
|
|
return r.ApiService.GetBenchmarkPresetsExecute(r)
|
|
}
|
|
|
|
/*
|
|
GetBenchmarkPresets Are the router blends available to compose from — a named set of model arms, the rank they escalate through and the panel width that bounds fan-out — each named for the model layer as enso-<name>.
|
|
|
|
Are the router blends available to compose from — a named set of model
|
|
arms, the rank they escalate through and the panel width that bounds fan-out —
|
|
each named for the model layer as enso-<name>.
|
|
|
|
The first row is the reference blend: a worked example written in models we
|
|
name, published as an example of the FORM. It is deliberately not the
|
|
composition of a Hanzo-served tier — the tier name exists to abstract that — so
|
|
fork it and swap arms by what the leaderboard measures on your own tasks rather
|
|
than reading it as a disclosure.
|
|
|
|
The rows after it are the caller's org's stored blends, by name, the latest
|
|
version of each. Anyone reads the reference; only a member of the org reads the
|
|
org's own, and a SuperAdmin acting in an org it does not belong to reads what a
|
|
stranger reads.
|
|
|
|
@param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
|
|
@return BenchmarkAPIGetBenchmarkPresetsRequest
|
|
*/
|
|
func (a *BenchmarkAPIService) GetBenchmarkPresets(ctx context.Context) BenchmarkAPIGetBenchmarkPresetsRequest {
|
|
return BenchmarkAPIGetBenchmarkPresetsRequest{
|
|
ApiService: a,
|
|
ctx: ctx,
|
|
}
|
|
}
|
|
|
|
// Execute executes the request
|
|
//
|
|
// @return BenchmarkPresetList
|
|
func (a *BenchmarkAPIService) GetBenchmarkPresetsExecute(r BenchmarkAPIGetBenchmarkPresetsRequest) (*BenchmarkPresetList, *http.Response, error) {
|
|
var (
|
|
localVarHTTPMethod = http.MethodGet
|
|
localVarPostBody interface{}
|
|
formFiles []formFile
|
|
localVarReturnValue *BenchmarkPresetList
|
|
)
|
|
|
|
localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "BenchmarkAPIService.GetBenchmarkPresets")
|
|
if err != nil {
|
|
return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
|
|
}
|
|
|
|
localVarPath := localBasePath + "/v1/benchmark/presets"
|
|
|
|
localVarHeaderParams := make(map[string]string)
|
|
localVarQueryParams := url.Values{}
|
|
localVarFormParams := url.Values{}
|
|
|
|
// to determine the Content-Type header
|
|
localVarHTTPContentTypes := []string{}
|
|
|
|
// set Content-Type header
|
|
localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
|
|
if localVarHTTPContentType != "" {
|
|
localVarHeaderParams["Content-Type"] = localVarHTTPContentType
|
|
}
|
|
|
|
// to determine the Accept header
|
|
localVarHTTPHeaderAccepts := []string{"application/json", "application/problem+json"}
|
|
|
|
// set Accept header
|
|
localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
|
|
if localVarHTTPHeaderAccept != "" {
|
|
localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
|
|
}
|
|
req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
|
|
if err != nil {
|
|
return localVarReturnValue, nil, err
|
|
}
|
|
|
|
localVarHTTPResponse, err := a.client.callAPI(req)
|
|
if err != nil || localVarHTTPResponse == nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
|
|
localVarHTTPResponse.Body.Close()
|
|
localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
|
|
if err != nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
if localVarHTTPResponse.StatusCode >= 300 {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: localVarHTTPResponse.Status,
|
|
}
|
|
var v ProblemDetails
|
|
err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr.error = err.Error()
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
|
|
newErr.model = v
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: err.Error(),
|
|
}
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
return localVarReturnValue, localVarHTTPResponse, nil
|
|
}
|
|
|
|
type BenchmarkAPIPostBenchmarkClaimsRequest struct {
|
|
ctx context.Context
|
|
ApiService *BenchmarkAPIService
|
|
benchmarkPutClaimsIn *BenchmarkPutClaimsIn
|
|
}
|
|
|
|
func (r BenchmarkAPIPostBenchmarkClaimsRequest) BenchmarkPutClaimsIn(benchmarkPutClaimsIn BenchmarkPutClaimsIn) BenchmarkAPIPostBenchmarkClaimsRequest {
|
|
r.benchmarkPutClaimsIn = &benchmarkPutClaimsIn
|
|
return r
|
|
}
|
|
|
|
func (r BenchmarkAPIPostBenchmarkClaimsRequest) Execute() (*BenchmarkPutClaimsOut, *http.Response, error) {
|
|
return r.ApiService.PostBenchmarkClaimsExecute(r)
|
|
}
|
|
|
|
/*
|
|
PostBenchmarkClaims Records claims for the caller's org: one to correct a number, many to import a leaderboard.
|
|
|
|
Records claims for the caller's org: one to correct a number, many to
|
|
import a leaderboard. Any signed-in caller may write; a caller with no verified
|
|
principal is refused 401.
|
|
|
|
The org and the author are the verified caller's. Nothing in the body names
|
|
either, and a body that tries is not read. Claims are private to that org unless
|
|
visibility is "public". Writing as org "admin" curates the leaderboard, so it
|
|
takes a SuperAdmin; anyone else acting there is refused 403, and the refusal is
|
|
audited.
|
|
|
|
Every row must carry a Source, because a claim without its citation is a
|
|
number nobody can check, and a benchmark id from /catalog, because an unknown
|
|
id would sit in the store invisible to every read. A row names its benchmark,
|
|
model, provider and protocol in at most 128 characters each, cites a source of
|
|
at most 2048 bytes, and scores a percentage from 0 to 100; a row outside that is
|
|
rejected by number.
|
|
|
|
A request carries at most 500 rows in at most 1 MiB (413 past either), and an
|
|
org writes at most 2000 rows per UTC day (429 past that, and nothing from the
|
|
request is written).
|
|
|
|
The trail takes the call's intent, naming every row, BEFORE the first row
|
|
lands; a trail that cannot take it answers 503 and nothing is written. A second
|
|
record then names which rows were stored and which failed.
|
|
|
|
Writes are append-only, so this never destroys the value it replaces. A
|
|
vendor restating a score leaves both rows on disk, which is how the restating
|
|
itself becomes visible.
|
|
|
|
@param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
|
|
@return BenchmarkAPIPostBenchmarkClaimsRequest
|
|
*/
|
|
func (a *BenchmarkAPIService) PostBenchmarkClaims(ctx context.Context) BenchmarkAPIPostBenchmarkClaimsRequest {
|
|
return BenchmarkAPIPostBenchmarkClaimsRequest{
|
|
ApiService: a,
|
|
ctx: ctx,
|
|
}
|
|
}
|
|
|
|
// Execute executes the request
|
|
//
|
|
// @return BenchmarkPutClaimsOut
|
|
func (a *BenchmarkAPIService) PostBenchmarkClaimsExecute(r BenchmarkAPIPostBenchmarkClaimsRequest) (*BenchmarkPutClaimsOut, *http.Response, error) {
|
|
var (
|
|
localVarHTTPMethod = http.MethodPost
|
|
localVarPostBody interface{}
|
|
formFiles []formFile
|
|
localVarReturnValue *BenchmarkPutClaimsOut
|
|
)
|
|
|
|
localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "BenchmarkAPIService.PostBenchmarkClaims")
|
|
if err != nil {
|
|
return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
|
|
}
|
|
|
|
localVarPath := localBasePath + "/v1/benchmark/claims"
|
|
|
|
localVarHeaderParams := make(map[string]string)
|
|
localVarQueryParams := url.Values{}
|
|
localVarFormParams := url.Values{}
|
|
if r.benchmarkPutClaimsIn == nil {
|
|
return localVarReturnValue, nil, reportError("benchmarkPutClaimsIn is required and must be specified")
|
|
}
|
|
|
|
// to determine the Content-Type header
|
|
localVarHTTPContentTypes := []string{"application/json"}
|
|
|
|
// set Content-Type header
|
|
localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
|
|
if localVarHTTPContentType != "" {
|
|
localVarHeaderParams["Content-Type"] = localVarHTTPContentType
|
|
}
|
|
|
|
// to determine the Accept header
|
|
localVarHTTPHeaderAccepts := []string{"application/json", "application/problem+json"}
|
|
|
|
// set Accept header
|
|
localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
|
|
if localVarHTTPHeaderAccept != "" {
|
|
localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
|
|
}
|
|
// body params
|
|
localVarPostBody = r.benchmarkPutClaimsIn
|
|
req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
|
|
if err != nil {
|
|
return localVarReturnValue, nil, err
|
|
}
|
|
|
|
localVarHTTPResponse, err := a.client.callAPI(req)
|
|
if err != nil || localVarHTTPResponse == nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
|
|
localVarHTTPResponse.Body.Close()
|
|
localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
|
|
if err != nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
if localVarHTTPResponse.StatusCode >= 300 {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: localVarHTTPResponse.Status,
|
|
}
|
|
var v ProblemDetails
|
|
err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr.error = err.Error()
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
|
|
newErr.model = v
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: err.Error(),
|
|
}
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
return localVarReturnValue, localVarHTTPResponse, nil
|
|
}
|
|
|
|
type BenchmarkAPIPostBenchmarkPresetsRequest struct {
|
|
ctx context.Context
|
|
ApiService *BenchmarkAPIService
|
|
benchmarkPreset *BenchmarkPreset
|
|
}
|
|
|
|
func (r BenchmarkAPIPostBenchmarkPresetsRequest) BenchmarkPreset(benchmarkPreset BenchmarkPreset) BenchmarkAPIPostBenchmarkPresetsRequest {
|
|
r.benchmarkPreset = &benchmarkPreset
|
|
return r
|
|
}
|
|
|
|
func (r BenchmarkAPIPostBenchmarkPresetsRequest) Execute() (*BenchmarkPresetStored, *http.Response, error) {
|
|
return r.ApiService.PostBenchmarkPresetsExecute(r)
|
|
}
|
|
|
|
/*
|
|
PostBenchmarkPresets Stores a router blend for the caller's org — its name, its arms, the rank they escalate through and the panel fan-out width — and answers with the stored blend and the enso-<name> it would be resolved under.
|
|
|
|
Stores a router blend for the caller's org — its name, its arms, the
|
|
rank they escalate through and the panel fan-out width — and answers with the
|
|
stored blend and the enso-<name> it would be resolved under.
|
|
|
|
Storing is not serving: the model layer resolves no stored preset, and the
|
|
answer's note says so.
|
|
|
|
Any signed-in caller may compose; a caller with no verified principal is refused
|
|
401. Owner, At and By are the server's, whatever the body says, so a blend is
|
|
never stored as another org's. Posting a name the org already holds replaces
|
|
that blend; an org holds at most 100 names and stores at most 500 versions per
|
|
UTC day (429 past that).
|
|
|
|
Defaults fill the shape rather than refusing it: an omitted rank becomes the arms
|
|
in declared order and a panel below 1 becomes 1. A blend with no name or no arms
|
|
is a 400. A name is the part after enso-: 1 to 64 lowercase letters, digits and
|
|
inner hyphens, never one of Hanzo's own tiers (auto, flash, pro, ultra, free,
|
|
embed) and never beginning with enso-, or it is a 422. So is a blend of more
|
|
than 32 arms, an arm named twice or past 128 characters, a rank naming an arm
|
|
the blend does not declare, a panel wider than the rank, or a note past 1024
|
|
characters.
|
|
|
|
@param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
|
|
@return BenchmarkAPIPostBenchmarkPresetsRequest
|
|
*/
|
|
func (a *BenchmarkAPIService) PostBenchmarkPresets(ctx context.Context) BenchmarkAPIPostBenchmarkPresetsRequest {
|
|
return BenchmarkAPIPostBenchmarkPresetsRequest{
|
|
ApiService: a,
|
|
ctx: ctx,
|
|
}
|
|
}
|
|
|
|
// Execute executes the request
|
|
//
|
|
// @return BenchmarkPresetStored
|
|
func (a *BenchmarkAPIService) PostBenchmarkPresetsExecute(r BenchmarkAPIPostBenchmarkPresetsRequest) (*BenchmarkPresetStored, *http.Response, error) {
|
|
var (
|
|
localVarHTTPMethod = http.MethodPost
|
|
localVarPostBody interface{}
|
|
formFiles []formFile
|
|
localVarReturnValue *BenchmarkPresetStored
|
|
)
|
|
|
|
localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "BenchmarkAPIService.PostBenchmarkPresets")
|
|
if err != nil {
|
|
return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
|
|
}
|
|
|
|
localVarPath := localBasePath + "/v1/benchmark/presets"
|
|
|
|
localVarHeaderParams := make(map[string]string)
|
|
localVarQueryParams := url.Values{}
|
|
localVarFormParams := url.Values{}
|
|
if r.benchmarkPreset == nil {
|
|
return localVarReturnValue, nil, reportError("benchmarkPreset is required and must be specified")
|
|
}
|
|
|
|
// to determine the Content-Type header
|
|
localVarHTTPContentTypes := []string{"application/json"}
|
|
|
|
// set Content-Type header
|
|
localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
|
|
if localVarHTTPContentType != "" {
|
|
localVarHeaderParams["Content-Type"] = localVarHTTPContentType
|
|
}
|
|
|
|
// to determine the Accept header
|
|
localVarHTTPHeaderAccepts := []string{"application/json", "application/problem+json"}
|
|
|
|
// set Accept header
|
|
localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
|
|
if localVarHTTPHeaderAccept != "" {
|
|
localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
|
|
}
|
|
// body params
|
|
localVarPostBody = r.benchmarkPreset
|
|
req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
|
|
if err != nil {
|
|
return localVarReturnValue, nil, err
|
|
}
|
|
|
|
localVarHTTPResponse, err := a.client.callAPI(req)
|
|
if err != nil || localVarHTTPResponse == nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
|
|
localVarHTTPResponse.Body.Close()
|
|
localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
|
|
if err != nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
if localVarHTTPResponse.StatusCode >= 300 {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: localVarHTTPResponse.Status,
|
|
}
|
|
var v ProblemDetails
|
|
err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr.error = err.Error()
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
|
|
newErr.model = v
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: err.Error(),
|
|
}
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
return localVarReturnValue, localVarHTTPResponse, nil
|
|
}
|
|
|
|
type BenchmarkAPIPostBenchmarkRunsRequest struct {
|
|
ctx context.Context
|
|
ApiService *BenchmarkAPIService
|
|
benchmarkSuite *BenchmarkSuite
|
|
}
|
|
|
|
func (r BenchmarkAPIPostBenchmarkRunsRequest) BenchmarkSuite(benchmarkSuite BenchmarkSuite) BenchmarkAPIPostBenchmarkRunsRequest {
|
|
r.benchmarkSuite = &benchmarkSuite
|
|
return r
|
|
}
|
|
|
|
func (r BenchmarkAPIPostBenchmarkRunsRequest) Execute() (*BenchmarkMeasurements, *http.Response, error) {
|
|
return r.ApiService.PostBenchmarkRunsExecute(r)
|
|
}
|
|
|
|
/*
|
|
PostBenchmarkRuns Answers what the arena already holds for a model on each benchmark asked about, and spends nothing.
|
|
|
|
Answers what the arena already holds for a model on each benchmark asked
|
|
about, and spends nothing.
|
|
|
|
For a benchmark the store holds attempts for, the answer is the model's latest
|
|
run — its id, when it was recorded, its accuracy and how many items it covered
|
|
— served from the attempts already recorded. A measurement held is never run
|
|
again from here.
|
|
|
|
For a benchmark it holds none for, nothing is queued, because this arena has
|
|
no executor: the answer says so and names the harness that measures the
|
|
benchmark, or, where none here does, the import a published run's items take.
|
|
There is no flag to force a run, because nothing here would act on it.
|
|
|
|
Any signed-in caller may ask; a caller with no verified principal is refused
|
|
401. The answer names the caller's verified org and user — nothing in the
|
|
body names either — and stores nothing.
|
|
|
|
Validation is up front and total — a request with no model is a 400, one with
|
|
no benchmarks is a 400, and any benchmark id outside the catalog is a 422
|
|
naming exactly which ids were unknown.
|
|
|
|
@param ctx context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background().
|
|
@return BenchmarkAPIPostBenchmarkRunsRequest
|
|
*/
|
|
func (a *BenchmarkAPIService) PostBenchmarkRuns(ctx context.Context) BenchmarkAPIPostBenchmarkRunsRequest {
|
|
return BenchmarkAPIPostBenchmarkRunsRequest{
|
|
ApiService: a,
|
|
ctx: ctx,
|
|
}
|
|
}
|
|
|
|
// Execute executes the request
|
|
//
|
|
// @return BenchmarkMeasurements
|
|
func (a *BenchmarkAPIService) PostBenchmarkRunsExecute(r BenchmarkAPIPostBenchmarkRunsRequest) (*BenchmarkMeasurements, *http.Response, error) {
|
|
var (
|
|
localVarHTTPMethod = http.MethodPost
|
|
localVarPostBody interface{}
|
|
formFiles []formFile
|
|
localVarReturnValue *BenchmarkMeasurements
|
|
)
|
|
|
|
localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "BenchmarkAPIService.PostBenchmarkRuns")
|
|
if err != nil {
|
|
return localVarReturnValue, nil, &GenericOpenAPIError{error: err.Error()}
|
|
}
|
|
|
|
localVarPath := localBasePath + "/v1/benchmark/runs"
|
|
|
|
localVarHeaderParams := make(map[string]string)
|
|
localVarQueryParams := url.Values{}
|
|
localVarFormParams := url.Values{}
|
|
if r.benchmarkSuite == nil {
|
|
return localVarReturnValue, nil, reportError("benchmarkSuite is required and must be specified")
|
|
}
|
|
|
|
// to determine the Content-Type header
|
|
localVarHTTPContentTypes := []string{"application/json"}
|
|
|
|
// set Content-Type header
|
|
localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes)
|
|
if localVarHTTPContentType != "" {
|
|
localVarHeaderParams["Content-Type"] = localVarHTTPContentType
|
|
}
|
|
|
|
// to determine the Accept header
|
|
localVarHTTPHeaderAccepts := []string{"application/json", "application/problem+json"}
|
|
|
|
// set Accept header
|
|
localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts)
|
|
if localVarHTTPHeaderAccept != "" {
|
|
localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept
|
|
}
|
|
// body params
|
|
localVarPostBody = r.benchmarkSuite
|
|
req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, formFiles)
|
|
if err != nil {
|
|
return localVarReturnValue, nil, err
|
|
}
|
|
|
|
localVarHTTPResponse, err := a.client.callAPI(req)
|
|
if err != nil || localVarHTTPResponse == nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
localVarBody, err := io.ReadAll(localVarHTTPResponse.Body)
|
|
localVarHTTPResponse.Body.Close()
|
|
localVarHTTPResponse.Body = io.NopCloser(bytes.NewBuffer(localVarBody))
|
|
if err != nil {
|
|
return localVarReturnValue, localVarHTTPResponse, err
|
|
}
|
|
|
|
if localVarHTTPResponse.StatusCode >= 300 {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: localVarHTTPResponse.Status,
|
|
}
|
|
var v ProblemDetails
|
|
err = a.client.decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr.error = err.Error()
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
newErr.error = formatErrorMessage(localVarHTTPResponse.Status, &v)
|
|
newErr.model = v
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
|
|
if err != nil {
|
|
newErr := &GenericOpenAPIError{
|
|
body: localVarBody,
|
|
error: err.Error(),
|
|
}
|
|
return localVarReturnValue, localVarHTTPResponse, newErr
|
|
}
|
|
|
|
return localVarReturnValue, localVarHTTPResponse, nil
|
|
}
|