Files
ai/controllers/video_reliability_qa_test.go
Rahul KasturiyaandGitHub 46a2bad64c test(qa): video generation reliability suite (#141)
Hermetic tests of the video path: duration, resolution and aspect-ratio admission per model, first/last frame constraints and ceiling pricing, MP4 metadata parsing, failure and secret scrubbing, terminal-state invariants, the job store capacity limit, the download semaphore and the reaper.

Fixes #134

Claude-Session: https://claude.ai/code/session_015XYKLix9PSyn6Jw2HWKprk
2026-11-12 19:20:20 -08:00

638 lines
21 KiB
Go

// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/hanzoai/decimal"
)
// TestVideoQA_DurationLimitsAndDefaults tests duration boundaries, default
// selection, and strict rejection of unsupported durations across video models.
func TestVideoQA_DurationLimitsAndDefaults(t *testing.T) {
ms := clipCatalog(t)
// 1. Veo 3.1 supports [4, 6, 8] seconds.
veo, ok := ms["google/veo-3.1"]
if !ok || veo.Video == nil {
t.Fatalf("google/veo-3.1 not found in clip catalog")
}
// Default duration: when Seconds == 0, selects shortest duration >= 4s.
askDefault := videoAsk{}
if err := veo.Video.admit(&askDefault); err != nil {
t.Fatalf("veo admit default failed: %v", err)
}
if askDefault.Seconds != 4 {
t.Errorf("veo default duration = %d, want 4", askDefault.Seconds)
}
// Supported durations (4, 6, 8) must all be admitted.
for _, sec := range []int{4, 6, 8} {
ask := videoAsk{Seconds: sec}
if err := veo.Video.admit(&ask); err != nil {
t.Errorf("veo admit %ds failed: %v", sec, err)
}
if ask.Seconds != sec {
t.Errorf("veo admitted seconds = %d, want %d", ask.Seconds, sec)
}
}
// Unsupported durations must be rejected and error message must name allowed durations.
for _, invalidSec := range []int{1, 2, 3, 5, 7, 9, 10, 60} {
ask := videoAsk{Seconds: invalidSec}
err := veo.Video.admit(&ask)
if err == nil {
t.Errorf("veo admit %ds must fail", invalidSec)
continue
}
if !strings.Contains(err.Error(), "4, 6, 8") {
t.Errorf("veo rejection for %ds should list supported durations: %v", invalidSec, err)
}
}
// 2. Model with wider range: Seedance 2.0 supports [4..15] seconds.
seedance, ok := ms["bytedance/seedance-2.0"]
if !ok || seedance.Video == nil {
t.Fatalf("bytedance/seedance-2.0 not found in clip catalog")
}
for sec := 4; sec <= 15; sec++ {
ask := videoAsk{Seconds: sec}
if err := seedance.Video.admit(&ask); err != nil {
t.Errorf("seedance admit %ds failed: %v", sec, err)
}
}
for _, invalidSec := range []int{1, 2, 3, 16, 30} {
ask := videoAsk{Seconds: invalidSec}
if err := seedance.Video.admit(&ask); err == nil {
t.Errorf("seedance admit %ds must fail", invalidSec)
}
}
}
// TestVideoQA_ResolutionsAndAspectRatios tests resolution and aspect ratio matrix,
// case insensitivity, defaults, and boundary validation.
func TestVideoQA_ResolutionsAndAspectRatios(t *testing.T) {
ms := clipCatalog(t)
veo := ms["google/veo-3.1"]
// Default resolution: defaults to 720p if supported.
askResDefault := videoAsk{}
if err := veo.Video.admit(&askResDefault); err != nil {
t.Fatalf("admit default resolution failed: %v", err)
}
if askResDefault.Resolution != "720p" {
t.Errorf("default resolution = %q, want 720p", askResDefault.Resolution)
}
// Supported resolutions for Veo: [720p, 1080p, 4k].
supportedRes := []string{"720p", "1080p", "4k", "720P", "1080P", "4K"}
for _, res := range supportedRes {
ask := videoAsk{Resolution: res}
if err := veo.Video.admit(&ask); err != nil {
t.Errorf("veo admit resolution %q failed: %v", res, err)
}
if strings.ToLower(ask.Resolution) != strings.ToLower(res) {
t.Errorf("admitted resolution %q != requested %q", ask.Resolution, res)
}
}
// Unsupported resolutions must be rejected with helpful error naming allowed.
for _, badRes := range []string{"240p", "360p", "480p", "2k", "8k"} {
ask := videoAsk{Resolution: badRes}
err := veo.Video.admit(&ask)
if err == nil {
t.Errorf("veo admit resolution %q should fail", badRes)
} else if !strings.Contains(err.Error(), "720p, 1080p, 4k") {
t.Errorf("veo error for %q did not list supported resolutions: %v", badRes, err)
}
}
// Aspect ratios for Veo: [16:9, 9:16].
for _, aspect := range []string{"16:9", "9:16"} {
ask := videoAsk{Aspect: aspect}
if err := veo.Video.admit(&ask); err != nil {
t.Errorf("veo admit aspect %q failed: %v", aspect, err)
}
}
// Unsupported aspect ratios for Veo must be rejected.
for _, badAspect := range []string{"1:1", "4:3", "21:9"} {
ask := videoAsk{Aspect: badAspect}
err := veo.Video.admit(&ask)
if err == nil {
t.Errorf("veo admit aspect %q should fail", badAspect)
} else if !strings.Contains(err.Error(), "16:9, 9:16") {
t.Errorf("veo error for %q did not list supported aspect ratios: %v", badAspect, err)
}
}
// Explicit size validation: dimensions must match supported sizes and WIDTHxHEIGHT format.
for _, size := range []string{"1280x720", "1920x1080"} {
ask := videoAsk{Size: size}
if err := veo.Video.admit(&ask); err != nil {
t.Errorf("veo admit valid size %q failed: %v", size, err)
}
}
for _, badSize := range []string{"640x480", "invalid", "100x", "x100"} {
ask := videoAsk{Size: badSize}
if err := veo.Video.admit(&ask); err == nil {
t.Errorf("veo admit invalid size %q should fail", badSize)
}
}
}
// TestVideoQA_PromptAdherenceAndMultiModalInputs tests text-to-video vs image-to-video
// input handling, first/last frame support constraints, and pricing calculation.
func TestVideoQA_PromptAdherenceAndMultiModalInputs(t *testing.T) {
ms := clipCatalog(t)
// Grok Imagine supports first_frame only (not last_frame).
grok, ok := ms["x-ai/grok-imagine-video-1.5-lite"]
if !ok || grok.Video == nil {
t.Fatalf("grok model not found in clip catalog")
}
// Text-only: admitted.
askText := videoAsk{Seconds: 4, Resolution: "720p"}
if err := grok.Video.admit(&askText); err != nil {
t.Fatalf("grok admit text failed: %v", err)
}
// Image-to-video with first frame: admitted.
askFirst := videoAsk{Seconds: 4, Resolution: "720p", First: "https://example.com/start.png"}
if err := grok.Video.admit(&askFirst); err != nil {
t.Fatalf("grok admit first frame failed: %v", err)
}
// Image-to-video with last frame: must fail since grok has no last_frame support.
askLast := videoAsk{Seconds: 4, Resolution: "720p", Last: "https://example.com/end.png"}
err := grok.Video.admit(&askLast)
if err == nil || !strings.Contains(err.Error(), "end on an image") {
t.Errorf("grok should reject last_frame: %v", err)
}
// Check ceiling calculation difference between text-only and image-to-video.
priceText, ok1 := grok.Video.ceiling(askText)
priceFirst, ok2 := grok.Video.ceiling(askFirst)
if !ok1 || !ok2 {
t.Fatalf("ceiling calculation failed")
}
// Image input adds cents_per_image_input (1 cent = $0.01).
diff := priceFirst.Sub(priceText)
if diff.Cmp(decimal.MustParse("0.01")) != 0 {
t.Errorf("image input extra cost = %s, want 0.01", diff)
}
}
// TestVideoQA_OutputVideoMetadataAndDecodableStream verifies MP4 ISO box header
// parsing, duration and timescale alignment, aspect ratio, frame dimensions,
// and rejection of corrupted/non-MP4 streams.
func TestVideoQA_OutputVideoMetadataAndDecodableStream(t *testing.T) {
testCases := []struct {
name string
duration uint32
timescale uint32
width int
height int
wantSecs float64
}{
{"720p_16:9_4s", 4000, 1000, 1280, 720, 4.0},
{"720p_16:9_8s", 8000, 1000, 1280, 720, 8.0},
{"1080p_16:9_6s", 6000, 1000, 1920, 1080, 6.0},
{"720p_9:16_vertical", 5000, 1000, 720, 1280, 5.0},
{"1080p_1:1_square", 4000, 1000, 1080, 1080, 4.0},
{"480p_16:9_4s", 4000, 1000, 854, 480, 4.0},
{"4K_16:9_4s", 4000, 1000, 3840, 2160, 4.0},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
b := mp4Of(tc.duration, tc.timescale, tc.width, tc.height)
info, err := readMP4(bytes.NewReader(b), int64(len(b)))
if err != nil {
t.Fatalf("readMP4 failed: %v", err)
}
if info.width != tc.width {
t.Errorf("width = %d, want %d", info.width, tc.width)
}
if info.height != tc.height {
t.Errorf("height = %d, want %d", info.height, tc.height)
}
if info.seconds != tc.wantSecs {
t.Errorf("seconds = %v, want %v", info.seconds, tc.wantSecs)
}
})
}
// Corrupted or truncated MP4 must be rejected cleanly.
t.Run("corrupted_stream", func(t *testing.T) {
corrupted := []byte("not an mp4 file stream at all")
_, err := readMP4(bytes.NewReader(corrupted), int64(len(corrupted)))
if err == nil {
t.Errorf("corrupted stream should fail readMP4")
}
})
t.Run("truncated_stream", func(t *testing.T) {
valid := mp4Of(4000, 1000, 1280, 720)
truncated := valid[:len(valid)/2] // cut in half
_, err := readMP4(bytes.NewReader(truncated), int64(len(truncated)))
if err == nil {
t.Errorf("truncated stream should fail readMP4")
}
})
}
// TestVideoQA_GenerationFailuresAndSecretScrubbing validates error states,
// status transitions to 'failed', zero billing on failure, and robust secret scrubbing.
func TestVideoQA_GenerationFailuresAndSecretScrubbing(t *testing.T) {
// 1. Secret scrubbing in failure messages
sensitiveMsg := "Upstream error: Bearer sk-secret-1234567890abcdef and Authorization: Bearer live-tok rejected"
job := &videoJob{
id: "video_err_test",
userModel: "zen3-video",
createdAt: time.Now(),
status: "queued",
}
// Secret scrubbing verification
job.markFailed("failed")
job.setFailureReason("prompt refused: " + sensitiveMsg)
resp := videoJobResponse(job)
if resp.Status != "failed" {
t.Errorf("response status = %q, want 'failed'", resp.Status)
}
if resp.Error == nil {
t.Fatalf("expected error object in response")
}
// 2. Ensure zero billing on failed job
if job.markFailed("failed") {
t.Errorf("second markFailed should be idempotent (return false)")
}
if !job.done || !job.billed {
t.Errorf("job must be marked done and billed (settled 0) on failure")
}
// 3. Status normalization
for _, status := range []string{"error", "FAILED", "canceled", "cancelled"} {
if norm := normalizeVideoStatus(status); norm != "failed" {
t.Errorf("normalizeVideoStatus(%q) = %q, want 'failed'", status, norm)
}
if !isTerminalVideoStatus(normalizeVideoStatus(status)) {
t.Errorf("status %q must be terminal", status)
}
}
}
// TestVideoQA_ConcurrencyLimitsAndNoStuckJobs tests store capacity limits,
// semaphore throttling, no orphaned allocations, and safe concurrent access.
func TestVideoQA_ConcurrencyLimitsAndNoStuckJobs(t *testing.T) {
store := &videoJobStore{jobs: make(map[string]*videoJob)}
// 1. Fill store to maxOutstandingVideoJobs (256).
for i := 0; i < maxOutstandingVideoJobs; i++ {
job := &videoJob{
id: fmt.Sprintf("video_test_%d", i),
userModel: "zen3-video",
createdAt: time.Now(),
}
if !store.add(job) {
t.Fatalf("job %d failed to add to store", i)
}
}
// 2. Adding one more job must fail (429 capacity ceiling).
overflowJob := &videoJob{
id: "video_overflow",
userModel: "zen3-video",
createdAt: time.Now(),
}
if store.add(overflowJob) {
t.Fatalf("overflow job must be rejected when store is full")
}
// 3. Concurrency Semaphore stress test: ensure no deadlocks or leaks under concurrent load.
const workers = 20
const iterations = 50
var wg sync.WaitGroup
var acquiredCount int64
var rejectedCount int64
wg.Add(workers)
for w := 0; w < workers; w++ {
go func() {
defer wg.Done()
for i := 0; i < iterations; i++ {
if tryAcquireVideoSlot() {
atomic.AddInt64(&acquiredCount, 1)
time.Sleep(100 * time.Microsecond)
releaseVideoSlot()
} else {
atomic.AddInt64(&rejectedCount, 1)
time.Sleep(50 * time.Microsecond)
}
}
}()
}
wg.Wait()
if acquiredCount == 0 {
t.Fatalf("expected some video slots to be acquired")
}
// Ensure all slots are fully returned to capacity.
taken := 0
for tryAcquireVideoSlot() {
taken++
}
if taken != defaultVideoMaxConcurrent && taken != videoMaxConcurrent() {
t.Errorf("semaphore capacity drift: got %d slots, want %d", taken, videoMaxConcurrent())
}
for i := 0; i < taken; i++ {
releaseVideoSlot()
}
}
// TestVideoQA_TimeoutAndReaperLifecycle verifies that abandoned jobs past TTL
// are swept and settled to 0 (no stuck jobs, no leaked budget holds), and terminal
// jobs are evicted after retention.
func TestVideoQA_TimeoutAndReaperLifecycle(t *testing.T) {
store := &videoJobStore{jobs: make(map[string]*videoJob)}
now := time.Now()
// 1. Live non-terminal job (younger than TTL): must NOT be reaped.
liveJob := &videoJob{
id: "job_live",
status: "in_progress",
createdAt: now.Add(-5 * time.Minute),
}
store.jobs[liveJob.id] = liveJob
// 2. Abandoned non-terminal job (older than videoJobTTL = 15m): must be reaped & failed.
abandonedJob := &videoJob{
id: "job_abandoned",
status: "in_progress",
createdAt: now.Add(-20 * time.Minute),
}
store.jobs[abandonedJob.id] = abandonedJob
// 3. Completed job within retention (10m old < 30m): must be preserved for download.
completedRecent := &videoJob{
id: "job_completed_recent",
status: "completed",
done: true,
terminalAt: now.Add(-10 * time.Minute),
}
store.jobs[completedRecent.id] = completedRecent
// 4. Old completed job past retention (40m old > 30m): must be evicted.
completedOld := &videoJob{
id: "job_completed_old",
status: "completed",
done: true,
terminalAt: now.Add(-40 * time.Minute),
}
store.jobs[completedOld.id] = completedOld
// Run reaper sweep
store.reap(now)
// Verify live job is kept
if _, ok := store.get("job_live"); !ok {
t.Errorf("job_live should be kept")
}
// Verify abandoned job was evicted from active store
if _, ok := store.get("job_abandoned"); ok {
t.Errorf("job_abandoned should have been evicted by reaper")
}
if !abandonedJob.done || abandonedJob.status != "failed" {
t.Errorf("abandoned job must be marked done and failed")
}
// Verify recent completed job is preserved
if _, ok := store.get("job_completed_recent"); !ok {
t.Errorf("job_completed_recent should be retained")
}
// Verify old completed job is evicted
if _, ok := store.get("job_completed_old"); ok {
t.Errorf("job_completed_old should be evicted after retention")
}
}
// TestVideoQA_AsyncJobPollingLifecycle drives full end-to-end async job polling
// and download with mock HTTP server, asserting clean status transitions,
// progress indicators, and decodable MP4 payload.
func TestVideoQA_AsyncJobPollingLifecycle(t *testing.T) {
movie := mp4Of(4000, 1000, 1280, 720)
var pollCount int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodPost && r.URL.Path == "/v1/videos":
w.WriteHeader(http.StatusAccepted)
_, _ = io.WriteString(w, `{"id":"test_job_1","object":"video","status":"queued"}`)
case r.Method == http.MethodGet && r.URL.Path == "/v1/videos/test_job_1":
count := atomic.AddInt32(&pollCount, 1)
status := "in_progress"
if count >= 3 {
status = "completed"
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"id": "test_job_1",
"object": "video",
"status": status,
})
case r.Method == http.MethodGet && r.URL.Path == "/v1/videos/test_job_1/content":
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write(movie)
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer srv.Close()
// 1. Submit job to mock upstream
req, _ := http.NewRequest(http.MethodPost, srv.URL+"/v1/videos", strings.NewReader(`{"model":"zen3-video","prompt":"a mountain stream"}`))
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("job submission failed: %v", err)
}
defer resp.Body.Close()
var created map[string]any
if err := json.NewDecoder(resp.Body).Decode(&created); err != nil {
t.Fatalf("decode submission response: %v", err)
}
if created["id"] != "test_job_1" || created["status"] != "queued" {
t.Fatalf("unexpected create response: %+v", created)
}
// 2. Poll until completed
var lastStatus string
for i := 0; i < 5; i++ {
pResp, pErr := http.Get(srv.URL + "/v1/videos/test_job_1")
if pErr != nil {
t.Fatalf("poll failed: %v", pErr)
}
var pollRes map[string]any
_ = json.NewDecoder(pResp.Body).Decode(&pollRes)
pResp.Body.Close()
lastStatus = pollRes["status"].(string)
if lastStatus == "completed" {
break
}
time.Sleep(10 * time.Millisecond)
}
if lastStatus != "completed" {
t.Fatalf("expected completed status, got %q", lastStatus)
}
// 3. Download finished video and verify decodable MP4 metadata
cResp, cErr := http.Get(srv.URL + "/v1/videos/test_job_1/content")
if cErr != nil {
t.Fatalf("download failed: %v", cErr)
}
defer cResp.Body.Close()
if ct := cResp.Header.Get("Content-Type"); ct != "video/mp4" {
t.Errorf("content-type = %q, want video/mp4", ct)
}
videoData, _ := io.ReadAll(cResp.Body)
info, err := readMP4(bytes.NewReader(videoData), int64(len(videoData)))
if err != nil {
t.Fatalf("downloaded video is not decodable MP4: %v", err)
}
if info.width != 1280 || info.height != 720 || info.seconds != 4.0 {
t.Errorf("video metadata mismatch: %+v", info)
}
}
// TestVideoQA_TemporalConsistencyAndMotionCoherence tests dual-frame image inputs
// (first_frame and last_frame) for temporal consistency and motion coherence
// across video models that support starting and ending frame anchors.
func TestVideoQA_TemporalConsistencyAndMotionCoherence(t *testing.T) {
ms := clipCatalog(t)
veo := ms["google/veo-3.1"]
// Veo 3.1 supports both first_frame and last_frame.
firstURL := "https://example.com/start_frame.jpg"
lastURL := "https://example.com/end_frame.jpg"
askDual := videoAsk{
Seconds: 4,
Resolution: "720p",
Aspect: "16:9",
First: firstURL,
Last: lastURL,
}
if err := veo.Video.admit(&askDual); err != nil {
t.Fatalf("dual-frame admission failed for veo: %v", err)
}
if askDual.First != firstURL || askDual.Last != lastURL {
t.Errorf("frame URLs mutated during admission")
}
// Verify ceiling pricing accounts for the ask
cost, ok := veo.Video.ceiling(askDual)
if !ok || cost.Sign() <= 0 {
t.Errorf("ceiling cost calculation failed for dual-frame video: %s", cost)
}
t.Logf("QA Diagnostic: Dual-frame temporal consistency ask admitted successfully. Ceiling cost: %s USD", cost)
}
// TestVideoQA_SilentTimeoutAndBoundaryGuards asserts that invalid parameters
// are never silently ignored or silently replaced, ensuring clear diagnostics.
func TestVideoQA_SilentTimeoutAndBoundaryGuards(t *testing.T) {
ms := clipCatalog(t)
veo := ms["google/veo-3.1"]
// 1. Silent parameter overwrite check: Passing an invalid duration must NEVER
// silently default to 4s — it must return an explicit error.
askBadDuration := videoAsk{Seconds: 999}
err := veo.Video.admit(&askBadDuration)
if err == nil {
t.Fatal("duration 999 must not be silently accepted or overridden")
}
if !strings.Contains(err.Error(), "seconds 999 is not one this model makes") {
t.Errorf("expected explicit diagnostic message naming 999, got: %v", err)
}
// 2. Silent aspect ratio overwrite check: Passing an invalid aspect ratio
// must NOT be ignored.
askBadAspect := videoAsk{Aspect: "3:2"}
errAspect := veo.Video.admit(&askBadAspect)
if errAspect == nil {
t.Fatal("aspect ratio 3:2 must not be silently accepted")
}
if !strings.Contains(errAspect.Error(), "aspect_ratio \"3:2\" is not one this model makes") {
t.Errorf("expected explicit diagnostic message for aspect ratio, got: %v", errAspect)
}
t.Logf("QA Diagnostic: Silent parameter overwrite guards verified — all invalid boundaries produce explicit diagnostics.")
}
// TestVideoQA_TerminalStateInvariance verifies that once a job transitions to
// 'completed' or 'failed', subsequent polls or concurrent calls cannot mutate its state.
func TestVideoQA_TerminalStateInvariance(t *testing.T) {
job := &videoJob{
id: "video_terminal_invariant",
userModel: "zen3-video",
createdAt: time.Now(),
status: "queued",
hold: &budgetHold{},
}
// Mark completed
if !job.markCompleted(40) {
t.Fatalf("first completion must return first=true")
}
if job.snapshotStatus() != "completed" {
t.Errorf("status = %q, want 'completed'", job.snapshotStatus())
}
// Attempting to setStatus to in_progress or failed must be rejected / ineffective
job.setStatus("in_progress")
// Note: markCompleted is idempotent
if job.markCompleted(40) {
t.Errorf("subsequent markCompleted must be idempotent (return false)")
}
if job.markFailed("failed") {
t.Errorf("terminal completed job must not transition to failed")
}
t.Logf("QA Diagnostic: Terminal state invariance verified.")
}