mirror of
https://github.com/hanzoai/git-runner.git
synced 2026-11-14 02:38:33 +00:00
The forge serves its runner surface as five typed zip operations. This is the half that calls them: zip.Dial and zip.Call by op name, so the body carries the same bytes both ends hold in memory, and the address is derived rather than spelled — the forge registers POST /v1/runner/<name>, zip derives post_runner_<name>, and TestOpNames pins our five literals against zip.ID so the two cannot drift. The types moved with the wire, and both moves were forced by it rather than chosen. A map is refused at encode, so secrets, vars, needs and a job's outputs cross as []Pair and are indexed once, at the boundary act wants a map at. A time.Time is accepted and then arrives as the zero time with no error anywhere, so every instant is an int64 of unix nanoseconds. Falling out of it: - Task.Context is a struct. As a map the forge wrote git_runtime_token while this runner read gitea_runtime_token and quietly fell back to the task token, so ACTIONS_RUNTIME_TOKEN was the wrong secret and the mask that keeps a runtime token out of a job's log covered nothing. A field is a compile error. - A transport timeout is an error again. It was read as "the designed idle path for a long poll" — fiction, the forge has never blocked — which counted it as an empty response AND reset the error counter, so a forge answering in six seconds against a five second wait was indistinguishable from an idle one in every log line and every metric, with the counter that would have raised the alarm suppressed. There is no long poll: task answers immediately and TaskIn.Queue is what makes an idle ask cost nothing. - fetch_timeout is now the ceiling on the wire. A zip.Call cannot be cancelled once it has left, so a per-request context bounds nothing and the wait has to sit on the transport, where it bounds every call. - The credential is a field, so header.go is gone. client.Forge stamps it and no caller carries it. - logs -> log, Row -> Line, NoMore -> Last, SentOutputs -> Stored. You append to a log, and the other four addresses are singular. - Insecure() had no caller. client.take copies every string and []byte out of a reply, and it is not belt-and-braces. ZAP reads text zero-copy — zap.Object.Text is an unsafe.String over the message — and zip.Call hands the response back to fasthttp's pool as it returns, so a reply's strings point at memory the next call writes over. A token registered as "runner-secret" read back "1.2.3r-secret" one call later, the version string of the call in between laid over its first five bytes, and the forge answered 401 to a runner that was registered. TestOwn fails without the copy. Delete it when zip copies on decode. client.dial declares its own transport scheme rather than taking http and https: declaring a scheme REPLACES what is registered under that name, and zip's own entries carry a Serve half this one does not — taking them left the process able to dial plain HTTP through zip and no longer able to serve it. .runner is untouched. Its uuid and token are exactly the credential the new wire sends, so a registration made before the cutover keeps working after it, and a runner only has to be rebuilt. Requires github.com/hanzoai/git at 92b2e84dba or later. Nothing here has been exercised against a live forge.
285 lines
9.1 KiB
Go
285 lines
9.1 KiB
Go
// Copyright 2026 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package poll
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/git-runner/internal/pkg/client/mocks"
|
|
"github.com/hanzoai/git-runner/internal/pkg/config"
|
|
|
|
forge "github.com/hanzoai/git/modules/actions/runner"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/mock"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// TestPoller_WorkerStateCounters verifies that workerState correctly tracks
|
|
// consecutive empty responses independently per state instance, and that
|
|
// fetchTask increments only the relevant counter.
|
|
func TestPoller_WorkerStateCounters(t *testing.T) {
|
|
client := mocks.NewClient(t)
|
|
client.On("Task", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, _ *forge.TaskIn) (*forge.TaskOut, error) {
|
|
// Always answer with no work.
|
|
return &forge.TaskOut{}, nil
|
|
},
|
|
)
|
|
|
|
cfg, err := config.LoadDefault("")
|
|
require.NoError(t, err)
|
|
p := &Poller{client: client, cfg: cfg}
|
|
|
|
ctx := context.Background()
|
|
s1 := &workerState{}
|
|
s2 := &workerState{}
|
|
|
|
// Each worker independently observes one empty response.
|
|
_, ok := p.fetchTask(ctx, s1)
|
|
require.False(t, ok)
|
|
_, ok = p.fetchTask(ctx, s2)
|
|
require.False(t, ok)
|
|
|
|
assert.Equal(t, int64(1), s1.consecutiveEmpty, "worker 1 should only count its own empty response")
|
|
assert.Equal(t, int64(1), s2.consecutiveEmpty, "worker 2 should only count its own empty response")
|
|
|
|
// Worker 1 sees a second empty; worker 2 stays at 1.
|
|
_, ok = p.fetchTask(ctx, s1)
|
|
require.False(t, ok)
|
|
assert.Equal(t, int64(2), s1.consecutiveEmpty)
|
|
assert.Equal(t, int64(1), s2.consecutiveEmpty, "worker 2's counter must not be affected by worker 1's empty fetches")
|
|
}
|
|
|
|
// TestPoller_FetchErrorIncrementsErrorsOnly verifies that a fetch error
|
|
// increments only the per-worker error counter, not the empty counter.
|
|
func TestPoller_FetchErrorIncrementsErrorsOnly(t *testing.T) {
|
|
client := mocks.NewClient(t)
|
|
client.On("Task", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, _ *forge.TaskIn) (*forge.TaskOut, error) {
|
|
return nil, errors.New("network unreachable")
|
|
},
|
|
)
|
|
|
|
cfg, err := config.LoadDefault("")
|
|
require.NoError(t, err)
|
|
p := &Poller{client: client, cfg: cfg}
|
|
|
|
s := &workerState{}
|
|
_, ok := p.fetchTask(context.Background(), s)
|
|
require.False(t, ok)
|
|
assert.Equal(t, int64(1), s.consecutiveErrors)
|
|
assert.Equal(t, int64(0), s.consecutiveEmpty)
|
|
}
|
|
|
|
// TestPoller_TimeoutIsAnError pins the difference between a forge with nothing
|
|
// to give and a forge that did not answer.
|
|
//
|
|
// ⛔ These were once the same. A deadline was read as "the designed idle path
|
|
// for a long poll" — fiction, the forge has never blocked — so it counted as an
|
|
// empty response AND reset the error counter. A forge answering in six seconds
|
|
// against a five second wait was therefore indistinguishable from an idle one
|
|
// in every log line and every metric, while suppressing the very counter that
|
|
// would have raised an alarm.
|
|
func TestPoller_TimeoutIsAnError(t *testing.T) {
|
|
client := mocks.NewClient(t)
|
|
client.On("Task", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, _ *forge.TaskIn) (*forge.TaskOut, error) {
|
|
return nil, fmt.Errorf("call post_runner_task: %w", context.DeadlineExceeded)
|
|
},
|
|
)
|
|
|
|
cfg, err := config.LoadDefault("")
|
|
require.NoError(t, err)
|
|
p := &Poller{client: client, cfg: cfg}
|
|
|
|
s := &workerState{}
|
|
_, ok := p.fetchTask(context.Background(), s)
|
|
require.False(t, ok)
|
|
assert.Equal(t, int64(1), s.consecutiveErrors, "a forge that did not answer is an error")
|
|
assert.Equal(t, int64(0), s.consecutiveEmpty, "and it is not an idle forge")
|
|
}
|
|
|
|
// TestPoller_CalculateInterval verifies the exponential backoff math is
|
|
// correctly driven by the workerState counters.
|
|
func TestPoller_CalculateInterval(t *testing.T) {
|
|
cfg, err := config.LoadDefault("")
|
|
require.NoError(t, err)
|
|
cfg.Runner.FetchInterval = 2 * time.Second
|
|
cfg.Runner.FetchIntervalMax = 60 * time.Second
|
|
p := &Poller{cfg: cfg}
|
|
|
|
cases := []struct {
|
|
name string
|
|
empty, errs int64
|
|
wantInterval time.Duration
|
|
}{
|
|
{"first poll, no backoff", 0, 0, 2 * time.Second},
|
|
{"single empty, still base", 1, 0, 2 * time.Second},
|
|
{"two empties, doubled", 2, 0, 4 * time.Second},
|
|
{"five empties, capped path", 5, 0, 32 * time.Second},
|
|
{"many empties, capped at max", 20, 0, 60 * time.Second},
|
|
{"errors drive backoff too", 0, 3, 8 * time.Second},
|
|
{"max(empty, errors) wins", 2, 4, 16 * time.Second},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s := &workerState{consecutiveEmpty: tc.empty, consecutiveErrors: tc.errs}
|
|
assert.Equal(t, tc.wantInterval, p.calculateInterval(s))
|
|
})
|
|
}
|
|
}
|
|
|
|
// atomicMax atomically updates target to max(target, val).
|
|
func atomicMax(target *atomic.Int64, val int64) {
|
|
for {
|
|
old := target.Load()
|
|
if val <= old || target.CompareAndSwap(old, val) {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
type mockRunner struct {
|
|
delay time.Duration
|
|
running atomic.Int64
|
|
maxConcurrent atomic.Int64
|
|
totalCompleted atomic.Int64
|
|
}
|
|
|
|
func (m *mockRunner) Run(ctx context.Context, _ *forge.Task) error {
|
|
atomicMax(&m.maxConcurrent, m.running.Add(1))
|
|
select {
|
|
case <-time.After(m.delay):
|
|
case <-ctx.Done():
|
|
}
|
|
m.running.Add(-1)
|
|
m.totalCompleted.Add(1)
|
|
return nil
|
|
}
|
|
|
|
// TestPoller_ConcurrencyLimitedByCapacity verifies that with capacity=3 and
|
|
// 6 available tasks, at most 3 tasks run concurrently, and FetchTask is
|
|
// never called concurrently (single poller).
|
|
func TestPoller_ConcurrencyLimitedByCapacity(t *testing.T) {
|
|
const (
|
|
capacity = 3
|
|
totalTasks = 6
|
|
taskDelay = 50 * time.Millisecond
|
|
)
|
|
|
|
var (
|
|
tasksReturned atomic.Int64
|
|
fetchConcur atomic.Int64
|
|
maxFetchConcur atomic.Int64
|
|
)
|
|
|
|
cli := mocks.NewClient(t)
|
|
cli.On("Task", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, _ *forge.TaskIn) (*forge.TaskOut, error) {
|
|
atomicMax(&maxFetchConcur, fetchConcur.Add(1))
|
|
defer fetchConcur.Add(-1)
|
|
|
|
n := tasksReturned.Add(1)
|
|
if n <= totalTasks {
|
|
return &forge.TaskOut{Task: &forge.Task{ID: n}}, nil
|
|
}
|
|
return &forge.TaskOut{}, nil
|
|
},
|
|
)
|
|
|
|
runner := &mockRunner{delay: taskDelay}
|
|
|
|
cfg, err := config.LoadDefault("")
|
|
require.NoError(t, err)
|
|
cfg.Runner.Capacity = capacity
|
|
cfg.Runner.FetchInterval = 10 * time.Millisecond
|
|
cfg.Runner.FetchIntervalMax = 10 * time.Millisecond
|
|
|
|
poller := New(cfg, cli, runner)
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Go(poller.Poll)
|
|
|
|
require.Eventually(t, func() bool {
|
|
return runner.totalCompleted.Load() >= totalTasks
|
|
}, 2*time.Second, 10*time.Millisecond, "all tasks should complete")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
err = poller.Shutdown(ctx)
|
|
require.NoError(t, err)
|
|
wg.Wait()
|
|
|
|
assert.LessOrEqual(t, runner.maxConcurrent.Load(), int64(capacity),
|
|
"concurrent running tasks must not exceed capacity")
|
|
assert.GreaterOrEqual(t, runner.maxConcurrent.Load(), int64(2),
|
|
"with 6 tasks and capacity 3, at least 2 should overlap")
|
|
assert.Equal(t, int64(1), maxFetchConcur.Load(),
|
|
"the task operation must never be called concurrently (single poller)")
|
|
assert.Equal(t, int64(totalTasks), runner.totalCompleted.Load(),
|
|
"all tasks should have been executed")
|
|
}
|
|
|
|
// TestPoller_ShutdownForcesJobsOnTimeout locks in the fix for a
|
|
// pre-existing bug where Shutdown's timeout branch used a blocking
|
|
// `<-p.done` receive, leaving p.shutdownJobs() unreachable. With a
|
|
// task parked on jobsCtx and a Shutdown deadline shorter than the
|
|
// task's natural completion, Shutdown must force-cancel via
|
|
// shutdownJobs() and return ctx.Err() promptly — not block until the
|
|
// task would have finished on its own.
|
|
func TestPoller_ShutdownForcesJobsOnTimeout(t *testing.T) {
|
|
var served atomic.Bool
|
|
cli := mocks.NewClient(t)
|
|
cli.On("Task", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, _ *forge.TaskIn) (*forge.TaskOut, error) {
|
|
if served.CompareAndSwap(false, true) {
|
|
return &forge.TaskOut{Task: &forge.Task{ID: 1}}, nil
|
|
}
|
|
return &forge.TaskOut{}, nil
|
|
},
|
|
)
|
|
|
|
// delay >> Shutdown timeout: Run only returns when jobsCtx is
|
|
// cancelled by shutdownJobs().
|
|
runner := &mockRunner{delay: 30 * time.Second}
|
|
|
|
cfg, err := config.LoadDefault("")
|
|
require.NoError(t, err)
|
|
cfg.Runner.Capacity = 1
|
|
cfg.Runner.FetchInterval = 10 * time.Millisecond
|
|
cfg.Runner.FetchIntervalMax = 10 * time.Millisecond
|
|
|
|
poller := New(cfg, cli, runner)
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Go(poller.Poll)
|
|
|
|
require.Eventually(t, func() bool {
|
|
return runner.running.Load() == 1
|
|
}, time.Second, 10*time.Millisecond, "task should start running")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
|
defer cancel()
|
|
start := time.Now()
|
|
err = poller.Shutdown(ctx)
|
|
elapsed := time.Since(start)
|
|
|
|
require.ErrorIs(t, err, context.DeadlineExceeded)
|
|
// With the fix, Shutdown returns shortly after the deadline once
|
|
// the forced job unwinds. Without the fix, the blocking <-p.done
|
|
// would hang for the full 30s mockRunner delay.
|
|
assert.Less(t, elapsed, 5*time.Second,
|
|
"Shutdown must not block on the parked task; shutdownJobs() must run on timeout")
|
|
|
|
wg.Wait()
|
|
assert.Equal(t, int64(1), runner.totalCompleted.Load(),
|
|
"the parked task must be cancelled and unwound")
|
|
}
|