mirror of
https://github.com/hanzoai/git-runner.git
synced 2026-11-14 02:38:33 +00:00
The forge serves its runner surface as five typed zip operations. This is the half that calls them: zip.Dial and zip.Call by op name, so the body carries the same bytes both ends hold in memory, and the address is derived rather than spelled — the forge registers POST /v1/runner/<name>, zip derives post_runner_<name>, and TestOpNames pins our five literals against zip.ID so the two cannot drift. The types moved with the wire, and both moves were forced by it rather than chosen. A map is refused at encode, so secrets, vars, needs and a job's outputs cross as []Pair and are indexed once, at the boundary act wants a map at. A time.Time is accepted and then arrives as the zero time with no error anywhere, so every instant is an int64 of unix nanoseconds. Falling out of it: - Task.Context is a struct. As a map the forge wrote git_runtime_token while this runner read gitea_runtime_token and quietly fell back to the task token, so ACTIONS_RUNTIME_TOKEN was the wrong secret and the mask that keeps a runtime token out of a job's log covered nothing. A field is a compile error. - A transport timeout is an error again. It was read as "the designed idle path for a long poll" — fiction, the forge has never blocked — which counted it as an empty response AND reset the error counter, so a forge answering in six seconds against a five second wait was indistinguishable from an idle one in every log line and every metric, with the counter that would have raised the alarm suppressed. There is no long poll: task answers immediately and TaskIn.Queue is what makes an idle ask cost nothing. - fetch_timeout is now the ceiling on the wire. A zip.Call cannot be cancelled once it has left, so a per-request context bounds nothing and the wait has to sit on the transport, where it bounds every call. - The credential is a field, so header.go is gone. client.Forge stamps it and no caller carries it. - logs -> log, Row -> Line, NoMore -> Last, SentOutputs -> Stored. You append to a log, and the other four addresses are singular. - Insecure() had no caller. client.take copies every string and []byte out of a reply, and it is not belt-and-braces. ZAP reads text zero-copy — zap.Object.Text is an unsafe.String over the message — and zip.Call hands the response back to fasthttp's pool as it returns, so a reply's strings point at memory the next call writes over. A token registered as "runner-secret" read back "1.2.3r-secret" one call later, the version string of the call in between laid over its first five bytes, and the forge answered 401 to a runner that was registered. TestOwn fails without the copy. Delete it when zip copies on decode. client.dial declares its own transport scheme rather than taking http and https: declaring a scheme REPLACES what is registered under that name, and zip's own entries carry a Serve half this one does not — taking them left the process able to dial plain HTTP through zip and no longer able to serve it. .runner is untouched. Its uuid and token are exactly the credential the new wire sends, so a registration made before the cutover keeps working after it, and a runner only has to be rebuilt. Requires github.com/hanzoai/git at 92b2e84dba or later. Nothing here has been exercised against a live forge.
276 lines
6.8 KiB
Go
276 lines
6.8 KiB
Go
// Copyright 2023 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package poll
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math/rand/v2"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/hanzoai/git-runner/internal/pkg/client"
|
|
"github.com/hanzoai/git-runner/internal/pkg/config"
|
|
"github.com/hanzoai/git-runner/internal/pkg/metrics"
|
|
|
|
forge "github.com/hanzoai/git/modules/actions/runner"
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// TaskRunner abstracts task execution so the poller can be tested
|
|
// without a real runner.
|
|
type TaskRunner interface {
|
|
Run(ctx context.Context, task *forge.Task) error
|
|
}
|
|
|
|
type Poller struct {
|
|
client client.Client
|
|
runner TaskRunner
|
|
cfg *config.Config
|
|
queue atomic.Int64 // the queue version this runner last saw; see fetchTask.
|
|
|
|
pollingCtx context.Context
|
|
shutdownPolling context.CancelFunc
|
|
|
|
jobsCtx context.Context
|
|
shutdownJobs context.CancelFunc
|
|
|
|
done chan struct{}
|
|
}
|
|
|
|
// workerState holds the single poller's backoff state. Consecutive empty or
|
|
// error responses drive exponential backoff; a successful task fetch resets
|
|
// both counters so the next poll fires immediately.
|
|
type workerState struct {
|
|
consecutiveEmpty int64
|
|
consecutiveErrors int64
|
|
// lastBackoff is the last interval reported to the PollBackoffSeconds gauge;
|
|
// used to suppress redundant no-op Set calls when the backoff plateaus
|
|
// (e.g. at FetchIntervalMax).
|
|
lastBackoff time.Duration
|
|
}
|
|
|
|
func New(cfg *config.Config, client client.Client, runner TaskRunner) *Poller {
|
|
pollingCtx, shutdownPolling := context.WithCancel(context.Background())
|
|
|
|
jobsCtx, shutdownJobs := context.WithCancel(context.Background())
|
|
|
|
done := make(chan struct{})
|
|
|
|
return &Poller{
|
|
client: client,
|
|
runner: runner,
|
|
cfg: cfg,
|
|
|
|
pollingCtx: pollingCtx,
|
|
shutdownPolling: shutdownPolling,
|
|
|
|
jobsCtx: jobsCtx,
|
|
shutdownJobs: shutdownJobs,
|
|
|
|
done: done,
|
|
}
|
|
}
|
|
|
|
func (p *Poller) Poll() {
|
|
sem := make(chan struct{}, p.cfg.Runner.Capacity)
|
|
wg := &sync.WaitGroup{}
|
|
s := &workerState{}
|
|
|
|
defer func() {
|
|
wg.Wait()
|
|
close(p.done)
|
|
}()
|
|
|
|
for {
|
|
select {
|
|
case sem <- struct{}{}:
|
|
case <-p.pollingCtx.Done():
|
|
return
|
|
}
|
|
|
|
task, ok := p.fetchTask(p.pollingCtx, s)
|
|
if !ok {
|
|
<-sem
|
|
if !p.waitBackoff(s) {
|
|
return
|
|
}
|
|
continue
|
|
}
|
|
|
|
s.resetBackoff()
|
|
|
|
wg.Add(1)
|
|
go func(t *forge.Task) {
|
|
defer wg.Done()
|
|
defer func() { <-sem }()
|
|
p.runTaskWithRecover(p.jobsCtx, t)
|
|
}(task)
|
|
}
|
|
}
|
|
|
|
func (p *Poller) PollOnce() {
|
|
defer close(p.done)
|
|
s := &workerState{}
|
|
for {
|
|
task, ok := p.fetchTask(p.pollingCtx, s)
|
|
if !ok {
|
|
if !p.waitBackoff(s) {
|
|
return
|
|
}
|
|
continue
|
|
}
|
|
s.resetBackoff()
|
|
p.runTaskWithRecover(p.jobsCtx, task)
|
|
return
|
|
}
|
|
}
|
|
|
|
func (p *Poller) Shutdown(ctx context.Context) error {
|
|
p.shutdownPolling()
|
|
|
|
select {
|
|
// graceful shutdown completed succesfully
|
|
case <-p.done:
|
|
return nil
|
|
|
|
// our timeout for shutting down ran out
|
|
case <-ctx.Done():
|
|
// Both the timeout and the graceful shutdown may fire
|
|
// simultaneously. Do a non-blocking check to avoid forcing
|
|
// a shutdown when graceful already completed.
|
|
select {
|
|
case <-p.done:
|
|
return nil
|
|
default:
|
|
}
|
|
|
|
// force a shutdown of all running jobs
|
|
p.shutdownJobs()
|
|
|
|
// wait for running jobs to report their status to the server
|
|
<-p.done
|
|
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (s *workerState) resetBackoff() {
|
|
s.consecutiveEmpty = 0
|
|
s.consecutiveErrors = 0
|
|
s.lastBackoff = 0
|
|
}
|
|
|
|
// waitBackoff sleeps for the current backoff interval (with jitter).
|
|
// Returns false if the polling context was cancelled during the wait.
|
|
func (p *Poller) waitBackoff(s *workerState) bool {
|
|
base := p.calculateInterval(s)
|
|
if base != s.lastBackoff {
|
|
metrics.PollBackoffSeconds.Set(base.Seconds())
|
|
s.lastBackoff = base
|
|
}
|
|
timer := time.NewTimer(addJitter(base))
|
|
select {
|
|
case <-timer.C:
|
|
return true
|
|
case <-p.pollingCtx.Done():
|
|
timer.Stop()
|
|
return false
|
|
}
|
|
}
|
|
|
|
// calculateInterval returns the polling interval with exponential backoff based on
|
|
// consecutive empty or error responses. The interval starts at FetchInterval and
|
|
// doubles with each consecutive empty/error, capped at FetchIntervalMax.
|
|
func (p *Poller) calculateInterval(s *workerState) time.Duration {
|
|
base := p.cfg.Runner.FetchInterval
|
|
maxInterval := p.cfg.Runner.FetchIntervalMax
|
|
|
|
n := max(s.consecutiveEmpty, s.consecutiveErrors)
|
|
if n <= 1 {
|
|
return base
|
|
}
|
|
|
|
// Capped exponential backoff: base * 2^(n-1), max shift=5 so multiplier <= 32
|
|
shift := min(n-1, 5)
|
|
interval := base * time.Duration(int64(1)<<shift)
|
|
return min(interval, maxInterval)
|
|
}
|
|
|
|
// addJitter adds +/- 20% random jitter to the given duration to avoid thundering herd.
|
|
func addJitter(d time.Duration) time.Duration {
|
|
if d <= 0 {
|
|
return d
|
|
}
|
|
// jitter range: [-20%, +20%] of d
|
|
jitterRange := int64(d) * 2 / 5 // 40% total range
|
|
if jitterRange <= 0 {
|
|
return d
|
|
}
|
|
jitter := rand.Int64N(jitterRange) - jitterRange/2
|
|
return d + time.Duration(jitter)
|
|
}
|
|
|
|
func (p *Poller) runTaskWithRecover(ctx context.Context, task *forge.Task) {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
err := fmt.Errorf("panic: %v", r)
|
|
log.WithError(err).Error("panic in runTaskWithRecover")
|
|
}
|
|
}()
|
|
|
|
if err := p.runner.Run(ctx, task); err != nil {
|
|
log.WithError(err).Error("failed to run task")
|
|
}
|
|
}
|
|
|
|
// fetchTask asks the forge for work once. The forge answers immediately,
|
|
// always: queue is the version the runner last saw, and a forge whose version
|
|
// still matches has queued nothing since and answers without opening the
|
|
// assignment transaction. Cadence is this poller's business — see
|
|
// calculateInterval — and the wait for an answer is the wire's, so a forge that
|
|
// takes too long produces an error here rather than a silence that reads like
|
|
// an idle forge.
|
|
func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*forge.Task, bool) {
|
|
// Load the version value that was in the cache when the request was sent.
|
|
v := p.queue.Load()
|
|
start := time.Now()
|
|
out, err := p.client.Task(ctx, &forge.TaskIn{Queue: v})
|
|
metrics.PollFetchDuration.Observe(time.Since(start).Seconds())
|
|
|
|
if err != nil {
|
|
log.WithError(err).Error("failed to fetch task")
|
|
s.consecutiveErrors++
|
|
metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultError).Inc()
|
|
metrics.ClientErrors.WithLabelValues(metrics.LabelOpTask).Inc()
|
|
return nil, false
|
|
}
|
|
|
|
// Successful response — reset error counter.
|
|
s.consecutiveErrors = 0
|
|
|
|
if out == nil {
|
|
s.consecutiveEmpty++
|
|
metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultEmpty).Inc()
|
|
return nil, false
|
|
}
|
|
|
|
if out.Queue > v {
|
|
p.queue.CompareAndSwap(v, out.Queue)
|
|
}
|
|
|
|
if out.Task == nil {
|
|
s.consecutiveEmpty++
|
|
metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultEmpty).Inc()
|
|
return nil, false
|
|
}
|
|
|
|
// got a task, so forget the queue version: the next request must look.
|
|
p.queue.CompareAndSwap(out.Queue, 0)
|
|
|
|
metrics.PollFetchTotal.WithLabelValues(metrics.LabelResultTask).Inc()
|
|
return out.Task, true
|
|
}
|