Files
forge/modules/setting/actions.go
zeekayandHanzo Dev 5643098eea actions: a job no runner can ever run fails, instead of waiting forever
A job whose `runs-on` names a label that no registered runner carries is not
slow, it is unroutable — and nothing in the scheduler said so. It stayed
Waiting. CancelAbandonedJobs eventually swept it, but that decides by elapsed
time: ABANDONED_JOB_TIMEOUT defaults to a day and is checked every six hours, so
such a job occupied the queue for up to thirty hours before being quietly
cancelled.

Whether a runner carrying `macos-13` exists is knowable when the job is created
and does not become more true by waiting thirty hours to ask.

Left to time alone they accumulate. One repository whose workflow named a matrix
of platforms we own no runners for leaked two permanently-unclaimable jobs per
push; a day's worth built up, and a release build queued behind them on a
saturated fleet. They also lie while they wait: the run looks like it is about to
start, right up until it is cancelled a day later.

So FailUnsatisfiableJobs sweeps every five minutes and fails jobs no registered
runner can match, on their own run, where whoever wrote the label can see it.
Two things it deliberately does not do:

  * It tests REGISTRATION, not liveness. An offline runner is coming back and
    must keep its queue; only a label nothing on the instance claims at all is
    unsatisfiable. Failing queues during a fleet rollout would be the worse bug.

  * It ignores jobs with no labels at all. Those are unrouted, which is a
    different fault, and not something this reaper has evidence about.

UNSATISFIABLE_JOB_GRACE (15m) covers the window where a job is created before its
runner registers. The write goes through FailWaitingJobs, conditioned on the row
still being (task_id = 0, waiting), so a runner claiming the job in the same
moment simply wins and the job is absent from the returned slice.

Tests cover both directions, since a reaper that eats routable work would be
worse than the backlog it prevents: unroutable fails; routable keeps waiting no
matter how old; an offline-but-registered runner keeps its jobs; the grace period
protects young jobs; unlabelled jobs are untouched; a claimed job survives.
Each was checked against a mutant — satisfiable forced true, the grace ignored,
and the claim guard dropped — and each mutant is caught by a distinct test.

Co-authored-by: Hanzo Dev <[email protected]>
2026-07-30 17:12:51 -07:00

190 lines
6.8 KiB
Go

// Copyright 2026 Hanzo AI, Inc. All rights reserved.
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package setting
import (
"errors"
"fmt"
"strings"
"time"
"github.com/hanzoai/git/modules/log"
)
const defaultMaxRerunAttempts = 50
const defaultMaxConcurrentTaskPicks = 16
// Actions settings
var (
Actions = struct {
Enabled bool
LogStorage *Storage // how the created logs should be stored
LogRetentionDays int64 `ini:"LOG_RETENTION_DAYS"`
LogCompression logCompression `ini:"LOG_COMPRESSION"`
ArtifactStorage *Storage // how the created artifacts should be stored
ArtifactRetentionDays int64 `ini:"ARTIFACT_RETENTION_DAYS"`
DefaultActionsURL defaultActionsURL `ini:"DEFAULT_ACTIONS_URL"`
ZombieTaskTimeout time.Duration `ini:"ZOMBIE_TASK_TIMEOUT"`
EndlessTaskTimeout time.Duration `ini:"ENDLESS_TASK_TIMEOUT"`
AbandonedJobTimeout time.Duration `ini:"ABANDONED_JOB_TIMEOUT"`
// UnsatisfiableJobGrace is how long a waiting job is given to find a runner
// carrying its labels before it is failed as unroutable. It only has to cover
// the window where a job is created before its runner has registered, such as
// during a fleet rollout — it is not a queueing allowance, since a job whose
// labels nothing carries is no more satisfiable an hour later.
UnsatisfiableJobGrace time.Duration `ini:"UNSATISFIABLE_JOB_GRACE"`
SkipWorkflowStrings []string `ini:"SKIP_WORKFLOW_STRINGS"`
WorkflowDirs []string `ini:"WORKFLOW_DIRS"`
ScopedWorkflowDirs []string `ini:"SCOPED_WORKFLOW_DIRS"`
MaxRerunAttempts int64 `ini:"MAX_RERUN_ATTEMPTS"`
// MaxConcurrentTaskPicks bounds how many runners may run the task-assignment
// transaction at once per Gitea instance, to avoid a thundering herd when many
// runners poll together. It is a per-process limit, not a cluster-wide one.
MaxConcurrentTaskPicks int `ini:"MAX_CONCURRENT_TASK_PICKS"`
}{
Enabled: true,
DefaultActionsURL: defaultActionsURLGitHub,
SkipWorkflowStrings: []string{"[skip ci]", "[ci skip]", "[no ci]", "[skip actions]", "[actions skip]"},
WorkflowDirs: []string{".hanzo/workflows", ".github/workflows"},
ScopedWorkflowDirs: []string{".hanzo/scoped_workflows"},
MaxRerunAttempts: defaultMaxRerunAttempts,
MaxConcurrentTaskPicks: defaultMaxConcurrentTaskPicks,
}
)
type defaultActionsURL string
func (url defaultActionsURL) URL() string {
switch url {
case defaultActionsURLGitHub:
return "https://github.com"
case defaultActionsURLSelf:
return strings.TrimSuffix(AppURL, "/")
default:
// This should never happen, but just in case, use GitHub as fallback
return "https://github.com"
}
}
const (
defaultActionsURLGitHub = "github" // https://github.com
defaultActionsURLSelf = "self" // the root URL of the self-hosted Gitea instance
// DefaultActionsURL only supports GitHub and the self-hosted Gitea.
// It's intentionally not supported more, so please be cautious before adding more like "gitea" or "gitlab".
// If you get some trouble with `uses: username/action_name@version` in your workflow,
// please consider to use `uses: https://the_url_you_want_to_use/username/action_name@version` instead.
)
type logCompression string
func (c logCompression) IsValid() bool {
return c.IsNone() || c.IsZstd()
}
func (c logCompression) IsNone() bool {
return string(c) == "none"
}
func (c logCompression) IsZstd() bool {
return c == "" || string(c) == "zstd"
}
func loadActionsFrom(rootCfg ConfigProvider) error {
sec := rootCfg.Section("actions")
err := sec.MapTo(&Actions)
if err != nil {
return fmt.Errorf("failed to map Actions settings: %v", err)
}
if urls := string(Actions.DefaultActionsURL); urls != defaultActionsURLGitHub && urls != defaultActionsURLSelf {
url := strings.Split(urls, ",")[0]
if strings.HasPrefix(url, "https://") || strings.HasPrefix(url, "http://") {
log.Error("[actions] DEFAULT_ACTIONS_URL does not support %q as custom URL any longer, fallback to %q",
urls,
defaultActionsURLGitHub,
)
Actions.DefaultActionsURL = defaultActionsURLGitHub
} else {
return fmt.Errorf("unsupported [actions] DEFAULT_ACTIONS_URL: %q", urls)
}
}
// don't support to read configuration from [actions]
Actions.LogStorage, err = getStorage(rootCfg, "actions_log", "", nil)
if err != nil {
return err
}
// default to 1 year
if Actions.LogRetentionDays <= 0 {
Actions.LogRetentionDays = 365
}
actionsSec, _ := rootCfg.GetSection("actions.artifacts")
Actions.ArtifactStorage, err = getStorage(rootCfg, "actions_artifacts", "", actionsSec)
if err != nil {
return err
}
// default to 90 days in Github Actions
if Actions.ArtifactRetentionDays <= 0 {
Actions.ArtifactRetentionDays = 90
}
Actions.ZombieTaskTimeout = sec.Key("ZOMBIE_TASK_TIMEOUT").MustDuration(10 * time.Minute)
Actions.EndlessTaskTimeout = sec.Key("ENDLESS_TASK_TIMEOUT").MustDuration(3 * time.Hour)
Actions.AbandonedJobTimeout = sec.Key("ABANDONED_JOB_TIMEOUT").MustDuration(24 * time.Hour)
Actions.UnsatisfiableJobGrace = sec.Key("UNSATISFIABLE_JOB_GRACE").MustDuration(15 * time.Minute)
if Actions.MaxRerunAttempts <= 0 {
Actions.MaxRerunAttempts = defaultMaxRerunAttempts
}
if Actions.MaxConcurrentTaskPicks <= 0 {
Actions.MaxConcurrentTaskPicks = defaultMaxConcurrentTaskPicks
}
if !Actions.LogCompression.IsValid() {
return fmt.Errorf("invalid [actions] LOG_COMPRESSION: %q", Actions.LogCompression)
}
workflowDirs := normalizeWorkflowDirs(Actions.WorkflowDirs)
if len(workflowDirs) == 0 {
return errors.New("[actions] WORKFLOW_DIRS must contain at least one entry")
}
Actions.WorkflowDirs = workflowDirs
// SCOPED_WORKFLOW_DIRS may be empty (feature disabled), but it must not overlap with WORKFLOW_DIRS:
// a scoped dir nested in (or equal to) a workflow dir would make the same file run both repo-level and scope-level.
Actions.ScopedWorkflowDirs = normalizeWorkflowDirs(Actions.ScopedWorkflowDirs)
for _, scopedDir := range Actions.ScopedWorkflowDirs {
for _, workflowDir := range Actions.WorkflowDirs {
if scopedDir == workflowDir ||
strings.HasPrefix(scopedDir, workflowDir+"/") ||
strings.HasPrefix(workflowDir, scopedDir+"/") {
return fmt.Errorf("[actions] SCOPED_WORKFLOW_DIRS entry %q overlaps with WORKFLOW_DIRS entry %q", scopedDir, workflowDir)
}
}
}
return nil
}
// normalizeWorkflowDirs trims, normalizes separators and drops empty/trailing-slash entries.
func normalizeWorkflowDirs(dirs []string) []string {
normalized := make([]string, 0, len(dirs))
for _, dir := range dirs {
dir = strings.TrimSpace(dir)
if dir == "" {
continue
}
dir = strings.ReplaceAll(dir, `\`, `/`)
dir = strings.TrimRight(dir, "/")
normalized = append(normalized, dir)
}
return normalized
}