mirror of
https://github.com/hanzoai/git-runner.git
synced 2026-11-14 02:38:33 +00:00
The forge serves its runner surface as five typed zip operations. This is the half that calls them: zip.Dial and zip.Call by op name, so the body carries the same bytes both ends hold in memory, and the address is derived rather than spelled — the forge registers POST /v1/runner/<name>, zip derives post_runner_<name>, and TestOpNames pins our five literals against zip.ID so the two cannot drift. The types moved with the wire, and both moves were forced by it rather than chosen. A map is refused at encode, so secrets, vars, needs and a job's outputs cross as []Pair and are indexed once, at the boundary act wants a map at. A time.Time is accepted and then arrives as the zero time with no error anywhere, so every instant is an int64 of unix nanoseconds. Falling out of it: - Task.Context is a struct. As a map the forge wrote git_runtime_token while this runner read gitea_runtime_token and quietly fell back to the task token, so ACTIONS_RUNTIME_TOKEN was the wrong secret and the mask that keeps a runtime token out of a job's log covered nothing. A field is a compile error. - A transport timeout is an error again. It was read as "the designed idle path for a long poll" — fiction, the forge has never blocked — which counted it as an empty response AND reset the error counter, so a forge answering in six seconds against a five second wait was indistinguishable from an idle one in every log line and every metric, with the counter that would have raised the alarm suppressed. There is no long poll: task answers immediately and TaskIn.Queue is what makes an idle ask cost nothing. - fetch_timeout is now the ceiling on the wire. A zip.Call cannot be cancelled once it has left, so a per-request context bounds nothing and the wait has to sit on the transport, where it bounds every call. - The credential is a field, so header.go is gone. client.Forge stamps it and no caller carries it. - logs -> log, Row -> Line, NoMore -> Last, SentOutputs -> Stored. You append to a log, and the other four addresses are singular. - Insecure() had no caller. client.take copies every string and []byte out of a reply, and it is not belt-and-braces. ZAP reads text zero-copy — zap.Object.Text is an unsafe.String over the message — and zip.Call hands the response back to fasthttp's pool as it returns, so a reply's strings point at memory the next call writes over. A token registered as "runner-secret" read back "1.2.3r-secret" one call later, the version string of the call in between laid over its first five bytes, and the forge answered 401 to a runner that was registered. TestOwn fails without the copy. Delete it when zip copies on decode. client.dial declares its own transport scheme rather than taking http and https: declaring a scheme REPLACES what is registered under that name, and zip's own entries carry a Serve half this one does not — taking them left the process able to dial plain HTTP through zip and no longer able to serve it. .runner is untouched. Its uuid and token are exactly the credential the new wire sends, so a registration made before the cutover keeps working after it, and a runner only has to be rebuilt. Requires github.com/hanzoai/git at 92b2e84dba or later. Nothing here has been exercised against a live forge.
219 lines
6.6 KiB
Go
219 lines
6.6 KiB
Go
// Copyright 2026 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package metrics
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
|
|
forge "github.com/hanzoai/git/modules/actions/runner"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/collectors"
|
|
)
|
|
|
|
// Namespace is the Prometheus namespace for all git-runner metrics.
|
|
const Namespace = "git-runner"
|
|
|
|
// Label value constants for Prometheus metrics.
|
|
// Using constants prevents typos from silently creating new time-series.
|
|
//
|
|
// LabelResult* values are used on metrics with label key "result" (call outcomes).
|
|
// LabelStatus* values are used on metrics with label key "status" (job outcomes).
|
|
const (
|
|
LabelResultTask = "task"
|
|
LabelResultEmpty = "empty"
|
|
LabelResultError = "error"
|
|
LabelResultSuccess = "success"
|
|
|
|
// LabelOp* name the forge operation a call went to; they are the operation
|
|
// names themselves, so a metric and a URL cannot drift apart.
|
|
LabelOpTask = "task"
|
|
LabelOpLog = "log"
|
|
LabelOpState = "state"
|
|
|
|
LabelStatusSuccess = "success"
|
|
LabelStatusFailure = "failure"
|
|
LabelStatusCancelled = "cancelled"
|
|
LabelStatusSkipped = "skipped"
|
|
LabelStatusUnknown = "unknown"
|
|
)
|
|
|
|
// callDurationBuckets covers the expected latency range for the short logs and
|
|
// state calls. The task call has its own buckets: it long-polls and has a 10s tail.
|
|
var callDurationBuckets = []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2, 5}
|
|
|
|
// ResultToStatusLabel maps a forge.Result to the "status" label value used on job metrics.
|
|
func ResultToStatusLabel(r forge.Result) string {
|
|
switch r {
|
|
case forge.Success:
|
|
return LabelStatusSuccess
|
|
case forge.Failure:
|
|
return LabelStatusFailure
|
|
case forge.Cancelled:
|
|
return LabelStatusCancelled
|
|
case forge.Skipped:
|
|
return LabelStatusSkipped
|
|
default:
|
|
return LabelStatusUnknown
|
|
}
|
|
}
|
|
|
|
var (
|
|
RunnerInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Namespace: Namespace,
|
|
Name: "info",
|
|
Help: "Runner metadata. Always 1. Labels carry version and name.",
|
|
}, []string{"version", "name"})
|
|
|
|
RunnerCapacity = prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Namespace: Namespace,
|
|
Name: "capacity",
|
|
Help: "Configured maximum concurrent jobs.",
|
|
})
|
|
|
|
PollFetchTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "poll",
|
|
Name: "fetch_total",
|
|
Help: "Total number of task calls by result (task, empty, error).",
|
|
}, []string{"result"})
|
|
|
|
PollFetchDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "poll",
|
|
Name: "fetch_duration_seconds",
|
|
Help: "Latency of task calls, excluding expected long-poll timeouts.",
|
|
Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10},
|
|
})
|
|
|
|
PollBackoffSeconds = prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "poll",
|
|
Name: "backoff_seconds",
|
|
Help: "Last observed polling backoff interval in seconds.",
|
|
})
|
|
|
|
JobsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "job",
|
|
Name: "total",
|
|
Help: "Total jobs processed by status (success, failure, cancelled, skipped, unknown).",
|
|
}, []string{"status"})
|
|
|
|
JobDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "job",
|
|
Name: "duration_seconds",
|
|
Help: "Duration of job execution from start to finish.",
|
|
Buckets: prometheus.ExponentialBuckets(1, 2, 14), // 1s to ~4.5h
|
|
})
|
|
|
|
ReportLogTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "report",
|
|
Name: "log_total",
|
|
Help: "Total logs calls by result (success, error).",
|
|
}, []string{"result"})
|
|
|
|
ReportLogDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "report",
|
|
Name: "log_duration_seconds",
|
|
Help: "Latency of logs calls.",
|
|
Buckets: callDurationBuckets,
|
|
})
|
|
|
|
ReportStateTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "report",
|
|
Name: "state_total",
|
|
Help: "Total state calls by result (success, error).",
|
|
}, []string{"result"})
|
|
|
|
ReportStateDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "report",
|
|
Name: "state_duration_seconds",
|
|
Help: "Latency of state calls.",
|
|
Buckets: callDurationBuckets,
|
|
})
|
|
|
|
ReportLogBufferRows = prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "report",
|
|
Name: "log_buffer_rows",
|
|
Help: "Current number of buffered log rows awaiting send.",
|
|
})
|
|
|
|
ClientErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "client",
|
|
Name: "errors_total",
|
|
Help: "Total failed calls to the forge, by operation.",
|
|
}, []string{"op"})
|
|
)
|
|
|
|
// Registry is the custom Prometheus registry used by the runner.
|
|
var Registry = prometheus.NewRegistry()
|
|
|
|
var initOnce sync.Once
|
|
|
|
// Init registers all static metrics and the standard Go/process collectors.
|
|
// Safe to call multiple times; only the first call has effect.
|
|
func Init() {
|
|
initOnce.Do(func() {
|
|
Registry.MustRegister(
|
|
collectors.NewGoCollector(),
|
|
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
|
RunnerInfo, RunnerCapacity,
|
|
PollFetchTotal, PollFetchDuration, PollBackoffSeconds,
|
|
JobsTotal, JobDuration,
|
|
ReportLogTotal, ReportLogDuration,
|
|
ReportStateTotal, ReportStateDuration, ReportLogBufferRows,
|
|
ClientErrors,
|
|
)
|
|
})
|
|
}
|
|
|
|
// RegisterUptimeFunc registers a GaugeFunc that reports seconds since startTime.
|
|
func RegisterUptimeFunc(startTime time.Time) {
|
|
Registry.MustRegister(prometheus.NewGaugeFunc(
|
|
prometheus.GaugeOpts{
|
|
Namespace: Namespace,
|
|
Name: "uptime_seconds",
|
|
Help: "Seconds since the runner daemon started.",
|
|
},
|
|
func() float64 { return time.Since(startTime).Seconds() },
|
|
))
|
|
}
|
|
|
|
// RegisterRunningJobsFunc registers GaugeFuncs for the running job count and
|
|
// capacity utilisation ratio, evaluated lazily at Prometheus scrape time.
|
|
func RegisterRunningJobsFunc(countFn func() int64, capacity int) {
|
|
capF := float64(capacity)
|
|
Registry.MustRegister(prometheus.NewGaugeFunc(
|
|
prometheus.GaugeOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "job",
|
|
Name: "running",
|
|
Help: "Number of jobs currently executing.",
|
|
},
|
|
func() float64 { return float64(countFn()) },
|
|
))
|
|
Registry.MustRegister(prometheus.NewGaugeFunc(
|
|
prometheus.GaugeOpts{
|
|
Namespace: Namespace,
|
|
Subsystem: "job",
|
|
Name: "capacity_utilization_ratio",
|
|
Help: "Ratio of running jobs to configured capacity (0.0-1.0).",
|
|
},
|
|
func() float64 {
|
|
if capF <= 0 {
|
|
return 0
|
|
}
|
|
return float64(countFn()) / capF
|
|
},
|
|
))
|
|
}
|