mirror of
https://github.com/hanzoai/git-runner.git
synced 2026-11-14 02:38:33 +00:00
The forge serves its runner surface as five typed zip operations. This is the half that calls them: zip.Dial and zip.Call by op name, so the body carries the same bytes both ends hold in memory, and the address is derived rather than spelled — the forge registers POST /v1/runner/<name>, zip derives post_runner_<name>, and TestOpNames pins our five literals against zip.ID so the two cannot drift. The types moved with the wire, and both moves were forced by it rather than chosen. A map is refused at encode, so secrets, vars, needs and a job's outputs cross as []Pair and are indexed once, at the boundary act wants a map at. A time.Time is accepted and then arrives as the zero time with no error anywhere, so every instant is an int64 of unix nanoseconds. Falling out of it: - Task.Context is a struct. As a map the forge wrote git_runtime_token while this runner read gitea_runtime_token and quietly fell back to the task token, so ACTIONS_RUNTIME_TOKEN was the wrong secret and the mask that keeps a runtime token out of a job's log covered nothing. A field is a compile error. - A transport timeout is an error again. It was read as "the designed idle path for a long poll" — fiction, the forge has never blocked — which counted it as an empty response AND reset the error counter, so a forge answering in six seconds against a five second wait was indistinguishable from an idle one in every log line and every metric, with the counter that would have raised the alarm suppressed. There is no long poll: task answers immediately and TaskIn.Queue is what makes an idle ask cost nothing. - fetch_timeout is now the ceiling on the wire. A zip.Call cannot be cancelled once it has left, so a per-request context bounds nothing and the wait has to sit on the transport, where it bounds every call. - The credential is a field, so header.go is gone. client.Forge stamps it and no caller carries it. - logs -> log, Row -> Line, NoMore -> Last, SentOutputs -> Stored. You append to a log, and the other four addresses are singular. - Insecure() had no caller. client.take copies every string and []byte out of a reply, and it is not belt-and-braces. ZAP reads text zero-copy — zap.Object.Text is an unsafe.String over the message — and zip.Call hands the response back to fasthttp's pool as it returns, so a reply's strings point at memory the next call writes over. A token registered as "runner-secret" read back "1.2.3r-secret" one call later, the version string of the call in between laid over its first five bytes, and the forge answered 401 to a runner that was registered. TestOwn fails without the copy. Delete it when zip copies on decode. client.dial declares its own transport scheme rather than taking http and https: declaring a scheme REPLACES what is registered under that name, and zip's own entries carry a Serve half this one does not — taking them left the process able to dial plain HTTP through zip and no longer able to serve it. .runner is untouched. Its uuid and token are exactly the credential the new wire sends, so a registration made before the cutover keeps working after it, and a runner only has to be rebuilt. Requires github.com/hanzoai/git at 92b2e84dba or later. Nothing here has been exercised against a live forge.
605 lines
20 KiB
Go
605 lines
20 KiB
Go
// Copyright 2023 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package report
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/git-runner/internal/pkg/client/mocks"
|
|
"github.com/hanzoai/git-runner/internal/pkg/config"
|
|
|
|
forge "github.com/hanzoai/git/modules/actions/runner"
|
|
log "github.com/sirupsen/logrus"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/mock"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestReporter_parseLine(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
debugOutputEnabled bool
|
|
args []string
|
|
want []string
|
|
}{
|
|
{
|
|
"No command", false,
|
|
[]string{"Hello, world!"},
|
|
[]string{"Hello, world!"},
|
|
},
|
|
{
|
|
"Add-mask", false,
|
|
[]string{
|
|
"foo mysecret bar",
|
|
"::add-mask::mysecret",
|
|
"foo mysecret bar",
|
|
},
|
|
[]string{
|
|
"foo mysecret bar",
|
|
"<nil>",
|
|
"foo *** bar",
|
|
},
|
|
},
|
|
{
|
|
"Debug enabled", true,
|
|
[]string{
|
|
"::debug::GitHub Actions runtime token access controls",
|
|
},
|
|
[]string{
|
|
"GitHub Actions runtime token access controls",
|
|
},
|
|
},
|
|
{
|
|
"Debug not enabled", false,
|
|
[]string{
|
|
"::debug::GitHub Actions runtime token access controls",
|
|
},
|
|
[]string{
|
|
"<nil>",
|
|
},
|
|
},
|
|
{
|
|
"notice", false,
|
|
[]string{
|
|
"::notice file=file.name,line=42,endLine=48,title=Cool Title::Gosh, that's not going to work",
|
|
},
|
|
[]string{
|
|
"::notice file=file.name,line=42,endLine=48,title=Cool Title::Gosh, that's not going to work",
|
|
},
|
|
},
|
|
{
|
|
"warning", false,
|
|
[]string{
|
|
"::warning file=file.name,line=42,endLine=48,title=Cool Title::Gosh, that's not going to work",
|
|
},
|
|
[]string{
|
|
"::warning file=file.name,line=42,endLine=48,title=Cool Title::Gosh, that's not going to work",
|
|
},
|
|
},
|
|
{
|
|
"error", false,
|
|
[]string{
|
|
"::error file=file.name,line=42,endLine=48,title=Cool Title::Gosh, that's not going to work",
|
|
},
|
|
[]string{
|
|
"::error file=file.name,line=42,endLine=48,title=Cool Title::Gosh, that's not going to work",
|
|
},
|
|
},
|
|
{
|
|
"group", false,
|
|
[]string{
|
|
"::group::",
|
|
"::endgroup::",
|
|
},
|
|
[]string{
|
|
"::group::",
|
|
"::endgroup::",
|
|
},
|
|
},
|
|
{
|
|
"stop-commands", false,
|
|
[]string{
|
|
"::add-mask::foo",
|
|
"::stop-commands::myverycoolstoptoken",
|
|
"::add-mask::bar",
|
|
"::debug::Stuff",
|
|
"myverycoolstoptoken",
|
|
"::add-mask::baz",
|
|
"::myverycoolstoptoken::",
|
|
"::add-mask::wibble",
|
|
"foo bar baz wibble",
|
|
},
|
|
[]string{
|
|
"<nil>",
|
|
"<nil>",
|
|
"::add-mask::bar",
|
|
"::debug::Stuff",
|
|
"myverycoolstoptoken",
|
|
"::add-mask::baz",
|
|
"<nil>",
|
|
"<nil>",
|
|
"*** bar baz ***",
|
|
},
|
|
},
|
|
{
|
|
"unknown command", false,
|
|
[]string{
|
|
"::set-mask::foo",
|
|
},
|
|
[]string{
|
|
"::set-mask::foo",
|
|
},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
r := &Reporter{
|
|
logReplacer: strings.NewReplacer(),
|
|
debugOutputEnabled: tt.debugOutputEnabled,
|
|
}
|
|
for idx, arg := range tt.args {
|
|
rv, ok := r.parseLine(&log.Entry{Message: arg})
|
|
got := "<nil>"
|
|
|
|
if ok {
|
|
got = rv.Content
|
|
}
|
|
|
|
assert.Equal(t, tt.want[idx], got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReporter_Fire(t *testing.T) {
|
|
t.Run("ignore command lines", func(t *testing.T) {
|
|
client := mocks.NewClient(t)
|
|
client.On("Log", mock.Anything, mock.Anything).Return(func(_ context.Context, in *forge.LogIn) (*forge.LogOut, error) {
|
|
t.Logf("Received logs: index %d, %d rows", in.Index, len(in.Lines))
|
|
return &forge.LogOut{Ack: in.Index + int64(len(in.Lines))}, nil
|
|
})
|
|
client.On("State", mock.Anything, mock.Anything).Return(func(_ context.Context, in *forge.StateIn) (*forge.StateOut, error) {
|
|
t.Logf("Received state: task %d, result %q", in.State.ID, in.State.Result)
|
|
return &forge.StateOut{}, nil
|
|
})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cfg, _ := config.LoadDefault("")
|
|
reporter := NewReporter(ctx, cancel, client, &forge.Task{}, cfg)
|
|
reporter.RunDaemon()
|
|
defer func() {
|
|
require.NoError(t, reporter.Close(""))
|
|
}()
|
|
reporter.ResetSteps(5)
|
|
|
|
dataStep0 := map[string]any{
|
|
"stage": "Main",
|
|
"stepNumber": 0,
|
|
"raw_output": true,
|
|
}
|
|
|
|
require.NoError(t, reporter.Fire(&log.Entry{Message: "regular log line", Data: dataStep0}))
|
|
require.NoError(t, reporter.Fire(&log.Entry{Message: "::debug::debug log line", Data: dataStep0}))
|
|
require.NoError(t, reporter.Fire(&log.Entry{Message: "regular log line", Data: dataStep0}))
|
|
require.NoError(t, reporter.Fire(&log.Entry{Message: "::debug::debug log line", Data: dataStep0}))
|
|
require.NoError(t, reporter.Fire(&log.Entry{Message: "::debug::debug log line", Data: dataStep0}))
|
|
require.NoError(t, reporter.Fire(&log.Entry{Message: "regular log line", Data: dataStep0}))
|
|
require.NoError(t, reporter.Fire(&log.Entry{Message: "composite step result", Data: map[string]any{
|
|
"stage": "Main",
|
|
"stepID": []string{"0", "0"},
|
|
"stepNumber": 0,
|
|
"raw_output": true,
|
|
"stepResult": "failure",
|
|
}}))
|
|
assert.Equal(t, forge.Pending, reporter.state.Steps[0].Result)
|
|
require.NoError(t, reporter.Fire(&log.Entry{Message: "step result", Data: map[string]any{
|
|
"stage": "Main",
|
|
"stepNumber": 0,
|
|
"raw_output": true,
|
|
"stepResult": "success",
|
|
}}))
|
|
assert.Equal(t, forge.Success, reporter.state.Steps[0].Result)
|
|
|
|
assert.Equal(t, int64(5), reporter.state.Steps[0].LogLength)
|
|
})
|
|
}
|
|
|
|
// TestReporter_EphemeralRunnerDeletion reproduces the exact scenario from
|
|
// https://github.com/hanzoai/git-runner/issues/793:
|
|
//
|
|
// 1. RunDaemon calls ReportLog(false) — runner is still alive
|
|
// 2. Close() updates state to Result=FAILURE (between RunDaemon's ReportLog and ReportState)
|
|
// 3. RunDaemon's ReportState() would clone the completed state and send it,
|
|
// but the fix makes ReportState return early when closed, preventing this
|
|
// 4. Close's ReportLog(true) succeeds because the runner was not deleted
|
|
func TestReporter_EphemeralRunnerDeletion(t *testing.T) {
|
|
runnerDeleted := false
|
|
|
|
client := mocks.NewClient(t)
|
|
client.On("Log", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, in *forge.LogIn) (*forge.LogOut, error) {
|
|
if runnerDeleted {
|
|
return nil, errors.New("runner has been deleted")
|
|
}
|
|
return &forge.LogOut{Ack: in.Index + int64(len(in.Lines))}, nil
|
|
},
|
|
)
|
|
client.On("State", mock.Anything, mock.Anything).Maybe().Return(
|
|
func(_ context.Context, in *forge.StateIn) (*forge.StateOut, error) {
|
|
// Server deletes ephemeral runner when it receives a completed state
|
|
if in.State.Result != forge.Pending {
|
|
runnerDeleted = true
|
|
}
|
|
return &forge.StateOut{}, nil
|
|
},
|
|
)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
cfg, _ := config.LoadDefault("")
|
|
reporter := NewReporter(ctx, cancel, client, &forge.Task{}, cfg)
|
|
reporter.ResetSteps(1)
|
|
|
|
// Fire a log entry to create pending data
|
|
require.NoError(t, reporter.Fire(&log.Entry{
|
|
Message: "build output",
|
|
Data: log.Fields{"stage": "Main", "stepNumber": 0, "raw_output": true},
|
|
}))
|
|
|
|
// Step 1: RunDaemon calls ReportLog(false) — runner is still alive
|
|
require.NoError(t, reporter.ReportLog(false))
|
|
|
|
// Step 2: Close() updates state — sets Result=FAILURE and marks steps cancelled.
|
|
// In the real race, this happens while RunDaemon is between ReportLog and ReportState.
|
|
reporter.stateMu.Lock()
|
|
reporter.closed = true
|
|
for i := range reporter.state.Steps {
|
|
if reporter.state.Steps[i].Result == forge.Pending {
|
|
reporter.state.Steps[i].Result = forge.Cancelled
|
|
}
|
|
}
|
|
reporter.state.Result = forge.Failure
|
|
reporter.logLines = append(reporter.logLines, forge.Line{
|
|
Time: time.Now().UnixNano(),
|
|
Content: "Early termination",
|
|
})
|
|
reporter.state.Stopped = time.Now().UnixNano()
|
|
reporter.stateMu.Unlock()
|
|
|
|
// Step 3: RunDaemon's ReportState() — with the fix, this returns early
|
|
// because closed=true, preventing the server from deleting the runner.
|
|
require.NoError(t, reporter.ReportState(false))
|
|
assert.False(t, runnerDeleted, "runner must not be deleted by RunDaemon's ReportState")
|
|
|
|
// Step 4: Close's final log upload succeeds because the runner is still alive.
|
|
// Flush pending rows first, then send the noMore signal (matching Close's retry behavior).
|
|
require.NoError(t, reporter.ReportLog(false))
|
|
// Acknowledge Close as done in daemon
|
|
close(reporter.daemon)
|
|
err := reporter.ReportLog(true)
|
|
require.NoError(t, err, "final log upload must not fail: runner should not be deleted before Close finishes sending logs")
|
|
err = reporter.ReportState(true)
|
|
require.NoError(t, err, "final state update should work: runner should not be deleted before Close finishes sending logs")
|
|
}
|
|
|
|
func TestReporter_RunDaemonClose_Race(t *testing.T) {
|
|
client := mocks.NewClient(t)
|
|
client.On("Log", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, in *forge.LogIn) (*forge.LogOut, error) {
|
|
return &forge.LogOut{Ack: in.Index + int64(len(in.Lines))}, nil
|
|
},
|
|
)
|
|
client.On("State", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, in *forge.StateIn) (*forge.StateOut, error) {
|
|
return &forge.StateOut{}, nil
|
|
},
|
|
)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cfg, _ := config.LoadDefault("")
|
|
reporter := NewReporter(ctx, cancel, client, &forge.Task{}, cfg)
|
|
reporter.ResetSteps(1)
|
|
|
|
// Start the daemon loop — RunDaemon spawns a goroutine internally.
|
|
reporter.RunDaemon()
|
|
|
|
// Close concurrently — this races with the daemon goroutine on r.closed.
|
|
require.NoError(t, reporter.Close(""))
|
|
|
|
// Cancel context so the daemon goroutine exits cleanly.
|
|
cancel()
|
|
}
|
|
|
|
// TestReporter_MaxLatencyTimer verifies that the maxLatencyTimer flushes a
|
|
// single buffered log row before the periodic logTicker fires.
|
|
//
|
|
// Setup: logReportInterval=10s (effectively never), maxLatency=100ms.
|
|
// Fire one log line, then assert UpdateLog is called within 500ms.
|
|
func TestReporter_MaxLatencyTimer(t *testing.T) {
|
|
var updateLogCalls atomic.Int64
|
|
|
|
client := mocks.NewClient(t)
|
|
client.On("Log", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, in *forge.LogIn) (*forge.LogOut, error) {
|
|
updateLogCalls.Add(1)
|
|
return &forge.LogOut{Ack: in.Index + int64(len(in.Lines))}, nil
|
|
},
|
|
)
|
|
client.On("State", mock.Anything, mock.Anything).Maybe().Return(
|
|
func(_ context.Context, _ *forge.StateIn) (*forge.StateOut, error) {
|
|
return &forge.StateOut{}, nil
|
|
},
|
|
)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
// Custom config: logTicker=10s (won't fire during test), maxLatency=100ms
|
|
cfg, _ := config.LoadDefault("")
|
|
cfg.Runner.LogReportInterval = 10 * time.Second
|
|
cfg.Runner.LogReportMaxLatency = 100 * time.Millisecond
|
|
cfg.Runner.LogReportBatchSize = 1000 // won't trigger batch flush
|
|
|
|
reporter := NewReporter(ctx, cancel, client, &forge.Task{}, cfg)
|
|
reporter.ResetSteps(1)
|
|
reporter.RunDaemon()
|
|
defer func() {
|
|
_ = reporter.Close("")
|
|
}()
|
|
|
|
// Fire a single log line — not enough to trigger batch flush
|
|
require.NoError(t, reporter.Fire(&log.Entry{
|
|
Message: "single log line",
|
|
Data: log.Fields{"stage": "Main", "stepNumber": 0, "raw_output": true},
|
|
}))
|
|
|
|
// maxLatencyTimer should flush within ~100ms. Wait up to 500ms.
|
|
assert.Eventually(t, func() bool {
|
|
return updateLogCalls.Load() > 0
|
|
}, 500*time.Millisecond, 10*time.Millisecond,
|
|
"maxLatencyTimer should have flushed the log before logTicker (10s)")
|
|
}
|
|
|
|
// TestReporter_BatchSizeFlush verifies that reaching logBatchSize triggers
|
|
// an immediate log flush without waiting for any timer.
|
|
func TestReporter_BatchSizeFlush(t *testing.T) {
|
|
var updateLogCalls atomic.Int64
|
|
|
|
client := mocks.NewClient(t)
|
|
client.On("Log", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, in *forge.LogIn) (*forge.LogOut, error) {
|
|
updateLogCalls.Add(1)
|
|
return &forge.LogOut{Ack: in.Index + int64(len(in.Lines))}, nil
|
|
},
|
|
)
|
|
client.On("State", mock.Anything, mock.Anything).Maybe().Return(
|
|
func(_ context.Context, _ *forge.StateIn) (*forge.StateOut, error) {
|
|
return &forge.StateOut{}, nil
|
|
},
|
|
)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
// Custom config: large timers, small batch size
|
|
cfg, _ := config.LoadDefault("")
|
|
cfg.Runner.LogReportInterval = 10 * time.Second
|
|
cfg.Runner.LogReportMaxLatency = 10 * time.Second
|
|
cfg.Runner.LogReportBatchSize = 5
|
|
|
|
reporter := NewReporter(ctx, cancel, client, &forge.Task{}, cfg)
|
|
reporter.ResetSteps(1)
|
|
reporter.RunDaemon()
|
|
defer func() {
|
|
_ = reporter.Close("")
|
|
}()
|
|
|
|
// Fire exactly batchSize log lines
|
|
for i := range 5 {
|
|
require.NoError(t, reporter.Fire(&log.Entry{
|
|
Message: fmt.Sprintf("log line %d", i),
|
|
Data: log.Fields{"stage": "Main", "stepNumber": 0, "raw_output": true},
|
|
}))
|
|
}
|
|
|
|
// Batch threshold should trigger immediate flush
|
|
assert.Eventually(t, func() bool {
|
|
return updateLogCalls.Load() > 0
|
|
}, 500*time.Millisecond, 10*time.Millisecond,
|
|
"batch size threshold should have triggered immediate flush")
|
|
}
|
|
|
|
// TestReporter_StateChangedNotLostDuringReport asserts that a Fire() arriving
|
|
// mid-UpdateTask re-dirties the flag so the change is picked up by the next report.
|
|
func TestReporter_StateChangedNotLostDuringReport(t *testing.T) {
|
|
var updateTaskCalls atomic.Int64
|
|
inFlight := make(chan struct{})
|
|
release := make(chan struct{})
|
|
|
|
client := mocks.NewClient(t)
|
|
client.On("State", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, _ *forge.StateIn) (*forge.StateOut, error) {
|
|
n := updateTaskCalls.Add(1)
|
|
if n == 1 {
|
|
// Signal that the first UpdateTask is in flight, then block until released.
|
|
close(inFlight)
|
|
<-release
|
|
}
|
|
return &forge.StateOut{}, nil
|
|
},
|
|
)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
cfg, _ := config.LoadDefault("")
|
|
reporter := NewReporter(ctx, cancel, client, &forge.Task{}, cfg)
|
|
reporter.ResetSteps(2)
|
|
|
|
// Mark stateChanged=true so the first ReportState proceeds to UpdateTask.
|
|
reporter.stateMu.Lock()
|
|
reporter.stateChanged = true
|
|
reporter.stateMu.Unlock()
|
|
|
|
// Kick off the first ReportState in a goroutine — it will block in UpdateTask.
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
done <- reporter.ReportState(false)
|
|
}()
|
|
|
|
// Wait until UpdateTask is in flight (snapshot taken, flag consumed).
|
|
<-inFlight
|
|
|
|
// Concurrent Fire() modifies state — must re-flip stateChanged so the
|
|
// change is not lost when the in-flight ReportState finishes.
|
|
require.NoError(t, reporter.Fire(&log.Entry{
|
|
Message: "step starts",
|
|
Data: log.Fields{"stage": "Main", "stepNumber": 1, "raw_output": true},
|
|
}))
|
|
|
|
// Release the in-flight UpdateTask and wait for it to return.
|
|
close(release)
|
|
require.NoError(t, <-done)
|
|
|
|
// stateChanged must still be true so the next ReportState picks up the
|
|
// concurrent Fire()'s change instead of skipping via the early-return path.
|
|
reporter.stateMu.RLock()
|
|
changed := reporter.stateChanged
|
|
reporter.stateMu.RUnlock()
|
|
assert.True(t, changed, "stateChanged must remain true after a concurrent Fire() during in-flight ReportState")
|
|
|
|
// And the next ReportState must actually send a second UpdateTask.
|
|
require.NoError(t, reporter.ReportState(false))
|
|
assert.Equal(t, int64(2), updateTaskCalls.Load(), "concurrent Fire() change must trigger a second UpdateTask, not be silently lost")
|
|
}
|
|
|
|
// TestReporter_StateChangedRestoredOnError verifies that when UpdateTask fails,
|
|
// the dirty flag is restored so the snapshotted change isn't silently lost.
|
|
func TestReporter_StateChangedRestoredOnError(t *testing.T) {
|
|
var updateTaskCalls atomic.Int64
|
|
|
|
client := mocks.NewClient(t)
|
|
client.On("State", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, _ *forge.StateIn) (*forge.StateOut, error) {
|
|
n := updateTaskCalls.Add(1)
|
|
if n == 1 {
|
|
return nil, errors.New("transient network error")
|
|
}
|
|
return &forge.StateOut{}, nil
|
|
},
|
|
)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
cfg, _ := config.LoadDefault("")
|
|
reporter := NewReporter(ctx, cancel, client, &forge.Task{}, cfg)
|
|
reporter.ResetSteps(1)
|
|
|
|
reporter.stateMu.Lock()
|
|
reporter.stateChanged = true
|
|
reporter.stateMu.Unlock()
|
|
|
|
// First ReportState fails — flag must be restored to true.
|
|
require.Error(t, reporter.ReportState(false))
|
|
|
|
reporter.stateMu.RLock()
|
|
changed := reporter.stateChanged
|
|
reporter.stateMu.RUnlock()
|
|
assert.True(t, changed, "stateChanged must be restored to true after UpdateTask error so the change is retried")
|
|
|
|
// The next ReportState should still issue a request because the flag was restored.
|
|
require.NoError(t, reporter.ReportState(false))
|
|
assert.Equal(t, int64(2), updateTaskCalls.Load())
|
|
}
|
|
|
|
// TestReporter_StateNotifyFlush verifies that step transitions trigger
|
|
// an immediate state flush via the stateNotify channel.
|
|
func TestReporter_StateNotifyFlush(t *testing.T) {
|
|
var updateTaskCalls atomic.Int64
|
|
|
|
client := mocks.NewClient(t)
|
|
client.On("Log", mock.Anything, mock.Anything).Maybe().Return(
|
|
func(_ context.Context, in *forge.LogIn) (*forge.LogOut, error) {
|
|
return &forge.LogOut{Ack: in.Index + int64(len(in.Lines))}, nil
|
|
},
|
|
)
|
|
client.On("State", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, _ *forge.StateIn) (*forge.StateOut, error) {
|
|
updateTaskCalls.Add(1)
|
|
return &forge.StateOut{}, nil
|
|
},
|
|
)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
// Custom config: large state interval so only stateNotify can trigger
|
|
cfg, _ := config.LoadDefault("")
|
|
cfg.Runner.StateReportInterval = 10 * time.Second
|
|
cfg.Runner.LogReportInterval = 10 * time.Second
|
|
|
|
reporter := NewReporter(ctx, cancel, client, &forge.Task{}, cfg)
|
|
reporter.ResetSteps(1)
|
|
reporter.RunDaemon()
|
|
defer func() {
|
|
_ = reporter.Close("")
|
|
}()
|
|
|
|
// Fire a log entry that starts a step — this triggers stateNotify
|
|
require.NoError(t, reporter.Fire(&log.Entry{
|
|
Message: "step starting",
|
|
Data: log.Fields{"stage": "Main", "stepNumber": 0, "raw_output": true},
|
|
}))
|
|
|
|
// stateNotify should trigger immediate UpdateTask call
|
|
assert.Eventually(t, func() bool {
|
|
return updateTaskCalls.Load() > 0
|
|
}, 500*time.Millisecond, 10*time.Millisecond,
|
|
"step transition should have triggered immediate state flush via stateNotify")
|
|
}
|
|
|
|
// TestReporter_StateHeartbeat verifies that ReportState sends a heartbeat
|
|
// UpdateTask once stateReportInterval has elapsed since the last successful
|
|
// report, even when nothing has changed. Without this, long-running silent
|
|
// jobs (no log output, no step transitions) cause the server to time the
|
|
// task out and cancel it (#826).
|
|
func TestReporter_StateHeartbeat(t *testing.T) {
|
|
var updateTaskCalls atomic.Int64
|
|
|
|
client := mocks.NewClient(t)
|
|
client.On("State", mock.Anything, mock.Anything).Return(
|
|
func(_ context.Context, _ *forge.StateIn) (*forge.StateOut, error) {
|
|
updateTaskCalls.Add(1)
|
|
return &forge.StateOut{}, nil
|
|
},
|
|
)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
cfg, _ := config.LoadDefault("")
|
|
cfg.Runner.StateReportInterval = 50 * time.Millisecond
|
|
reporter := NewReporter(ctx, cancel, client, &forge.Task{}, cfg)
|
|
reporter.ResetSteps(1)
|
|
|
|
// First call has no prior report — sends to seed lastReportedAt.
|
|
reporter.stateMu.Lock()
|
|
reporter.stateChanged = true
|
|
reporter.stateMu.Unlock()
|
|
require.NoError(t, reporter.ReportState(false))
|
|
require.Equal(t, int64(1), updateTaskCalls.Load())
|
|
|
|
// Second call immediately after with nothing changed — must skip.
|
|
require.NoError(t, reporter.ReportState(false))
|
|
assert.Equal(t, int64(1), updateTaskCalls.Load(), "no-op ReportState within stateReportInterval must skip")
|
|
|
|
// After stateReportInterval elapses, a heartbeat must fire even with no changes.
|
|
time.Sleep(2 * cfg.Runner.StateReportInterval)
|
|
require.NoError(t, reporter.ReportState(false))
|
|
assert.Equal(t, int64(2), updateTaskCalls.Load(), "ReportState must heartbeat after stateReportInterval even with no state change")
|
|
}
|