mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
* refactor: Standardize main.go style - Add package doc - Move flag declarations into a var() block - Add description strings to each flag - Add section dividers for readability - Group imports into stdlib / third-party / project - Remove dead initialization of runtimeClassConfigPath (overwritten by flag default) No behavior change. Signed-off-by: JiangJiaWei1103 <[email protected]> * fix: Surface configuration errors clearly - Use logrus.Fatalf(...) for consistent log formatting - Add required-flag check on --runtime-class-name for fast fail Signed-off-by: JiangJiaWei1103 <[email protected]> * Refactor: Switch to context-based shutdown via signal.NotifyContext - Replace manual sigChan + signal.Notify with signal.NotifyContext - Add bridge goroutine that closes the legacy stop ch when serverCtx fires No behavior change. Signed-off-by: JiangJiaWei1103 <[email protected]> * feat: Add ProcessSingleSession for synchronous per-session loading Signed-off-by: JiangJiaWei1103 <[email protected]> * feat: Lazy session processing via Supervisor with isDead probe - Add a Supervisor as /enter_cluster broker - Use in-process loaded set to record processed sessions - Add a Pipeline that wraps the parse steps - Remove eager processing on startup and ticker Signed-off-by: JiangJiaWei1103 <[email protected]> * docs: Add smoke tests Signed-off-by: JiangJiaWei1103 <[email protected]> * test: Cover Supervisor error propagation and no-retry contracts Signed-off-by: JiangJiaWei1103 <[email protected]> * docs: Add missing sa Signed-off-by: JiangJiaWei1103 <[email protected]> * fix: Surface storage outage error in ProcessSingleSession Signed-off-by: JiangJiaWei1103 <[email protected]> * fix: Handle isDead for RayCluster recreation with the same namespacedname Signed-off-by: JiangJiaWei1103 <[email protected]> * fix: Avoid setting cookie before a successful load or a live-skip Signed-off-by: JiangJiaWei1103 <[email protected]> * fix: Use live sentinel to avoid querying empty in-mem state Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Skip singleflight for already-loaded sessions Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Reuse k8s client Signed-off-by: JiangJiaWei1103 <[email protected]> * fix: Surface errors of two subsystems Signed-off-by: JiangJiaWei1103 <[email protected]> * docs: Remove redundant comments Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Rename to SessionLoader and LoadSession Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Rename to SessionProcessor for clarity Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Address PR review nits for cleanness Signed-off-by: JiangJiaWei1103 <[email protected]> * docs: Clean up logs and comments Signed-off-by: JiangJiaWei1103 <[email protected]> * docs: Clean up docstrings Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Rename to SessionStatusClusterStateUnknown to indicate status can not be determined Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Push ctx into ProcessSingleSession to avoid dup ctx err polling Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Consolidate ProcessSingleSession test into eventserver_test.go Signed-off-by: JiangJiaWei1103 <[email protected]> * docs: Clean up more comments Signed-off-by: JiangJiaWei1103 <[email protected]> * docs: Address more nits for clarity Signed-off-by: JiangJiaWei1103 <[email protected]> * fix: Fix assertion string Signed-off-by: JiangJiaWei1103 <[email protected]> * fix: Add session input validation and resolve Nary's nits Signed-off-by: JiangJiaWei1103 <[email protected]> * fix: Add timeout for a single cold-load to avoid resource leakage Signed-off-by: JiangJiaWei1103 <[email protected]> * feat: Add HTTPLiveSessionResolver for determining session name of live cluster Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Remove client-side redundant timeout setup Signed-off-by: JiangJiaWei1103 <[email protected]> * fix: Enable cross-ns dns lookup Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Fix nits Signed-off-by: JiangJiaWei1103 <[email protected]> * docs: Clarify comments Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Improve error msg, comments and fix nits Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Session key as explicit param of storeEvent without mutating event obj Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Rename process session timeout and make it configurable Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Remove all dead code Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Remove HTTPLiveSessionResolver to simplify live-session detection Signed-off-by: JiangJiaWei1103 <[email protected]> * fix: Add zero-value session status as a defensive guard Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Fix nits Signed-off-by: JiangJiaWei1103 <[email protected]> * refactor: Reuse sess key gen fn Signed-off-by: JiangJiaWei1103 <[email protected]> * docs: Clarify known limitation Signed-off-by: JiangJiaWei1103 <[email protected]> --------- Signed-off-by: JiangJiaWei1103 <[email protected]>
257 lines
9.6 KiB
Go
257 lines
9.6 KiB
Go
package eventserver
|
|
|
|
import (
|
|
"bufio"
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/ray-project/kuberay/historyserver/pkg/eventserver/types"
|
|
"github.com/ray-project/kuberay/historyserver/pkg/utils"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// --- Test-local mock reader for LogEventReader tests ---
|
|
|
|
type logEventMockReader struct {
|
|
files map[string]map[string]string
|
|
dirs map[string]map[string][]string
|
|
}
|
|
|
|
func newLogEventMockReader() *logEventMockReader {
|
|
return &logEventMockReader{
|
|
files: make(map[string]map[string]string),
|
|
dirs: make(map[string]map[string][]string),
|
|
}
|
|
}
|
|
|
|
func (m *logEventMockReader) addFile(clusterID, filePath, content string) {
|
|
if m.files[clusterID] == nil {
|
|
m.files[clusterID] = make(map[string]string)
|
|
}
|
|
m.files[clusterID][filePath] = content
|
|
}
|
|
|
|
func (m *logEventMockReader) addDir(clusterID, dirPath string, entries []string) {
|
|
if m.dirs[clusterID] == nil {
|
|
m.dirs[clusterID] = make(map[string][]string)
|
|
}
|
|
m.dirs[clusterID][dirPath] = entries
|
|
}
|
|
|
|
func (m *logEventMockReader) List() []utils.ClusterInfo { return nil }
|
|
|
|
func (m *logEventMockReader) GetContent(clusterID string, fileName string) io.Reader {
|
|
if cd, ok := m.files[clusterID]; ok {
|
|
if content, ok := cd[fileName]; ok {
|
|
return strings.NewReader(content)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *logEventMockReader) ListFiles(clusterID string, dir string) []string {
|
|
if cd, ok := m.dirs[clusterID]; ok {
|
|
if entries, ok := cd[dir]; ok {
|
|
return entries
|
|
}
|
|
}
|
|
return []string{}
|
|
}
|
|
|
|
// --- Tests ---
|
|
|
|
func TestReadLineWithLimit(t *testing.T) {
|
|
t.Run("normal line", func(t *testing.T) {
|
|
r := bufio.NewReader(strings.NewReader("hello world\n"))
|
|
line, n, tooLong, err := readLineWithLimit(r, 1024)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "hello world\n", string(line))
|
|
assert.Equal(t, 12, n)
|
|
assert.False(t, tooLong)
|
|
})
|
|
|
|
t.Run("line exceeding limit is skipped and reader recovers", func(t *testing.T) {
|
|
content := strings.Repeat("x", 200) + "\nshort\n"
|
|
r := bufio.NewReader(strings.NewReader(content))
|
|
|
|
line1, _, tooLong1, err1 := readLineWithLimit(r, 100)
|
|
require.NoError(t, err1)
|
|
assert.True(t, tooLong1)
|
|
assert.Nil(t, line1)
|
|
|
|
line2, _, tooLong2, err2 := readLineWithLimit(r, 100)
|
|
require.NoError(t, err2)
|
|
assert.False(t, tooLong2)
|
|
assert.Equal(t, "short\n", string(line2))
|
|
})
|
|
|
|
t.Run("EOF without trailing newline", func(t *testing.T) {
|
|
r := bufio.NewReader(strings.NewReader("no newline"))
|
|
line, n, tooLong, err := readLineWithLimit(r, 1024)
|
|
assert.ErrorIs(t, err, io.EOF)
|
|
assert.Equal(t, "no newline", string(line))
|
|
assert.Equal(t, 10, n)
|
|
assert.False(t, tooLong)
|
|
})
|
|
|
|
t.Run("boundary: exactly at limit vs one over", func(t *testing.T) {
|
|
atLimit := strings.Repeat("a", 50) + "\n"
|
|
r1 := bufio.NewReader(strings.NewReader(atLimit))
|
|
_, _, tooLong1, _ := readLineWithLimit(r1, 51)
|
|
assert.False(t, tooLong1, "line exactly at limit should not be too long")
|
|
|
|
overLimit := strings.Repeat("a", 51) + "\n"
|
|
r2 := bufio.NewReader(strings.NewReader(overLimit))
|
|
_, _, tooLong2, _ := readLineWithLimit(r2, 51)
|
|
assert.True(t, tooLong2, "line one byte over limit should be too long")
|
|
})
|
|
}
|
|
|
|
// Realistic event JSON lines, modeled after actual Ray event log files.
|
|
const (
|
|
testJobEvent = `{"event_id":"1Ac5EF8dC3B20F78","source_type":"JOBS","source_hostname":"raycluster-head-6gvqk","source_pid":568,"severity":"INFO","message":"Started a ray job rayjob-abc.","timestamp":"1770635705","custom_fields":{"job_id":"01000000","submission_id":"rayjob-abc"}}`
|
|
testGlobalEvent = `{"event_id":"C798f8dC4Dce8B8b","source_type":"GCS","source_hostname":"raycluster-head-6gvqk","source_pid":568,"severity":"WARNING","message":"Node disconnected.","timestamp":"1770635720"}`
|
|
testNewlineEvent = `{"event_id":"D1E2F3A4B5C6D7E8","source_type":"GCS","source_hostname":"raycluster-head-6gvqk","source_pid":568,"severity":"INFO","message":"line1\\nline2\\rline3","timestamp":"1770635740"}`
|
|
)
|
|
|
|
func TestReadEventFile(t *testing.T) {
|
|
t.Run("groups by job_id and deduplicates by event_id", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
content := strings.Join([]string{
|
|
testJobEvent,
|
|
testGlobalEvent,
|
|
testJobEvent, // duplicate event_id — should be deduped
|
|
}, "\n") + "\n"
|
|
mock.addFile("cluster_ns", "session/logs/node1/events/event_JOBS.log", content)
|
|
|
|
reader := NewLogEventReader(mock)
|
|
jobEventMap := types.NewJobEventMap()
|
|
err := reader.readEventFile("cluster_ns", "session/logs/node1/events/event_JOBS.log", jobEventMap)
|
|
require.NoError(t, err)
|
|
|
|
events := jobEventMap.GetAllEvents()
|
|
assert.Len(t, events["01000000"], 1, "should deduplicate by event_id")
|
|
assert.Len(t, events["global"], 1, "event without job_id goes to global")
|
|
})
|
|
|
|
t.Run("skips invalid JSON and events without event_id", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
noIDEvent := `{"source_type":"GCS","severity":"INFO","message":"no id","timestamp":"1770635730"}`
|
|
content := strings.Join([]string{
|
|
testGlobalEvent,
|
|
"INVALID JSON",
|
|
noIDEvent,
|
|
}, "\n") + "\n"
|
|
mock.addFile("cluster_ns", "session/logs/node1/events/event_GCS.log", content)
|
|
|
|
reader := NewLogEventReader(mock)
|
|
jobEventMap := types.NewJobEventMap()
|
|
err := reader.readEventFile("cluster_ns", "session/logs/node1/events/event_GCS.log", jobEventMap)
|
|
require.NoError(t, err)
|
|
|
|
events := jobEventMap.GetAllEvents()
|
|
assert.Len(t, events["global"], 1, "should skip invalid JSON and events without event_id")
|
|
})
|
|
|
|
t.Run("restores escaped newlines in message", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
mock.addFile("cluster_ns", "session/logs/node1/events/event_GCS.log", testNewlineEvent+"\n")
|
|
|
|
reader := NewLogEventReader(mock)
|
|
jobEventMap := types.NewJobEventMap()
|
|
err := reader.readEventFile("cluster_ns", "session/logs/node1/events/event_GCS.log", jobEventMap)
|
|
require.NoError(t, err)
|
|
|
|
events := jobEventMap.GetAllEvents()
|
|
require.Len(t, events["global"], 1)
|
|
assert.Equal(t, "line1\nline2\nline3", events["global"][0]["message"])
|
|
})
|
|
|
|
t.Run("handles empty file and missing file", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
mock.addFile("cluster_ns", "session/logs/node1/events/event_GCS.log", "")
|
|
reader := NewLogEventReader(mock)
|
|
|
|
jobEventMap := types.NewJobEventMap()
|
|
err := reader.readEventFile("cluster_ns", "session/logs/node1/events/event_GCS.log", jobEventMap)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, jobEventMap.GetAllEvents())
|
|
|
|
err = reader.readEventFile("cluster_ns", "nonexistent", types.NewJobEventMap())
|
|
assert.Error(t, err)
|
|
})
|
|
}
|
|
|
|
func TestReadLogEvents(t *testing.T) {
|
|
t.Run("reads events from multiple nodes, skips non-event files", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
|
|
mock.addDir("cluster_ns", "session1/logs", []string{"node1/", "node2/", "stray_file.txt"})
|
|
mock.addDir("cluster_ns", "session1/logs/node1/events", []string{"event_GCS.log", "debug.log"})
|
|
mock.addDir("cluster_ns", "session1/logs/node2/events", []string{"event_RAYLET.log"})
|
|
|
|
mock.addFile("cluster_ns", "session1/logs/node1/events/event_GCS.log",
|
|
`{"event_id":"e1","source_type":"GCS","severity":"INFO","message":"from node1","timestamp":"1770635700"}`+"\n")
|
|
mock.addFile("cluster_ns", "session1/logs/node2/events/event_RAYLET.log",
|
|
`{"event_id":"e2","source_type":"RAYLET","severity":"WARNING","message":"from node2","timestamp":"1770635800"}`+"\n")
|
|
|
|
reader := NewLogEventReader(mock)
|
|
store := types.NewClusterLogEventMap()
|
|
clusterInfo := utils.ClusterInfo{Name: "cluster", Namespace: "ns", SessionName: "session1"}
|
|
|
|
err := reader.ReadLogEvents(clusterInfo, "cluster_ns_session1", store)
|
|
require.NoError(t, err)
|
|
|
|
events := store.GetAllEvents("cluster_ns_session1")
|
|
assert.Len(t, events["global"], 2, "should read events from both nodes")
|
|
})
|
|
|
|
t.Run("handles empty cluster with no nodes", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
mock.addDir("cluster_ns", "session1/logs", []string{})
|
|
|
|
reader := NewLogEventReader(mock)
|
|
store := types.NewClusterLogEventMap()
|
|
clusterInfo := utils.ClusterInfo{Name: "cluster", Namespace: "ns", SessionName: "session1"}
|
|
|
|
err := reader.ReadLogEvents(clusterInfo, "cluster_ns_session1", store)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, store.GetAllEvents("cluster_ns_session1"))
|
|
})
|
|
|
|
t.Run("returns error when every listed file fails to read", func(t *testing.T) {
|
|
// Intentionally skip addFile.
|
|
mock := newLogEventMockReader()
|
|
mock.addDir("cluster_ns", "session1/logs", []string{"node1/"})
|
|
mock.addDir("cluster_ns", "session1/logs/node1/events", []string{"event_GCS.log", "event_RAYLET.log"})
|
|
|
|
reader := NewLogEventReader(mock)
|
|
store := types.NewClusterLogEventMap()
|
|
clusterInfo := utils.ClusterInfo{Name: "cluster", Namespace: "ns", SessionName: "session1"}
|
|
|
|
err := reader.ReadLogEvents(clusterInfo, "cluster_ns_session1", store)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "read 0 of 2")
|
|
})
|
|
|
|
t.Run("partial read surfaces error but preserves successful events", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
mock.addDir("cluster_ns", "session1/logs", []string{"node1/"})
|
|
mock.addDir("cluster_ns", "session1/logs/node1/events", []string{"event_GCS.log", "event_RAYLET.log"})
|
|
mock.addFile("cluster_ns", "session1/logs/node1/events/event_GCS.log",
|
|
`{"event_id":"e1","source_type":"GCS","severity":"INFO","message":"ok","timestamp":"1770635700"}`+"\n")
|
|
|
|
reader := NewLogEventReader(mock)
|
|
store := types.NewClusterLogEventMap()
|
|
clusterInfo := utils.ClusterInfo{Name: "cluster", Namespace: "ns", SessionName: "session1"}
|
|
|
|
err := reader.ReadLogEvents(clusterInfo, "cluster_ns_session1", store)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "read 1 of 2")
|
|
events := store.GetAllEvents("cluster_ns_session1")
|
|
assert.Len(t, events["global"], 1)
|
|
})
|
|
}
|