Files
fscnickandGitHub 99f542a42b [History server] fix mismatch func signature (#4930)
* [History server] fix mismatch func signature

Signed-off-by: fscnick <[email protected]>

* [History server] fix func description

Signed-off-by: fscnick <[email protected]>

---------

Signed-off-by: fscnick <[email protected]>
2026-06-21 21:48:57 -07:00

1096 lines
35 KiB
Go

package eventserver
import (
"context"
"encoding/json"
"fmt"
"io"
"regexp"
"sort"
"strconv"
"strings"
"time"
"github.com/sirupsen/logrus"
"github.com/ray-project/kuberay/historyserver/pkg/compression"
"github.com/ray-project/kuberay/historyserver/pkg/eventserver/types"
"github.com/ray-project/kuberay/historyserver/pkg/storage"
"github.com/ray-project/kuberay/historyserver/pkg/utils"
)
type EventHandler struct {
reader storage.StorageReader
ClusterTaskMap *types.ClusterTaskMap
ClusterActorMap *types.ClusterActorMap
ClusterJobMap *types.ClusterJobMap
ClusterNodeMap *types.ClusterNodeMap
ClusterLogEventMap *types.ClusterLogEventMap // For /events API (Log Events from logs/events/)
}
var eventFilePattern = regexp.MustCompile(`-\d{4}-\d{2}-\d{2}-\d{2}(\.gz)?$`)
// taskPrefix is extracted to avoid hard-coded "task::" usage
const taskPrefix = "task::"
func isValidEventFile(fileName string) bool {
// Skip directories
if strings.HasSuffix(fileName, "/") {
return false
}
// Only files matching {nodeId}-{YYYY-MM-DD-HH} format are valid event files
return eventFilePattern.MatchString(fileName)
}
func NewEventHandler(reader storage.StorageReader) *EventHandler {
return &EventHandler{
reader: reader,
ClusterTaskMap: &types.ClusterTaskMap{
ClusterTaskMap: make(map[string]*types.TaskMap),
},
ClusterActorMap: &types.ClusterActorMap{
ClusterActorMap: make(map[string]*types.ActorMap),
},
ClusterJobMap: &types.ClusterJobMap{
ClusterJobMap: make(map[string]*types.JobMap),
},
ClusterNodeMap: &types.ClusterNodeMap{
ClusterNodeMap: make(map[string]*types.NodeMap),
},
ClusterLogEventMap: types.NewClusterLogEventMap(),
}
}
// storeEvent unmarshals the event map into the correct actor/task struct and then stores it into the corresonding list.
func (h *EventHandler) storeEvent(clusterSessionKey string, eventMap map[string]any) error {
eventTypeVal, ok := eventMap["eventType"]
if !ok {
return fmt.Errorf("event missing 'eventType' field")
}
eventTypeStr, ok := eventTypeVal.(string)
if !ok {
return fmt.Errorf("eventType is not a string, got %T", eventTypeVal)
}
eventType := types.EventType(eventTypeStr)
logrus.Infof("current eventType: %v", eventType)
switch eventType {
case types.TASK_DEFINITION_EVENT:
return h.handleTaskDefinitionEvent(eventMap, clusterSessionKey, false)
case types.TASK_LIFECYCLE_EVENT:
return h.handleTaskLifecycleEvent(eventMap, clusterSessionKey)
case types.ACTOR_DEFINITION_EVENT:
actorDef, ok := eventMap["actorDefinitionEvent"]
if !ok {
return fmt.Errorf("event does not have 'actorDefinitionEvent'")
}
jsonActorDefinition, err := json.Marshal(actorDef)
if err != nil {
return err
}
var currActor types.Actor
if err := json.Unmarshal(jsonActorDefinition, &currActor); err != nil {
return err
}
normalizeActorIDsToHex(&currActor)
// Use CreateOrMergeActor pattern (same as Task)
actorMap := h.ClusterActorMap.GetOrCreateActorMap(clusterSessionKey)
actorMap.CreateOrMergeActor(currActor.ActorID, func(a *types.Actor) {
// Preserve lifecycle-derived fields that may have arrived first
existingEvents := a.Events
existingState := a.State
existingStartTime := a.StartTime
existingEndTime := a.EndTime
existingNumRestarts := a.NumRestarts
existingPID := a.PID
existingExitDetails := a.ExitDetails
existingAddress := a.Address
// Overwrite with definition fields
*a = currActor
// Restore lifecycle-derived fields if they existed
if len(existingEvents) > 0 {
a.Events = existingEvents
a.State = existingState
a.StartTime = existingStartTime
a.EndTime = existingEndTime
a.NumRestarts = existingNumRestarts
a.PID = existingPID
a.ExitDetails = existingExitDetails
a.Address = existingAddress
}
})
case types.ACTOR_LIFECYCLE_EVENT:
lifecycleEvent, ok := eventMap["actorLifecycleEvent"].(map[string]any)
if !ok {
return fmt.Errorf("invalid actorLifecycleEvent format")
}
actorId, _ := lifecycleEvent["actorId"].(string)
actorId = normalizeIDToHex(actorId)
transitions, _ := lifecycleEvent["stateTransitions"].([]any)
if len(transitions) == 0 || actorId == "" {
return nil
}
// Parse state transitions into ActorStateEvent slice
var stateEvents []types.ActorStateEvent
for _, transition := range transitions {
tr, ok := transition.(map[string]any)
if !ok {
continue
}
state, _ := tr["state"].(string)
timestampStr, _ := tr["timestamp"].(string)
nodeId, _ := tr["nodeId"].(string)
workerId, _ := tr["workerId"].(string)
reprName, _ := tr["reprName"].(string)
// Convert IDs from base64 to hex
nodeId = normalizeIDToHex(nodeId)
workerId = normalizeIDToHex(workerId)
var timestamp time.Time
if timestampStr != "" {
timestamp, _ = time.Parse(time.RFC3339Nano, timestampStr)
}
// DeathCause is a complex object, store as JSON string
var deathCause string
if dc, ok := tr["deathCause"]; ok {
if dcBytes, err := json.Marshal(dc); err == nil {
deathCause = string(dcBytes)
}
}
stateEvents = append(stateEvents, types.ActorStateEvent{
State: types.StateType(state),
Timestamp: timestamp,
NodeID: nodeId,
WorkerID: workerId,
ReprName: reprName,
DeathCause: deathCause,
})
}
if len(stateEvents) == 0 {
return nil
}
actorMap := h.ClusterActorMap.GetOrCreateActorMap(clusterSessionKey)
actorMap.CreateOrMergeActor(actorId, func(a *types.Actor) {
// Ensure ActorID is set (in case LIFECYCLE arrives before DEFINITION)
a.ActorID = actorId
// --- DEDUPLICATION using (State + Timestamp) as unique key ---
// Build a set of existing event keys to detect duplicates
type eventKey struct {
State string
Timestamp int64
}
existingKeys := make(map[eventKey]bool)
for _, e := range a.Events {
existingKeys[eventKey{string(e.State), e.Timestamp.UnixNano()}] = true
}
// Only append events that haven't been seen before
for _, e := range stateEvents {
key := eventKey{string(e.State), e.Timestamp.UnixNano()}
if !existingKeys[key] {
a.Events = append(a.Events, e)
existingKeys[key] = true
}
}
// Sort events by timestamp to ensure correct order
sort.Slice(a.Events, func(i, j int) bool {
return a.Events[i].Timestamp.Before(a.Events[j].Timestamp)
})
if len(a.Events) == 0 {
return
}
lastEvent := a.Events[len(a.Events)-1]
// --- UPDATE STATE ---
a.State = lastEvent.State
// --- UPDATE ADDRESS from ALIVE state ---
// NodeID and WorkerID are only populated in ALIVE state
for i := len(a.Events) - 1; i >= 0; i-- {
if a.Events[i].State == types.ALIVE && a.Events[i].NodeID != "" {
a.Address.NodeID = a.Events[i].NodeID
a.Address.WorkerID = a.Events[i].WorkerID
break
}
}
// --- UPDATE ReprName from latest ---
if lastEvent.ReprName != "" {
a.ReprName = lastEvent.ReprName
}
// --- CALCULATE StartTime (first ALIVE timestamp) ---
if a.StartTime.IsZero() {
for _, e := range a.Events {
if e.State == types.ALIVE {
a.StartTime = e.Timestamp
break
}
}
}
// --- HANDLE DEAD state ---
if lastEvent.State == types.DEAD {
a.EndTime = lastEvent.Timestamp
// Parse deathCause to extract PID, IP, errorMessage
if lastEvent.DeathCause != "" {
var deathCauseMap map[string]any
if err := json.Unmarshal([]byte(lastEvent.DeathCause), &deathCauseMap); err == nil {
if ctx, ok := deathCauseMap["actorDiedErrorContext"].(map[string]any); ok {
// Extract PID
if pid, ok := ctx["pid"].(float64); ok {
a.PID = int(pid)
}
// Extract IP address
if ip, ok := ctx["nodeIpAddress"].(string); ok {
a.Address.IPAddress = ip
}
// Extract error message as ExitDetails
if errMsg, ok := ctx["errorMessage"].(string); ok {
a.ExitDetails = errMsg
}
}
}
}
}
// --- COUNT RESTARTS ---
restartCount := 0
for _, e := range a.Events {
if e.State == types.RESTARTING {
restartCount++
}
}
a.NumRestarts = restartCount
})
case types.ACTOR_TASK_DEFINITION_EVENT:
return h.handleTaskDefinitionEvent(eventMap, clusterSessionKey, true)
case types.DRIVER_JOB_DEFINITION_EVENT:
// NOTE: When event comes in, JobID will be in base64, processing will convert it to Hex
jobDef, ok := eventMap["driverJobDefinitionEvent"]
if !ok {
return fmt.Errorf("event does not have 'driverJobDefinitionEvent'")
}
jsonDriverJobDefinition, err := json.Marshal(jobDef)
if err != nil {
return err
}
var currJob types.Job
if err := json.Unmarshal(jsonDriverJobDefinition, &currJob); err != nil {
return err
}
// Convert JobID from base64 to hex
currJob.JobID, err = utils.ConvertBase64ToHex(currJob.JobID)
if err != nil {
logrus.Errorf("Failed to convert JobID from base64 to Hex, will keep JobID in base64: %v", err)
}
// Convert DriverNodeID from base64 to hex
currJob.DriverNodeID, err = utils.ConvertBase64ToHex(currJob.DriverNodeID)
if err != nil {
logrus.Errorf("Failed to convert DriverNodeID from base64 to hex, will keep DriverNodeID in base64: %v", err)
}
jobMap := h.ClusterJobMap.GetOrCreateJobMap(clusterSessionKey)
jobMap.CreateOrMergeJob(currJob.JobID, func(j *types.Job) {
// If for some reason jobID is empty, we will keep whatever is in 'j'
var existingJobID string
if currJob.JobID == "" {
existingJobID = j.JobID
}
// ========== Lifecycle-derived fields ==========
// These fields are set by DRIVER_JOB_LIFECYCLE_EVENT
// We need to preserve them if lifecycle event arrived before definition event
existingStateTransitions := j.StateTransitions
existingState := j.State
existingStartTime := j.StartTime
existingEndTime := j.EndTime
// Overwrite with definition fields
*j = currJob
// Restore lifecycle-derived fields if they existed
if len(existingStateTransitions) > 0 {
if existingJobID != "" {
// This means that jobID was somehow empty.
j.JobID = existingJobID
}
j.StateTransitions = existingStateTransitions
j.State = existingState
j.StartTime = existingStartTime
j.EndTime = existingEndTime
}
// ========== Definition-only fields ==========
// Status, StatusTransitions, Message, DriverExitCode
// are ONLY set by DRIVER_JOB_DEFINITION_EVENT
// They are already in currJob, no need to restore
})
case types.DRIVER_JOB_LIFECYCLE_EVENT:
// NOTE: When event comes in, JobID will be in base64, processing will convert it to Hex
jobLifecycleEvent, ok := eventMap["driverJobLifecycleEvent"].(map[string]any)
if !ok {
return fmt.Errorf("invalid driverJobLifecycleEvent format")
}
// Get JobID and also convert JobID to hex from base64
// Will leave it as it if it's empty or somehow not a string
jobId, ok := jobLifecycleEvent["jobId"].(string)
if !ok {
logrus.Errorf("jobID is missing or is not a string, leaving it as is")
} else {
jobId, _ = utils.ConvertBase64ToHex(jobId)
}
stateTransitionUnstructed, _ := jobLifecycleEvent["stateTransitions"].([]any)
if len(stateTransitionUnstructed) == 0 || jobId == "" {
return nil
}
// TODO(chiayi): Will need to convert status timeline once it is added as well
// Following fields are related to status transition:
// - status
// - message
// - errorType
// - driverExitCode
var stateTransitions []types.JobStateTransition
for _, transition := range stateTransitionUnstructed {
tr, ok := transition.(map[string]any)
if !ok {
continue
}
state, _ := tr["state"].(string)
timestampStr, _ := tr["timestamp"].(string)
var timestamp time.Time
if timestampStr != "" {
timestamp, _ = time.Parse(time.RFC3339Nano, timestampStr)
}
stateTransitions = append(stateTransitions, types.JobStateTransition{
State: types.JobState(state),
Timestamp: timestamp,
})
}
if len(stateTransitions) == 0 {
return nil
}
jobMap := h.ClusterJobMap.GetOrCreateJobMap(clusterSessionKey)
jobMap.CreateOrMergeJob(jobId, func(j *types.Job) {
// TODO(chiayi): take care of status (job progress) state if part of DriverJobLifecycleEvent
j.JobID = jobId
type stateTransitionKey struct {
State string
Timestamp int64
}
existingStateKeys := make(map[stateTransitionKey]bool)
for _, t := range j.StateTransitions {
existingStateKeys[stateTransitionKey{string(t.State), t.Timestamp.UnixNano()}] = true
}
for _, t := range stateTransitions {
key := stateTransitionKey{string(t.State), t.Timestamp.UnixNano()}
if !existingStateKeys[key] {
j.StateTransitions = append(j.StateTransitions, t)
existingStateKeys[key] = true
}
}
sort.Slice(j.StateTransitions, func(i, k int) bool {
return j.StateTransitions[i].Timestamp.Before(j.StateTransitions[k].Timestamp)
})
if len(j.StateTransitions) == 0 {
return
}
lastStateTransition := j.StateTransitions[len(j.StateTransitions)-1]
j.State = lastStateTransition.State
if j.StartTime.IsZero() {
for _, t := range j.StateTransitions {
if t.State == types.CREATED {
j.StartTime = t.Timestamp
break
}
}
}
if lastStateTransition.State == types.JOBFINISHED {
j.EndTime = lastStateTransition.Timestamp
}
})
case types.NODE_DEFINITION_EVENT:
return h.handleNodeDefinitionEvent(eventMap, clusterSessionKey)
case types.NODE_LIFECYCLE_EVENT:
return h.handleNodeLifecycleEvent(eventMap, clusterSessionKey)
case types.TASK_PROFILE_EVENT:
return h.handleTaskProfileEvent(eventMap, clusterSessionKey)
default:
logrus.Infof("Event not supported, skipping: %v", eventMap)
}
return nil
}
// getAllJobEventFiles get all the job event files for the given cluster.
// Assuming that the events file object follow the format root/clustername/sessionid/job_events/{job-*}/*
func (h *EventHandler) getAllJobEventFiles(clusterInfo utils.ClusterInfo) []string {
var allJobFiles []string
clusterNameID := clusterInfo.Name + "_" + clusterInfo.Namespace
jobEventDirPrefix := clusterInfo.SessionName + "/job_events/"
jobDirList := h.reader.ListFiles(clusterNameID, jobEventDirPrefix)
for _, jobDir := range jobDirList {
// Skip non-directory entries
if !strings.HasSuffix(jobDir, "/") {
continue
}
jobDirPath := jobEventDirPrefix + jobDir
jobFiles := h.reader.ListFiles(clusterNameID, jobDirPath)
for _, jobFile := range jobFiles {
if isValidEventFile(jobFile) {
allJobFiles = append(allJobFiles, jobDirPath+jobFile)
}
}
}
return allJobFiles
}
// getAllNodeEventFiles retrieves all node event files for the given cluster
func (h *EventHandler) getAllNodeEventFiles(clusterInfo utils.ClusterInfo) []string {
clusterNameID := clusterInfo.Name + "_" + clusterInfo.Namespace
nodeEventDirPrefix := clusterInfo.SessionName + "/node_events/"
nodeEventFileNames := h.reader.ListFiles(clusterNameID, nodeEventDirPrefix)
// Filter out directories (items ending with /) and build full paths
var nodeEventFiles []string
for _, fileName := range nodeEventFileNames {
// Skip directories
if isValidEventFile(fileName) {
fullPath := nodeEventDirPrefix + fileName
nodeEventFiles = append(nodeEventFiles, fullPath)
}
}
return nodeEventFiles
}
// GetTasks returns a slice of thread-safe deep copies of all task attempts for a given cluster session.
// Deep copy ensures the returned data is safe to use after the lock is released.
func (h *EventHandler) getTasks(clusterSessionKey string) []types.Task {
h.ClusterTaskMap.RLock()
defer h.ClusterTaskMap.RUnlock()
taskMap, ok := h.ClusterTaskMap.ClusterTaskMap[clusterSessionKey]
if !ok {
// TODO(jiangjiawei1103): Add error handling.
logrus.Errorf("Task map not found for cluster session: %s", clusterSessionKey)
return []types.Task{}
}
taskMap.Lock()
defer taskMap.Unlock()
// Flatten all task attempts into a single slice with deep copy.
allTasks := make([]types.Task, 0)
for _, taskAttempts := range taskMap.TaskMap {
for _, taskAttempt := range taskAttempts {
allTasks = append(allTasks, taskAttempt.DeepCopy())
}
}
return allTasks
}
// GetActorsMap returns a thread-safe deep copy of all actors as a map for a given cluster
func (h *EventHandler) getActorsMap(clusterName string) map[string]types.Actor {
h.ClusterActorMap.RLock()
defer h.ClusterActorMap.RUnlock()
actorMap, ok := h.ClusterActorMap.ClusterActorMap[clusterName]
if !ok {
return map[string]types.Actor{}
}
actorMap.Lock()
defer actorMap.Unlock()
actors := make(map[string]types.Actor, len(actorMap.ActorMap))
for id, actor := range actorMap.ActorMap {
actors[id] = actor.DeepCopy()
}
return actors
}
func (h *EventHandler) getJobsMap(clusterName string) map[string]types.Job {
h.ClusterJobMap.RLock()
defer h.ClusterJobMap.RUnlock()
jobMap, ok := h.ClusterJobMap.ClusterJobMap[clusterName]
if !ok {
return map[string]types.Job{}
}
jobMap.Lock()
defer jobMap.Unlock()
jobs := make(map[string]types.Job, len(jobMap.JobMap))
for id, job := range jobMap.JobMap {
jobs[id] = job.DeepCopy()
}
return jobs
}
// getLogEventsByJobID returns a thread-safe deep copy of log events grouped by job ID.
func (h *EventHandler) getLogEventsByJobID(clusterSessionKey string) map[string][]types.LogEvent {
return h.ClusterLogEventMap.GetLogEventsByJobID(clusterSessionKey)
}
// handleTaskDefinitionEvent processes TASK_DEFINITION_EVENT or ACTOR_TASK_DEFINITION_EVENT and preserves the task attempt ordering.
func (h *EventHandler) handleTaskDefinitionEvent(eventMap map[string]any, clusterSessionKey string, isActorTask bool) error {
var taskDefField string
if isActorTask {
taskDefField = "actorTaskDefinitionEvent"
} else {
taskDefField = "taskDefinitionEvent"
}
taskDef, ok := eventMap[taskDefField]
if !ok {
return fmt.Errorf("event does not have '%s' field", taskDefField)
}
jsonTaskDefinition, err := json.Marshal(taskDef)
if err != nil {
return fmt.Errorf("failed to marshal %s event: %w", taskDefField, err)
}
var currTask types.Task
if err := json.Unmarshal(jsonTaskDefinition, &currTask); err != nil {
return fmt.Errorf("failed to unmarshal %s event: %w", taskDefField, err)
}
if currTask.TaskID == "" {
return fmt.Errorf("task ID is empty")
}
normalizeTaskIDsToHex(&currTask)
// Manually set the task type for an actor task.
if isActorTask {
currTask.TaskType = types.ACTOR_TASK
}
taskMap := h.ClusterTaskMap.GetOrCreateTaskMap(clusterSessionKey)
taskMap.CreateOrMergeAttempt(currTask.TaskID, currTask.TaskAttempt, func(task *types.Task) {
// Preserve existing lifecycle-derived fields to avoid overwriting.
existingStateTransitions := task.StateTransitions
existingRayErrorInfo := task.RayErrorInfo
existingNodeID := task.NodeID
existingWorkerID := task.WorkerID
existingWorkerPID := task.WorkerPID
existingIsDebuggerPaused := task.IsDebuggerPaused
existingActorReprName := task.ActorReprName
existingTaskLogInfo := task.TaskLogInfo
existingProfileData := task.ProfileData
existingFuncOrClassName := task.FuncOrClassName
existingCreationTime := task.CreationTime
existingStartTime := task.StartTime
existingEndTime := task.EndTime
*task = currTask
if len(existingStateTransitions) > 0 {
task.StateTransitions = existingStateTransitions
task.RayErrorInfo = existingRayErrorInfo
task.NodeID = existingNodeID
task.WorkerID = existingWorkerID
task.WorkerPID = existingWorkerPID
task.IsDebuggerPaused = existingIsDebuggerPaused
task.ActorReprName = existingActorReprName
task.TaskLogInfo = existingTaskLogInfo
task.State = task.GetLastState()
task.StartTime = existingStartTime
task.EndTime = existingEndTime
task.CreationTime = existingCreationTime
}
// Restore profile-derived fields (from TASK_PROFILE_EVENT)
// All three come from the same event, so check together
if existingProfileData != nil {
task.ProfileData = existingProfileData
if existingFuncOrClassName != "" {
task.FuncOrClassName = existingFuncOrClassName
}
}
})
return nil
}
// handleTaskLifecycleEvent processes TASK_LIFECYCLE_EVENT and merges state transitions for a given task attempt.
func (h *EventHandler) handleTaskLifecycleEvent(eventMap map[string]any, clusterSessionKey string) error {
taskLifecycle, ok := eventMap["taskLifecycleEvent"]
if !ok {
return fmt.Errorf("event does not have 'taskLifecycleEvent' field")
}
jsonTaskLifecycle, err := json.Marshal(taskLifecycle)
if err != nil {
return fmt.Errorf("failed to marshal task lifecycle event: %w", err)
}
var currTask types.Task
if err := json.Unmarshal(jsonTaskLifecycle, &currTask); err != nil {
return fmt.Errorf("failed to unmarshal task lifecycle event: %w", err)
}
if currTask.TaskID == "" {
return fmt.Errorf("task ID is empty")
}
normalizeTaskIDsToHex(&currTask)
// TODO(jiangjiawei1103): Clarify if there must be at least one state transition. Can one task have more than one state transition?
if len(currTask.StateTransitions) == 0 {
return fmt.Errorf("TASK_LIFECYCLE_EVENT must have at least one state transition")
}
taskMap := h.ClusterTaskMap.GetOrCreateTaskMap(clusterSessionKey)
taskMap.CreateOrMergeAttempt(currTask.TaskID, currTask.TaskAttempt, func(task *types.Task) {
// --- DEDUPLICATION using (State + Timestamp) as unique key ---
// Build a set of existing event keys to detect duplicates
type eventKey struct {
State string
Timestamp int64
}
existingKeys := make(map[eventKey]bool)
for _, e := range task.StateTransitions {
existingKeys[eventKey{string(e.State), e.Timestamp.UnixNano()}] = true
}
// Only append events that haven't been seen before
for _, e := range currTask.StateTransitions {
key := eventKey{string(e.State), e.Timestamp.UnixNano()}
if !existingKeys[key] {
task.StateTransitions = append(task.StateTransitions, e)
existingKeys[key] = true
}
}
// Sort events by timestamp to ensure correct order
sort.Slice(task.StateTransitions, func(i, j int) bool {
return task.StateTransitions[i].Timestamp.Before(task.StateTransitions[j].Timestamp)
})
if len(task.StateTransitions) == 0 {
return
}
// TODO(jiangjiawei1103): Before beta, the lifecycle-related fields are overwritten.
// In beta, the complete historical replay will be supported.
task.RayErrorInfo = currTask.RayErrorInfo
if currTask.JobID != "" {
task.JobID = currTask.JobID
}
if currTask.NodeID != "" {
task.NodeID = currTask.NodeID
}
if currTask.WorkerID != "" {
task.WorkerID = currTask.WorkerID
}
if currTask.WorkerPID != 0 {
task.WorkerPID = currTask.WorkerPID
}
task.IsDebuggerPaused = currTask.IsDebuggerPaused
task.ActorReprName = currTask.ActorReprName
task.TaskLogInfo = currTask.TaskLogInfo
task.State = task.GetLastState()
// Derive creation time, start time and end time from state transitions.
// Ref: https://github.com/ray-project/ray/blob/d0b1d151d8ea964a711e451d0ae736f8bf95b629/python/ray/util/state/common.py#L1660-L1685
for _, tr := range task.StateTransitions {
switch tr.State {
case types.PENDING_ARGS_AVAIL:
if task.CreationTime.IsZero() {
task.CreationTime = tr.Timestamp
}
case types.RUNNING:
if task.StartTime.IsZero() {
task.StartTime = tr.Timestamp
}
case types.FINISHED, types.FAILED:
// Take the latest timestamp as the end time.
task.EndTime = tr.Timestamp
}
}
})
return nil
}
// handleNodeDefinitionEvent processes NODE_DEFINITION_EVENT and merges it with the existing node map for a given cluster session.
func (h *EventHandler) handleNodeDefinitionEvent(eventMap map[string]any, clusterSessionID string) error {
nodeDef, exists := eventMap["nodeDefinitionEvent"]
if !exists {
return fmt.Errorf("event does not have 'nodeDefinitionEvent' field")
}
jsonNodeDefinition, err := json.Marshal(nodeDef)
if err != nil {
return fmt.Errorf("failed to marshal node definition event: %w", err)
}
var currNode types.Node
if err := json.Unmarshal(jsonNodeDefinition, &currNode); err != nil {
return fmt.Errorf("failed to unmarshal node definition event: %w", err)
}
if currNode.NodeID == "" {
return fmt.Errorf("node ID is empty")
}
currNode.NodeID, err = utils.ConvertBase64ToHex(currNode.NodeID)
if err != nil {
logrus.Errorf("Failed to convert node ID from base64 to hex, will keep node ID in base64: %v", err)
}
nodeMap := h.ClusterNodeMap.GetOrCreateNodeMap(clusterSessionID)
nodeMap.CreateOrMergeNode(currNode.NodeID, func(node *types.Node) {
existingStateTransitions := node.StateTransitions
*node = currNode
if len(existingStateTransitions) > 0 {
node.StateTransitions = existingStateTransitions
}
})
return nil
}
// handleNodeLifecycleEvent processes NODE_LIFECYCLE_EVENT and merges state transitions.
func (h *EventHandler) handleNodeLifecycleEvent(eventMap map[string]any, clusterSessionID string) error {
nodeLifecycle, exists := eventMap["nodeLifecycleEvent"]
if !exists {
return fmt.Errorf("event does not have 'nodeLifecycleEvent' field")
}
jsonNodeLifecycle, err := json.Marshal(nodeLifecycle)
if err != nil {
return fmt.Errorf("failed to marshal node lifecycle event: %w", err)
}
var currNode types.Node
if err := json.Unmarshal(jsonNodeLifecycle, &currNode); err != nil {
return fmt.Errorf("failed to unmarshal node lifecycle event: %w", err)
}
if currNode.NodeID == "" {
return fmt.Errorf("node ID is empty")
}
currNode.NodeID, err = utils.ConvertBase64ToHex(currNode.NodeID)
if err != nil {
logrus.Errorf("Failed to convert node ID from base64 to hex, will keep node ID in base64: %v", err)
}
// A NODE_LIFECYCLE_EVENT must have at least one state transition.
// Ref: https://github.com/ray-project/ray/blob/221a19395a836fada12ebb2bac7bff00a666faa5/src/ray/protobuf/public/events_node_lifecycle_event.proto#L58-L61.
if len(currNode.StateTransitions) == 0 {
return fmt.Errorf("node lifecycle event does not have any state transitions")
}
nodeMap := h.ClusterNodeMap.GetOrCreateNodeMap(clusterSessionID)
nodeMap.CreateOrMergeNode(currNode.NodeID, func(n *types.Node) {
// Merge state transitions.
n.StateTransitions = MergeStateTransitions[types.NodeStateTransition](
n.StateTransitions,
currNode.StateTransitions,
)
})
return nil
}
func (h *EventHandler) handleTaskProfileEvent(eventMap map[string]any, clusterSessionKey string) error {
taskProfileEvent, ok := eventMap["taskProfileEvents"]
if !ok {
return fmt.Errorf("event does not have 'taskProfileEvents'")
}
jsonBytes, err := json.Marshal(taskProfileEvent)
if err != nil {
return err
}
var profileData types.TaskProfileEvents
if err := json.Unmarshal(jsonBytes, &profileData); err != nil {
logrus.Errorf("Failed to unmarshal TASK_PROFILE_EVENT: %v", err)
return err
}
if profileData.TaskID == "" || len(profileData.ProfileEvents.Events) == 0 {
logrus.Debugf("TASK_PROFILE_EVENT has no taskId or events, skipping")
return nil
}
// Convert events to ProfileEventRaw format
var rawEvents = make([]types.ProfileEventRaw, 0, len(profileData.ProfileEvents.Events))
for _, e := range profileData.ProfileEvents.Events {
startTime, err := strconv.ParseInt(e.StartTime, 10, 64)
if err != nil {
logrus.Warnf("Failed to parse StartTime '%s': %v", e.StartTime, err)
continue
}
endTime, err := strconv.ParseInt(e.EndTime, 10, 64)
if err != nil {
logrus.Warnf("Failed to parse EndTime '%s': %v", e.EndTime, err)
continue
}
rawEvents = append(rawEvents, types.ProfileEventRaw{
EventName: e.EventName,
StartTime: startTime,
EndTime: endTime,
ExtraData: e.ExtraData,
})
}
// Convert IDs from base64 to hex before merge (consistent with JOB_DEFINITION_EVENT pattern)
profileData.TaskID, err = utils.ConvertBase64ToHex(profileData.TaskID)
if err != nil {
logrus.Errorf("Failed to convert TaskID from base64 to Hex, will use base64: %v", err)
}
profileData.JobID, err = utils.ConvertBase64ToHex(profileData.JobID)
if err != nil {
logrus.Errorf("Failed to convert JobID from base64 to Hex, will use base64: %v", err)
}
profileData.ProfileEvents.ComponentID, err = utils.ConvertBase64ToHex(profileData.ProfileEvents.ComponentID)
if err != nil {
logrus.Errorf("Failed to convert ComponentID from base64 to Hex, will use base64: %v", err)
}
taskMap := h.ClusterTaskMap.GetOrCreateTaskMap(clusterSessionKey)
taskMap.CreateOrMergeAttempt(profileData.TaskID, profileData.AttemptNumber, func(t *types.Task) {
// Ensure core identifiers are set
if t.TaskID == "" {
t.TaskID = profileData.TaskID
}
if t.JobID == "" {
t.JobID = profileData.JobID
}
// Set AttemptNumber to match the attempt we're merging into
t.TaskAttempt = profileData.AttemptNumber
// Initialize ProfileData if not exists
if t.ProfileData == nil {
t.ProfileData = &types.ProfileData{
ComponentID: profileData.ProfileEvents.ComponentID,
ComponentType: profileData.ProfileEvents.ComponentType,
NodeIPAddress: profileData.ProfileEvents.NodeIPAddress,
}
}
// Merge events with deduplication based on (eventName, startTime, endTime)
type eventKey struct {
EventName string
StartTime int64
EndTime int64
}
existingKeys := make(map[eventKey]struct{}, len(t.ProfileData.Events)+len(rawEvents))
for _, e := range t.ProfileData.Events {
existingKeys[eventKey{e.EventName, e.StartTime, e.EndTime}] = struct{}{}
}
for _, e := range rawEvents {
key := eventKey{e.EventName, e.StartTime, e.EndTime}
if _, ok := existingKeys[key]; !ok {
t.ProfileData.Events = append(t.ProfileData.Events, e)
existingKeys[key] = struct{}{}
}
}
// Extract func_or_class_name from extraData if available
for _, e := range rawEvents {
if strings.HasPrefix(e.EventName, taskPrefix) && e.ExtraData != "" {
var extra map[string]interface{}
if err := json.Unmarshal([]byte(e.ExtraData), &extra); err == nil {
if name, ok := extra["name"].(string); ok && name != "" {
// For actor methods, name might be just "increment" or "get_count"
// But eventName has the full form like "task::Counter.increment"
// Use eventName to get the full func_or_class_name
t.FuncOrClassName = strings.TrimPrefix(e.EventName, taskPrefix)
}
}
}
}
})
return nil
}
// normalizeIDToHex converts a single base64-encoded Ray ID to hex
func normalizeIDToHex(base64ID string) string {
if base64ID == "" {
return ""
}
hexID, err := utils.ConvertBase64ToHex(base64ID)
if err != nil {
logrus.Errorf("Failed to convert ID from base64 to hex, keeping original: %v", err)
return base64ID
}
return hexID
}
// normalizeTaskIDsToHex converts base64-encoded Ray IDs in task-related events:
// - TASK_DEFINITION_EVENT
// - ACTOR_TASK_DEFINITION_EVENT
// - TASK_LIFECYCLE_EVENT
//
// to hex so stored tasks match the live cluster API schema.
func normalizeTaskIDsToHex(task *types.Task) {
task.TaskID = normalizeIDToHex(task.TaskID)
task.ActorID = normalizeIDToHex(task.ActorID)
task.JobID = normalizeIDToHex(task.JobID)
task.ParentTaskID = normalizeIDToHex(task.ParentTaskID)
task.PlacementGroupID = normalizeIDToHex(task.PlacementGroupID)
task.NodeID = normalizeIDToHex(task.NodeID)
task.WorkerID = normalizeIDToHex(task.WorkerID)
}
// normalizeActorIDsToHex converts base64-encoded Ray IDs in actor events:
// - ACTOR_DEFINITION_EVENT
//
// to hex so stored actors match the live cluster API schema.
// Note: ACTOR_LIFECYCLE_EVENT IDs are normalized inline at parse time.
func normalizeActorIDsToHex(actor *types.Actor) {
actor.ActorID = normalizeIDToHex(actor.ActorID)
actor.JobID = normalizeIDToHex(actor.JobID)
actor.PlacementGroupID = normalizeIDToHex(actor.PlacementGroupID)
actor.Address.NodeID = normalizeIDToHex(actor.Address.NodeID)
actor.Address.WorkerID = normalizeIDToHex(actor.Address.WorkerID)
}
// getNodeMap returns a thread-safe deep copy of all nodes for a given cluster session.
func (h *EventHandler) getNodeMap(clusterSessionID string) map[string]types.Node {
h.ClusterNodeMap.RLock()
defer h.ClusterNodeMap.RUnlock()
nodeMap, ok := h.ClusterNodeMap.ClusterNodeMap[clusterSessionID]
if !ok {
return map[string]types.Node{}
}
nodeMap.Lock()
defer nodeMap.Unlock()
nodes := make(map[string]types.Node, len(nodeMap.NodeMap))
for id, node := range nodeMap.NodeMap {
nodes[id] = node.DeepCopy()
}
return nodes
}
// ProcessSingleSession reads all event files for a single session synchronously
// and populates the handler's in-memory maps.
//
// TODO(jiangjiawei1103): Empty event file list vs ListFiles outage is ambiguous without
// StorageReader interface surfacing errors.
func (h *EventHandler) ProcessSingleSession(ctx context.Context, clusterInfo utils.ClusterInfo) error {
clusterNameNamespace := clusterInfo.Name + "_" + clusterInfo.Namespace
clusterSessionKey := utils.BuildClusterSessionKey(clusterInfo.Name, clusterInfo.Namespace, clusterInfo.SessionName)
// ClusterLogEventMap backs only the /events endpoint, so log event read failures must not
// block marking the session as loaded and force subsequent Ray event re-processing.
logEventReader := NewLogEventReader(h.reader)
if err := logEventReader.ReadLogEvents(clusterInfo, clusterSessionKey, h.ClusterLogEventMap); err != nil {
logrus.Errorf("Incomplete Log Events read for %s: %v. /events endpoint may serve partial data.",
clusterSessionKey, err)
}
eventFileList := append(h.getAllJobEventFiles(clusterInfo), h.getAllNodeEventFiles(clusterInfo)...)
logrus.Debugf("current eventFileList for cluster %s is: %v", clusterInfo.Name, eventFileList)
rayEventsTotal := len(eventFileList)
rayEventsRead := 0
for _, eventFile := range eventFileList {
if err := ctx.Err(); err != nil {
return err
}
logrus.Debugf("Reading event file: %s", eventFile)
var eventioReader io.Reader
if strings.HasSuffix(eventFile, ".gz") {
rc, err := compression.ReadCompressedContent(h.reader, clusterNameNamespace, eventFile)
if err != nil {
logrus.Errorf("Failed to decompress event file %s: %v", eventFile, err)
continue
}
eventioReader = rc
} else {
eventioReader = h.reader.GetContent(clusterNameNamespace, eventFile)
}
if eventioReader == nil {
logrus.Errorf("Failed to get content for event file: %s, skipping", eventFile)
continue
}
eventbytes, err := io.ReadAll(eventioReader)
if closer, ok := eventioReader.(io.Closer); ok {
closer.Close()
}
if err != nil {
logrus.Errorf("Failed to read events for file %s: %v", eventFile, err)
continue
}
rayEventsRead++
// json.Unmarshal and storeEvent failures are treated as corrupt-data errors:
// retrying won't fix bad bytes, accepting partial loss.
var eventList []map[string]any
if err := json.Unmarshal(eventbytes, &eventList); err != nil {
logrus.Errorf("Failed to unmarshal events for file %s: %v", eventFile, err)
continue
}
for _, event := range eventList {
if event == nil {
continue
}
if err := h.storeEvent(clusterSessionKey, event); err != nil {
logrus.Errorf("Failed to store events for file %s: %v", eventFile, err)
continue
}
}
}
if rayEventsTotal > 0 && rayEventsRead == 0 {
return fmt.Errorf("read 0 of %d event files for %s: likely transient storage outage",
rayEventsTotal, clusterSessionKey)
}
return nil
}