mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
* [History server] fix mismatch func signature Signed-off-by: fscnick <[email protected]> * [History server] fix func description Signed-off-by: fscnick <[email protected]> --------- Signed-off-by: fscnick <[email protected]>
1088 lines
36 KiB
Go
1088 lines
36 KiB
Go
package eventserver
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/ray-project/kuberay/historyserver/pkg/eventserver/types"
|
|
"github.com/ray-project/kuberay/historyserver/pkg/utils"
|
|
)
|
|
|
|
func makeTaskEventMap(taskName, nodeID, taskID string, attempt int) map[string]any {
|
|
return map[string]any{
|
|
"eventType": string(types.TASK_DEFINITION_EVENT),
|
|
"taskDefinitionEvent": map[string]any{
|
|
"taskId": taskID,
|
|
"taskName": taskName,
|
|
"nodeId": nodeID,
|
|
"taskAttempt": attempt,
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestStoreEvent(t *testing.T) {
|
|
// IDs follow Ray's ID specification:
|
|
// ref: https://github.com/ray-project/ray/blob/f229d5376eb87b09a3fa0b991323450de84df890/src/ray/design_docs/id_specification.md
|
|
// Sizes: JobID=4B, ActorID=12B unique+JobID (16B), TaskID=8B unique+ActorID (24B), NodeID/WorkerID=28B
|
|
const (
|
|
testJobID = "aaaabbbb" // 4B
|
|
testActorID = "aaaabbbb1234aaaabbbb1234" + testJobID // 12B unique + JobID
|
|
testTaskID1 = "ccccdddd5678cccc" + testActorID // 8B unique + ActorID
|
|
testTaskID2 = "ccccdddd9012dddd" + testActorID // 8B unique + ActorID
|
|
testNodeID1 = "eeeeffff1234eeeeffff1234eeeeffff1234eeeeffff1234eeeeffff" // 28B
|
|
testNodeID2 = "eeeeffff9012eeeeffff9012eeeeffff9012eeeeffff9012eeeeffff" // 28B
|
|
testUpperNodeID1 = "AAAABBBB1234AAAABBBB1234AAAABBBB1234AAAABBBB1234AAAABBBB" // 28B uppercase
|
|
testLowerNodeID1 = "aaaabbbb1234aaaabbbb1234aaaabbbb1234aaaabbbb1234aaaabbbb" // 28B lowercase
|
|
testUpperTaskID1 = "AAAABBBB5678AAAABBBB5678AAAABBBB5678AAAABBBB5678" // 24B uppercase
|
|
testLowerTaskID1 = "aaaabbbb5678aaaabbbb5678aaaabbbb5678aaaabbbb5678" // 24B lowercase
|
|
|
|
// Base64 and Hex pairs to verify normalization of base64 to hex
|
|
testBase64ID1 = "AgAAAA=="
|
|
testBase64ID2 = "AwAAAA=="
|
|
testHexID1 = "02000000" // hex for base64 "AgAAAA=="
|
|
testHexID2 = "03000000" // hex for base64 "AwAAAA=="
|
|
|
|
testClusterName = "cluster1"
|
|
testTaskName1 = "Name_12345"
|
|
testTaskName2 = "Name_54321"
|
|
)
|
|
|
|
initialTask := types.Task{
|
|
TaskID: testTaskID1,
|
|
TaskName: testTaskName1,
|
|
NodeID: testNodeID1,
|
|
TaskAttempt: 0,
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
initialState *types.ClusterTaskMap
|
|
eventMap map[string]any
|
|
wantErr bool
|
|
wantClusterCount int
|
|
wantTaskInCluster string // Cluster to check for the task
|
|
wantTaskID string // TaskID to check
|
|
wantTasks []types.Task // Expected tasks (all attempts), nil if not applicable
|
|
}{
|
|
{
|
|
name: "unsupported event type",
|
|
initialState: &types.ClusterTaskMap{
|
|
ClusterTaskMap: make(map[string]*types.TaskMap),
|
|
},
|
|
eventMap: map[string]any{
|
|
"eventType": "UNKNOWN_TYPE",
|
|
},
|
|
wantErr: false,
|
|
wantClusterCount: 0,
|
|
},
|
|
{
|
|
name: "task event - new cluster and new task",
|
|
initialState: &types.ClusterTaskMap{
|
|
ClusterTaskMap: make(map[string]*types.TaskMap),
|
|
},
|
|
eventMap: makeTaskEventMap(testTaskName1, testNodeID1, testTaskID1, 0),
|
|
wantErr: false,
|
|
wantClusterCount: 1,
|
|
wantTaskInCluster: testClusterName,
|
|
wantTaskID: testTaskID1,
|
|
wantTasks: []types.Task{
|
|
{
|
|
TaskID: testTaskID1,
|
|
TaskName: testTaskName1,
|
|
NodeID: testNodeID1,
|
|
TaskAttempt: 0,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "task event - existing cluster, new task",
|
|
initialState: &types.ClusterTaskMap{
|
|
ClusterTaskMap: map[string]*types.TaskMap{
|
|
testClusterName: types.NewTaskMap(),
|
|
},
|
|
},
|
|
eventMap: makeTaskEventMap(testTaskName2, testNodeID2, testTaskID2, 1),
|
|
wantErr: false,
|
|
wantClusterCount: 1,
|
|
wantTaskInCluster: testClusterName,
|
|
wantTaskID: testTaskID2,
|
|
wantTasks: []types.Task{
|
|
{
|
|
TaskID: testTaskID2,
|
|
TaskName: testTaskName2,
|
|
NodeID: testNodeID2,
|
|
TaskAttempt: 1,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "task event - existing cluster and existing task with new attempt",
|
|
initialState: &types.ClusterTaskMap{
|
|
ClusterTaskMap: map[string]*types.TaskMap{
|
|
testClusterName: {
|
|
TaskMap: map[string][]types.Task{
|
|
testTaskID1: {initialTask},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
eventMap: makeTaskEventMap(testTaskName1, testNodeID1, testTaskID1, 2),
|
|
wantErr: false,
|
|
wantClusterCount: 1,
|
|
wantTaskInCluster: testClusterName,
|
|
wantTaskID: testTaskID1,
|
|
// Now expects BOTH attempts to be stored
|
|
wantTasks: []types.Task{
|
|
{
|
|
TaskID: testTaskID1,
|
|
TaskName: testTaskName1,
|
|
NodeID: testNodeID1,
|
|
TaskAttempt: 0,
|
|
},
|
|
{
|
|
TaskID: testTaskID1,
|
|
TaskName: testTaskName1,
|
|
NodeID: testNodeID1,
|
|
TaskAttempt: 2,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "task event - base64 ID is normalized to hex",
|
|
initialState: &types.ClusterTaskMap{
|
|
ClusterTaskMap: make(map[string]*types.TaskMap),
|
|
},
|
|
eventMap: makeTaskEventMap(testTaskName1, testBase64ID1, testBase64ID2, 0),
|
|
wantErr: false,
|
|
wantClusterCount: 1,
|
|
wantTaskInCluster: testClusterName,
|
|
wantTaskID: testHexID2,
|
|
wantTasks: []types.Task{
|
|
{
|
|
TaskID: testHexID2,
|
|
TaskName: testTaskName1,
|
|
NodeID: testHexID1,
|
|
TaskAttempt: 0,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "task event - uppercase hex IDs normalized to lowercase",
|
|
initialState: &types.ClusterTaskMap{
|
|
ClusterTaskMap: make(map[string]*types.TaskMap),
|
|
},
|
|
eventMap: makeTaskEventMap(testTaskName1, testUpperNodeID1, testUpperTaskID1, 0),
|
|
wantErr: false,
|
|
wantClusterCount: 1,
|
|
wantTaskInCluster: testClusterName,
|
|
wantTaskID: testLowerTaskID1,
|
|
wantTasks: []types.Task{
|
|
{
|
|
TaskID: testLowerTaskID1,
|
|
TaskName: testTaskName1,
|
|
NodeID: testLowerNodeID1,
|
|
TaskAttempt: 0,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "task event - lowercase hex IDs are returned as-is",
|
|
initialState: &types.ClusterTaskMap{
|
|
ClusterTaskMap: make(map[string]*types.TaskMap),
|
|
},
|
|
eventMap: makeTaskEventMap(testTaskName1, testLowerNodeID1, testLowerTaskID1, 0),
|
|
wantErr: false,
|
|
wantClusterCount: 1,
|
|
wantTaskInCluster: testClusterName,
|
|
wantTaskID: testLowerTaskID1,
|
|
wantTasks: []types.Task{
|
|
{
|
|
TaskID: testLowerTaskID1,
|
|
TaskName: testTaskName1,
|
|
NodeID: testLowerNodeID1,
|
|
TaskAttempt: 0,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "task event - missing taskDefinitionEvent",
|
|
initialState: &types.ClusterTaskMap{
|
|
ClusterTaskMap: make(map[string]*types.TaskMap),
|
|
},
|
|
eventMap: map[string]any{
|
|
"eventType": string(types.TASK_DEFINITION_EVENT),
|
|
},
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "task event - taskDefinitionEvent wrong type",
|
|
initialState: &types.ClusterTaskMap{
|
|
ClusterTaskMap: make(map[string]*types.TaskMap),
|
|
},
|
|
eventMap: map[string]any{
|
|
"eventType": string(types.TASK_DEFINITION_EVENT),
|
|
"taskDefinitionEvent": "not a map",
|
|
},
|
|
wantErr: true, // Marshal will fail
|
|
},
|
|
{
|
|
name: "task event - invalid task structure",
|
|
initialState: &types.ClusterTaskMap{
|
|
ClusterTaskMap: make(map[string]*types.TaskMap),
|
|
},
|
|
eventMap: map[string]any{
|
|
"eventType": string(types.TASK_DEFINITION_EVENT),
|
|
"taskDefinitionEvent": map[string]any{
|
|
"taskId": 123, // Should be string
|
|
"taskAttempt": 0,
|
|
},
|
|
},
|
|
wantErr: true, // Unmarshal will fail
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
h := &EventHandler{
|
|
ClusterTaskMap: tt.initialState,
|
|
}
|
|
if h.ClusterTaskMap == nil {
|
|
h.ClusterTaskMap = &types.ClusterTaskMap{
|
|
ClusterTaskMap: make(map[string]*types.TaskMap),
|
|
}
|
|
}
|
|
|
|
err := h.storeEvent(testClusterName, tt.eventMap)
|
|
|
|
if (err != nil) != tt.wantErr {
|
|
t.Fatalf("storeEvent() error = %v, wantErr %v", err, tt.wantErr)
|
|
}
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
gotClusterCount := len(h.ClusterTaskMap.ClusterTaskMap)
|
|
|
|
if gotClusterCount != tt.wantClusterCount {
|
|
t.Errorf("storeEvent() resulted in %d clusters, want %d", gotClusterCount, tt.wantClusterCount)
|
|
}
|
|
|
|
if tt.wantTasks != nil {
|
|
clusterObj, clusterExists := h.ClusterTaskMap.ClusterTaskMap[tt.wantTaskInCluster]
|
|
|
|
if !clusterExists {
|
|
t.Fatalf("storeEvent() cluster %s not found", tt.wantTaskInCluster)
|
|
}
|
|
|
|
clusterObj.Lock()
|
|
defer clusterObj.Unlock()
|
|
gotTasks, taskExists := clusterObj.TaskMap[tt.wantTaskID]
|
|
if !taskExists {
|
|
t.Fatalf("storeEvent() task %s not found in cluster %s", tt.wantTaskID, tt.wantTaskInCluster)
|
|
}
|
|
|
|
if diff := cmp.Diff(tt.wantTasks, gotTasks); diff != "" {
|
|
t.Errorf("storeEvent() tasks mismatch (-want +got):\n%s", diff)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestTaskLifecycleEventDeduplication verifies that duplicate events are correctly filtered
|
|
// and out-of-order events are properly sorted
|
|
func TestTaskLifecycleEventDeduplication(t *testing.T) {
|
|
// IDs follow Ray's ID spec; see TestStoreEvent for rationale.
|
|
const (
|
|
testJobID = "aaaabbbb" // 4B
|
|
testActorID = "aaaabbbb1234aaaabbbb1234" + testJobID // 12B unique + JobID
|
|
testTaskID = "ccccdddd5678cccc" + testActorID // 8B unique + ActorID
|
|
testNodeID = "eeeeffff1234eeeeffff1234eeeeffff1234eeeeffff1234eeeeffff" // 28B
|
|
testWorkerID = "eeeeffff0000eeeeffff0000eeeeffff0000eeeeffff0000eeeeffff" // 28B
|
|
testClusterName = "cluster1"
|
|
)
|
|
|
|
// Helper to create a StateEvent
|
|
makeStateEvent := func(state types.TaskStatus, timestampNano int64) types.TaskStateTransition {
|
|
return types.TaskStateTransition{
|
|
State: state,
|
|
Timestamp: time.Unix(0, timestampNano),
|
|
}
|
|
}
|
|
|
|
// Helper to create a TASK_LIFECYCLE_EVENT map
|
|
makeLifecycleEvent := func(transitions []map[string]any) map[string]any {
|
|
// Convert []map[string]any to []any for proper type assertion in storeEvent
|
|
transitionsAny := make([]any, len(transitions))
|
|
for i, t := range transitions {
|
|
transitionsAny[i] = t
|
|
}
|
|
return map[string]any{
|
|
"eventType": string(types.TASK_LIFECYCLE_EVENT),
|
|
"taskLifecycleEvent": map[string]any{
|
|
"taskId": testTaskID,
|
|
"taskAttempt": float64(0),
|
|
"stateTransitions": transitionsAny,
|
|
"nodeId": testNodeID,
|
|
"workerId": testWorkerID,
|
|
},
|
|
}
|
|
}
|
|
|
|
// Helper to create state transition
|
|
makeTransition := func(state string, timestampNano int64) map[string]any {
|
|
return map[string]any{
|
|
"state": state,
|
|
"timestamp": time.Unix(0, timestampNano).Format(time.RFC3339Nano),
|
|
}
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
existingEvents []types.TaskStateTransition // Events already in the task
|
|
newTransitions []map[string]any // New transitions to process
|
|
wantEvents []types.TaskStateTransition // Expected final events (sorted by timestamp)
|
|
wantState types.TaskStatus // Expected final state
|
|
}{
|
|
{
|
|
name: "Scenario 1: Normal deduplication - same events processed twice",
|
|
existingEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.PENDING_NODE_ASSIGNMENT, 1000),
|
|
makeStateEvent(types.RUNNING, 2000),
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("PENDING_NODE_ASSIGNMENT", 1000), // Duplicate
|
|
makeTransition("RUNNING", 2000), // Duplicate
|
|
},
|
|
wantEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.PENDING_NODE_ASSIGNMENT, 1000),
|
|
makeStateEvent(types.RUNNING, 2000),
|
|
},
|
|
wantState: types.RUNNING,
|
|
},
|
|
{
|
|
name: "Scenario 2: Out-of-order events - B(t=2) arrives after A(t=1), C(t=3)",
|
|
existingEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.PENDING_NODE_ASSIGNMENT, 1000), // A
|
|
makeStateEvent(types.FINISHED, 3000), // C
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("RUNNING", 2000), // B - should be inserted in the middle
|
|
},
|
|
wantEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.PENDING_NODE_ASSIGNMENT, 1000), // A
|
|
makeStateEvent(types.RUNNING, 2000), // B - now in correct position
|
|
makeStateEvent(types.FINISHED, 3000), // C
|
|
},
|
|
wantState: types.FINISHED,
|
|
},
|
|
{
|
|
name: "Scenario 3: Same timestamp, different states - both should be kept",
|
|
existingEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.PENDING_NODE_ASSIGNMENT, 1000),
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("RUNNING", 1000), // Same timestamp, different state
|
|
},
|
|
wantEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.PENDING_NODE_ASSIGNMENT, 1000),
|
|
makeStateEvent(types.RUNNING, 1000),
|
|
},
|
|
wantState: types.RUNNING, // Last after sort (order of same timestamp is stable)
|
|
},
|
|
{
|
|
name: "Scenario 4: Exact duplicate event - only one should remain",
|
|
existingEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.RUNNING, 1000),
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("RUNNING", 1000), // Exact duplicate
|
|
makeTransition("RUNNING", 1000), // Another duplicate
|
|
},
|
|
wantEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.RUNNING, 1000), // Only one
|
|
},
|
|
wantState: types.RUNNING,
|
|
},
|
|
{
|
|
name: "Scenario 5: Partial overlap - existing [A,B], new [B,C] -> result [A,B,C]",
|
|
existingEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.PENDING_NODE_ASSIGNMENT, 1000), // A
|
|
makeStateEvent(types.RUNNING, 2000), // B
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("RUNNING", 2000), // B - duplicate
|
|
makeTransition("FINISHED", 3000), // C - new
|
|
},
|
|
wantEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.PENDING_NODE_ASSIGNMENT, 1000), // A
|
|
makeStateEvent(types.RUNNING, 2000), // B
|
|
makeStateEvent(types.FINISHED, 3000), // C
|
|
},
|
|
wantState: types.FINISHED,
|
|
},
|
|
{
|
|
name: "Scenario 6: Empty initial events - add new events",
|
|
existingEvents: []types.TaskStateTransition{},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("PENDING_NODE_ASSIGNMENT", 1000),
|
|
makeTransition("RUNNING", 2000),
|
|
},
|
|
wantEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.PENDING_NODE_ASSIGNMENT, 1000),
|
|
makeStateEvent(types.RUNNING, 2000),
|
|
},
|
|
wantState: types.RUNNING,
|
|
},
|
|
{
|
|
name: "Scenario 7: Multiple reprocessing cycles - events should not grow",
|
|
existingEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.PENDING_NODE_ASSIGNMENT, 1000),
|
|
makeStateEvent(types.RUNNING, 2000),
|
|
makeStateEvent(types.FINISHED, 3000),
|
|
},
|
|
newTransitions: []map[string]any{
|
|
// Simulating reprocess of same file
|
|
makeTransition("PENDING_NODE_ASSIGNMENT", 1000),
|
|
makeTransition("RUNNING", 2000),
|
|
makeTransition("FINISHED", 3000),
|
|
},
|
|
wantEvents: []types.TaskStateTransition{
|
|
makeStateEvent(types.PENDING_NODE_ASSIGNMENT, 1000),
|
|
makeStateEvent(types.RUNNING, 2000),
|
|
makeStateEvent(types.FINISHED, 3000),
|
|
},
|
|
wantState: types.FINISHED,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
h := NewEventHandler(nil)
|
|
|
|
// Pre-populate existing events if any
|
|
if len(tt.existingEvents) > 0 {
|
|
taskMap := h.ClusterTaskMap.GetOrCreateTaskMap(testClusterName)
|
|
taskMap.CreateOrMergeAttempt(testTaskID, 0, func(task *types.Task) {
|
|
task.TaskID = testTaskID
|
|
task.StateTransitions = tt.existingEvents
|
|
if len(tt.existingEvents) > 0 {
|
|
task.State = tt.existingEvents[len(tt.existingEvents)-1].State
|
|
}
|
|
})
|
|
}
|
|
|
|
// Process the lifecycle event
|
|
eventMap := makeLifecycleEvent(tt.newTransitions)
|
|
err := h.storeEvent(testClusterName, eventMap)
|
|
if err != nil {
|
|
t.Fatalf("storeEvent() unexpected error: %v", err)
|
|
}
|
|
|
|
// Get the task and verify
|
|
taskMap := h.ClusterTaskMap.GetOrCreateTaskMap(testClusterName)
|
|
taskMap.Lock()
|
|
defer taskMap.Unlock()
|
|
|
|
tasks, exists := taskMap.TaskMap[testTaskID]
|
|
if !exists || len(tasks) == 0 {
|
|
t.Fatal("Task not found after processing")
|
|
}
|
|
|
|
task := tasks[0]
|
|
|
|
// Verify events
|
|
if diff := cmp.Diff(tt.wantEvents, task.StateTransitions); diff != "" {
|
|
t.Errorf("Events mismatch (-want +got):\n%s", diff)
|
|
}
|
|
|
|
// Verify final state
|
|
if task.State != tt.wantState {
|
|
t.Errorf("State = %v, want %v", task.State, tt.wantState)
|
|
}
|
|
|
|
// Verify event count (important for deduplication)
|
|
if len(task.StateTransitions) != len(tt.wantEvents) {
|
|
t.Errorf("Event count = %d, want %d", len(task.StateTransitions), len(tt.wantEvents))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestActorLifecycleEventDeduplication verifies that duplicate actor events are correctly filtered
|
|
func TestActorLifecycleEventDeduplication(t *testing.T) {
|
|
// IDs follow Ray's ID spec; see TestStoreEvent for rationale.
|
|
const (
|
|
testJobID = "aaaabbbb" // 4B
|
|
testActorID = "aaaabbbb1234aaaabbbb1234" + testJobID // 12B unique + JobID
|
|
testClusterName = "cluster1"
|
|
)
|
|
|
|
// Helper to create an ActorStateEvent
|
|
makeActorStateEvent := func(state types.StateType, timestampNano int64) types.ActorStateEvent {
|
|
return types.ActorStateEvent{
|
|
State: state,
|
|
Timestamp: time.Unix(0, timestampNano),
|
|
}
|
|
}
|
|
|
|
// Helper to create an ACTOR_LIFECYCLE_EVENT map
|
|
makeActorLifecycleEvent := func(transitions []map[string]any) map[string]any {
|
|
// Convert []map[string]any to []any for proper type assertion in storeEvent
|
|
transitionsAny := make([]any, len(transitions))
|
|
for i, t := range transitions {
|
|
transitionsAny[i] = t
|
|
}
|
|
return map[string]any{
|
|
"eventType": string(types.ACTOR_LIFECYCLE_EVENT),
|
|
"actorLifecycleEvent": map[string]any{
|
|
"actorId": testActorID,
|
|
"stateTransitions": transitionsAny,
|
|
},
|
|
}
|
|
}
|
|
|
|
// Helper to create state transition
|
|
makeTransition := func(state string, timestampNano int64) map[string]any {
|
|
return map[string]any{
|
|
"state": state,
|
|
"timestamp": time.Unix(0, timestampNano).Format(time.RFC3339Nano),
|
|
}
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
existingEvents []types.ActorStateEvent
|
|
newTransitions []map[string]any
|
|
wantEvents []types.ActorStateEvent
|
|
wantState types.StateType
|
|
}{
|
|
{
|
|
name: "Actor: Normal deduplication",
|
|
existingEvents: []types.ActorStateEvent{
|
|
makeActorStateEvent(types.PENDING_CREATION, 1000),
|
|
makeActorStateEvent(types.ALIVE, 2000),
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("PENDING_CREATION", 1000),
|
|
makeTransition("ALIVE", 2000),
|
|
},
|
|
wantEvents: []types.ActorStateEvent{
|
|
makeActorStateEvent(types.PENDING_CREATION, 1000),
|
|
makeActorStateEvent(types.ALIVE, 2000),
|
|
},
|
|
wantState: types.ALIVE,
|
|
},
|
|
{
|
|
name: "Actor: Out-of-order with sort",
|
|
existingEvents: []types.ActorStateEvent{
|
|
makeActorStateEvent(types.PENDING_CREATION, 1000),
|
|
makeActorStateEvent(types.DEAD, 3000),
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("ALIVE", 2000), // Should be inserted between
|
|
},
|
|
wantEvents: []types.ActorStateEvent{
|
|
makeActorStateEvent(types.PENDING_CREATION, 1000),
|
|
makeActorStateEvent(types.ALIVE, 2000),
|
|
makeActorStateEvent(types.DEAD, 3000),
|
|
},
|
|
wantState: types.DEAD,
|
|
},
|
|
{
|
|
name: "Actor: Exact duplicate should not increase count",
|
|
existingEvents: []types.ActorStateEvent{
|
|
makeActorStateEvent(types.ALIVE, 1000),
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("ALIVE", 1000),
|
|
makeTransition("ALIVE", 1000),
|
|
makeTransition("ALIVE", 1000),
|
|
},
|
|
wantEvents: []types.ActorStateEvent{
|
|
makeActorStateEvent(types.ALIVE, 1000),
|
|
},
|
|
wantState: types.ALIVE,
|
|
},
|
|
{
|
|
name: "Actor: Same timestamp different states should both be kept",
|
|
existingEvents: []types.ActorStateEvent{
|
|
makeActorStateEvent(types.PENDING_CREATION, 1000),
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("ALIVE", 1000), // Same timestamp, different state
|
|
},
|
|
wantEvents: []types.ActorStateEvent{
|
|
makeActorStateEvent(types.PENDING_CREATION, 1000),
|
|
makeActorStateEvent(types.ALIVE, 1000),
|
|
},
|
|
wantState: types.ALIVE,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
h := NewEventHandler(nil)
|
|
|
|
// Pre-populate existing events
|
|
if len(tt.existingEvents) > 0 {
|
|
actorMap := h.ClusterActorMap.GetOrCreateActorMap(testClusterName)
|
|
actorMap.CreateOrMergeActor(testActorID, func(a *types.Actor) {
|
|
a.ActorID = testActorID
|
|
a.Events = tt.existingEvents
|
|
if len(tt.existingEvents) > 0 {
|
|
a.State = tt.existingEvents[len(tt.existingEvents)-1].State
|
|
}
|
|
})
|
|
}
|
|
|
|
// Process the lifecycle event
|
|
eventMap := makeActorLifecycleEvent(tt.newTransitions)
|
|
err := h.storeEvent(testClusterName, eventMap)
|
|
if err != nil {
|
|
t.Fatalf("storeEvent() unexpected error: %v", err)
|
|
}
|
|
|
|
// Get the actor and verify
|
|
actor, found := h.getActorsMap(testClusterName)[testActorID]
|
|
if !found {
|
|
t.Fatal("Actor not found after processing")
|
|
}
|
|
|
|
// Verify events count (most important for dedup testing)
|
|
if len(actor.Events) != len(tt.wantEvents) {
|
|
t.Errorf("Event count = %d, want %d", len(actor.Events), len(tt.wantEvents))
|
|
}
|
|
|
|
// Verify final state
|
|
if actor.State != tt.wantState {
|
|
t.Errorf("State = %v, want %v", actor.State, tt.wantState)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestDriverJobLifeCycleEventDuplication tests that duplicate events are properly filtered and sorted
|
|
// TODO(chiayi): Update once more fields are added to driver job event
|
|
func TestDriverJobLifecycleEventDuplication(t *testing.T) {
|
|
// IDs follow Ray's ID spec; see TestStoreEvent for rationale.
|
|
const (
|
|
testJobID = "aaaabbbb" // 4B
|
|
testClusterName = "cluster1"
|
|
)
|
|
|
|
makeDriverJobStateTransitionEvent := func(state types.JobState, timestampNano int64) types.JobStateTransition {
|
|
return types.JobStateTransition{
|
|
State: state,
|
|
Timestamp: time.Unix(0, timestampNano),
|
|
}
|
|
}
|
|
|
|
makeDriverJobLifecycleEvent := func(transitions []map[string]any) map[string]any {
|
|
transitionsAny := make([]any, len(transitions))
|
|
for i, t := range transitions {
|
|
transitionsAny[i] = t
|
|
}
|
|
return map[string]any{
|
|
"eventType": string(types.DRIVER_JOB_LIFECYCLE_EVENT),
|
|
"driverJobLifecycleEvent": map[string]any{
|
|
"jobId": testJobID,
|
|
"stateTransitions": transitionsAny,
|
|
},
|
|
}
|
|
}
|
|
|
|
makeTransition := func(state string, timestampNano int64) map[string]any {
|
|
return map[string]any{
|
|
"state": state,
|
|
"timestamp": time.Unix(0, timestampNano).Format(time.RFC3339Nano),
|
|
}
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
// Available Job States are UNSPECIFIED, CREATED, JOBFINISHED
|
|
existingTransitions []types.JobStateTransition
|
|
newTransitions []map[string]any
|
|
wantTransitions []types.JobStateTransition
|
|
wantState types.JobState
|
|
}{
|
|
{
|
|
name: "Driver Job: Same event is processed twice",
|
|
existingTransitions: []types.JobStateTransition{
|
|
makeDriverJobStateTransitionEvent(types.UNSPECIFIED, 1000),
|
|
makeDriverJobStateTransitionEvent(types.CREATED, 2000),
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("UNSPECIFIED", 1000),
|
|
makeTransition("CREATED", 2000),
|
|
},
|
|
wantTransitions: []types.JobStateTransition{
|
|
makeDriverJobStateTransitionEvent(types.UNSPECIFIED, 1000),
|
|
makeDriverJobStateTransitionEvent(types.CREATED, 2000),
|
|
},
|
|
wantState: types.CREATED,
|
|
},
|
|
{
|
|
name: "Driver Job: Empty existing transitions",
|
|
existingTransitions: []types.JobStateTransition{},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("UNSPECIFIED", 1000),
|
|
makeTransition("CREATED", 2000),
|
|
makeTransition("FINISHED", 3000),
|
|
},
|
|
wantTransitions: []types.JobStateTransition{
|
|
makeDriverJobStateTransitionEvent(types.UNSPECIFIED, 1000),
|
|
makeDriverJobStateTransitionEvent(types.CREATED, 2000),
|
|
makeDriverJobStateTransitionEvent(types.JOBFINISHED, 3000),
|
|
},
|
|
wantState: types.JOBFINISHED,
|
|
},
|
|
{
|
|
name: "Driver Job: Out of order transition event",
|
|
existingTransitions: []types.JobStateTransition{
|
|
makeDriverJobStateTransitionEvent(types.UNSPECIFIED, 1000),
|
|
makeDriverJobStateTransitionEvent(types.JOBFINISHED, 3000),
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("CREATED", 2000),
|
|
},
|
|
wantTransitions: []types.JobStateTransition{
|
|
makeDriverJobStateTransitionEvent(types.UNSPECIFIED, 1000),
|
|
makeDriverJobStateTransitionEvent(types.CREATED, 2000),
|
|
makeDriverJobStateTransitionEvent(types.JOBFINISHED, 3000),
|
|
},
|
|
wantState: types.JOBFINISHED,
|
|
},
|
|
{
|
|
name: "Driver Job: Multiple duplicate of same transition event",
|
|
existingTransitions: []types.JobStateTransition{
|
|
makeDriverJobStateTransitionEvent(types.CREATED, 2000),
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("CREATED", 2000),
|
|
makeTransition("CREATED", 2000),
|
|
makeTransition("CREATED", 2000),
|
|
makeTransition("CREATED", 2000),
|
|
},
|
|
wantTransitions: []types.JobStateTransition{
|
|
makeDriverJobStateTransitionEvent(types.CREATED, 2000),
|
|
},
|
|
wantState: types.CREATED,
|
|
},
|
|
{
|
|
name: "Driver Job: Different transition events with same time stamp, all be kept",
|
|
existingTransitions: []types.JobStateTransition{
|
|
makeDriverJobStateTransitionEvent(types.UNSPECIFIED, 1000),
|
|
},
|
|
newTransitions: []map[string]any{
|
|
makeTransition("CREATED", 1000),
|
|
makeTransition("CREATED", 2000),
|
|
makeTransition("FINISHED", 2000),
|
|
},
|
|
wantTransitions: []types.JobStateTransition{
|
|
makeDriverJobStateTransitionEvent(types.UNSPECIFIED, 1000),
|
|
makeDriverJobStateTransitionEvent(types.CREATED, 1000),
|
|
makeDriverJobStateTransitionEvent(types.CREATED, 2000),
|
|
makeDriverJobStateTransitionEvent(types.JOBFINISHED, 2000),
|
|
},
|
|
wantState: types.JOBFINISHED,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
h := NewEventHandler(nil)
|
|
|
|
if len(tt.existingTransitions) > 0 {
|
|
jobMap := h.ClusterJobMap.GetOrCreateJobMap(testClusterName)
|
|
jobMap.CreateOrMergeJob(testJobID, func(job *types.Job) {
|
|
job.JobID = testJobID
|
|
job.StateTransitions = tt.existingTransitions
|
|
if len(tt.existingTransitions) > 0 {
|
|
job.State = tt.existingTransitions[len(tt.existingTransitions)-1].State
|
|
}
|
|
})
|
|
}
|
|
|
|
eventMap := makeDriverJobLifecycleEvent(tt.newTransitions)
|
|
err := h.storeEvent(testClusterName, eventMap)
|
|
if err != nil {
|
|
t.Fatalf("storeEvent() unexpected error: %v", err)
|
|
}
|
|
|
|
job, exists := h.getJobsMap(testClusterName)[testJobID]
|
|
if !exists {
|
|
t.Fatal("Job not found after processing")
|
|
}
|
|
|
|
if len(job.StateTransitions) != len(tt.wantTransitions) {
|
|
t.Errorf("Actual transition count %d but expected %d", len(job.StateTransitions), len(tt.wantTransitions))
|
|
}
|
|
|
|
if job.State != tt.wantState {
|
|
t.Errorf("Actual State %v but expected %v", job.State, tt.wantState)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestMultipleReprocessingCycles simulates hourly reprocessing and verifies no memory growth
|
|
func TestMultipleReprocessingCycles(t *testing.T) {
|
|
h := NewEventHandler(nil)
|
|
|
|
// IDs follow Ray's ID spec; see TestStoreEvent for rationale.
|
|
const (
|
|
testJobID = "aaaabbbb" // 4B
|
|
testActorID = "aaaabbbb1234aaaabbbb1234" + testJobID // 12B unique + JobID
|
|
testTaskID = "ccccdddd5678cccc" + testActorID // 8B unique + ActorID
|
|
testNodeID = "eeeeffff1234eeeeffff1234eeeeffff1234eeeeffff1234eeeeffff" // 28B
|
|
testWorkerID = "eeeeffff0000eeeeffff0000eeeeffff0000eeeeffff0000eeeeffff" // 28B
|
|
testClusterName = "cluster1"
|
|
)
|
|
|
|
// The same events that would be in an event file
|
|
// Use []any to match what storeEvent expects from JSON parsing
|
|
transitions := []any{
|
|
map[string]any{"state": "PENDING_NODE_ASSIGNMENT", "timestamp": time.Unix(0, 1000).Format(time.RFC3339Nano)},
|
|
map[string]any{"state": "RUNNING", "timestamp": time.Unix(0, 2000).Format(time.RFC3339Nano)},
|
|
map[string]any{"state": "FINISHED", "timestamp": time.Unix(0, 3000).Format(time.RFC3339Nano)},
|
|
}
|
|
|
|
eventMap := map[string]any{
|
|
"eventType": string(types.TASK_LIFECYCLE_EVENT),
|
|
"taskLifecycleEvent": map[string]any{
|
|
"taskId": testTaskID,
|
|
"taskAttempt": float64(0),
|
|
"stateTransitions": transitions,
|
|
"nodeId": testNodeID,
|
|
"workerId": testWorkerID,
|
|
},
|
|
}
|
|
|
|
// Simulate 10 hourly reprocessing cycles
|
|
for cycle := 0; cycle < 10; cycle++ {
|
|
err := h.storeEvent(testClusterName, eventMap)
|
|
if err != nil {
|
|
t.Fatalf("Cycle %d: storeEvent() error = %v", cycle, err)
|
|
}
|
|
|
|
// Check event count after each cycle
|
|
taskMap := h.ClusterTaskMap.GetOrCreateTaskMap(testClusterName)
|
|
taskMap.Lock()
|
|
tasks := taskMap.TaskMap[testTaskID]
|
|
eventCount := len(tasks[0].StateTransitions)
|
|
taskMap.Unlock()
|
|
|
|
// Should always be exactly 3 events, never growing
|
|
if eventCount != 3 {
|
|
t.Errorf("Cycle %d: Event count = %d, want 3 (events are duplicating!)", cycle, eventCount)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNormalizeIDToHex(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
input string
|
|
expected string
|
|
}{
|
|
{
|
|
name: "empty string returns empty",
|
|
input: "",
|
|
expected: "",
|
|
},
|
|
{
|
|
name: "valid base64 converts to hex",
|
|
input: "AgAAAA==",
|
|
expected: "02000000",
|
|
},
|
|
{
|
|
name: "invalid base64 returns original",
|
|
input: "not_valid_base64!!!",
|
|
expected: "not_valid_base64!!!",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := normalizeIDToHex(tt.input)
|
|
if got != tt.expected {
|
|
t.Errorf("normalizeIDToHex(%q) = %q, want %q", tt.input, got, tt.expected)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNormalizeActorIDsToHex(t *testing.T) {
|
|
base64Val := "AgAAAA=="
|
|
expectedHex, _ := utils.ConvertBase64ToHex(base64Val)
|
|
|
|
actor := types.Actor{
|
|
ActorID: base64Val,
|
|
JobID: base64Val,
|
|
PlacementGroupID: base64Val,
|
|
Address: types.Address{
|
|
NodeID: base64Val,
|
|
WorkerID: base64Val,
|
|
},
|
|
}
|
|
|
|
normalizeActorIDsToHex(&actor)
|
|
|
|
if actor.ActorID != expectedHex {
|
|
t.Errorf("ActorID = %q, want %q", actor.ActorID, expectedHex)
|
|
}
|
|
if actor.JobID != expectedHex {
|
|
t.Errorf("JobID = %q, want %q", actor.JobID, expectedHex)
|
|
}
|
|
if actor.PlacementGroupID != expectedHex {
|
|
t.Errorf("PlacementGroupID = %q, want %q", actor.PlacementGroupID, expectedHex)
|
|
}
|
|
if actor.Address.NodeID != expectedHex {
|
|
t.Errorf("Address.NodeID = %q, want %q", actor.Address.NodeID, expectedHex)
|
|
}
|
|
if actor.Address.WorkerID != expectedHex {
|
|
t.Errorf("Address.WorkerID = %q, want %q", actor.Address.WorkerID, expectedHex)
|
|
}
|
|
}
|
|
|
|
func TestProcessSingleSession(t *testing.T) {
|
|
clusterInfo := utils.ClusterInfo{Name: "cluster", Namespace: "ns", SessionName: "session1"}
|
|
|
|
t.Run("returns error when every listed file fails I/O", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
mock.addDir("cluster_ns", "session1/job_events/", []string{"job-01000000/"})
|
|
mock.addDir("cluster_ns", "session1/job_events/job-01000000/",
|
|
[]string{"01000000-2024-01-01-00.gz"})
|
|
mock.addDir("cluster_ns", "session1/node_events/",
|
|
[]string{"node1-2024-01-01-00.gz"})
|
|
mock.addDir("cluster_ns", "session1/logs", []string{})
|
|
|
|
h := NewEventHandler(mock)
|
|
err := h.ProcessSingleSession(context.Background(), clusterInfo)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "read 0 of 2")
|
|
})
|
|
|
|
t.Run("empty file list returns nil (legit empty session)", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
mock.addDir("cluster_ns", "session1/job_events/", []string{})
|
|
mock.addDir("cluster_ns", "session1/node_events/", []string{})
|
|
mock.addDir("cluster_ns", "session1/logs", []string{})
|
|
|
|
h := NewEventHandler(mock)
|
|
err := h.ProcessSingleSession(context.Background(), clusterInfo)
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
t.Run("partial success does not return error", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
mock.addDir("cluster_ns", "session1/node_events/",
|
|
[]string{"node1-2024-01-01-00.gz", "node2-2024-01-01-00.gz"})
|
|
|
|
var buf bytes.Buffer
|
|
gw := gzip.NewWriter(&buf)
|
|
_, err := gw.Write([]byte(`[{"eventType":"NODE_DEFINITION_EVENT","nodeDefinitionEvent":{"nodeId":"YWJjZA=="}}]`))
|
|
require.NoError(t, err)
|
|
require.NoError(t, gw.Close())
|
|
|
|
mock.addFile("cluster_ns", "session1/node_events/node1-2024-01-01-00.gz", buf.String())
|
|
mock.addDir("cluster_ns", "session1/job_events/", []string{})
|
|
mock.addDir("cluster_ns", "session1/logs", []string{})
|
|
|
|
h := NewEventHandler(mock)
|
|
err = h.ProcessSingleSession(context.Background(), clusterInfo)
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
t.Run("all corrupt JSON does not return error", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
mock.addDir("cluster_ns", "session1/node_events/",
|
|
[]string{"node1-2024-01-01-00.gz", "node2-2024-01-01-00.gz"})
|
|
|
|
var buf bytes.Buffer
|
|
gw := gzip.NewWriter(&buf)
|
|
_, err := gw.Write([]byte("this is not json"))
|
|
require.NoError(t, err)
|
|
require.NoError(t, gw.Close())
|
|
|
|
mock.addFile("cluster_ns", "session1/node_events/node1-2024-01-01-00.gz", buf.String())
|
|
mock.addFile("cluster_ns", "session1/node_events/node2-2024-01-01-00.gz", buf.String())
|
|
mock.addDir("cluster_ns", "session1/job_events/", []string{})
|
|
mock.addDir("cluster_ns", "session1/logs", []string{})
|
|
|
|
h := NewEventHandler(mock)
|
|
err = h.ProcessSingleSession(context.Background(), clusterInfo)
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
t.Run("log events failure alone does not surface as error", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
mock.addDir("cluster_ns", "session1/job_events/", []string{})
|
|
mock.addDir("cluster_ns", "session1/node_events/", []string{})
|
|
mock.addDir("cluster_ns", "session1/logs", []string{"node1/"})
|
|
mock.addDir("cluster_ns", "session1/logs/node1/events", []string{"event_GCS.log"})
|
|
|
|
h := NewEventHandler(mock)
|
|
err := h.ProcessSingleSession(context.Background(), clusterInfo)
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
t.Run("ray events failure surfaces; log events failure stays silent", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
mock.addDir("cluster_ns", "session1/node_events/", []string{"node1-2024-01-01-00.gz"})
|
|
mock.addDir("cluster_ns", "session1/job_events/", []string{})
|
|
mock.addDir("cluster_ns", "session1/logs", []string{"node1/"})
|
|
mock.addDir("cluster_ns", "session1/logs/node1/events", []string{"event_GCS.log"})
|
|
|
|
h := NewEventHandler(mock)
|
|
err := h.ProcessSingleSession(context.Background(), clusterInfo)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "read 0 of 1 event files")
|
|
assert.NotContains(t, err.Error(), "log event")
|
|
})
|
|
|
|
t.Run("transparent decompression of compressed .gz files and reading uncompressed legacy files", func(t *testing.T) {
|
|
mock := newLogEventMockReader()
|
|
mock.addDir("cluster_ns", "session1/node_events/",
|
|
[]string{"node1-2024-01-01-00.gz", "node2-2024-01-01-00"})
|
|
|
|
// 1. Compress node1 event file
|
|
var buf bytes.Buffer
|
|
gw := gzip.NewWriter(&buf)
|
|
_, err := gw.Write([]byte(`[{"eventType":"NODE_DEFINITION_EVENT","nodeDefinitionEvent":{"nodeId":"YWJjZA=="}}]`))
|
|
require.NoError(t, err)
|
|
require.NoError(t, gw.Close())
|
|
mock.addFile("cluster_ns", "session1/node_events/node1-2024-01-01-00.gz", buf.String())
|
|
|
|
// 2. Keep node2 event file uncompressed
|
|
mock.addFile("cluster_ns", "session1/node_events/node2-2024-01-01-00",
|
|
`[{"eventType":"NODE_DEFINITION_EVENT","nodeDefinitionEvent":{"nodeId":"ZWZnaA=="}}]`)
|
|
|
|
mock.addDir("cluster_ns", "session1/job_events/", []string{})
|
|
mock.addDir("cluster_ns", "session1/logs", []string{})
|
|
|
|
h := NewEventHandler(mock)
|
|
err = h.ProcessSingleSession(context.Background(), clusterInfo)
|
|
require.NoError(t, err)
|
|
|
|
nodeMap := h.getNodeMap("cluster_ns_session1")
|
|
assert.Len(t, nodeMap, 2)
|
|
|
|
// "YWJjZA==" -> hex "61626364" (abcd)
|
|
_, ok := nodeMap["61626364"]
|
|
assert.True(t, ok, "node1 (compressed) should be successfully loaded")
|
|
|
|
// "ZWZnaA==" -> hex "65666768" (efgh)
|
|
_, ok = nodeMap["65666768"]
|
|
assert.True(t, ok, "node2 (uncompressed) should be successfully loaded")
|
|
})
|
|
}
|