Files
fscnickandGitHub 79b5c304b3 background goroutine get job info (#4160)
* [RayJob] background job info poc

* [RayJob] add implement some methods

* [RayJob] encapsulate the worker pool

* [RayJob] replace concurrency map with lru cache

* [RayJob] remove cache on stop and config flag

* [RayJob] expiry cache cleanup goroutine

Signed-off-by: fscnick <[email protected]>

* [RayJob] code and comment minor fix

Signed-off-by: fscnick <[email protected]>

* [RayJob] task check contain or not befor add

Signed-off-by: fscnick <[email protected]>

* [RayJob] remove delete cache from deleteClusterResources and add lock for cache

Signed-off-by: fscnick <[email protected]>

* [Helm] add argument for useBackgroundGoroutine

Signed-off-by: fscnick <[email protected]>

* [RayJob] repeated error did not update

Signed-off-by: fscnick <[email protected]>

* [RayJob] remove unused function and background goroutine observability

Signed-off-by: fscnick <[email protected]>

* [RayJob] cache client support graceful shutdown

Signed-off-by: fscnick <[email protected]>

* [RayJob] rename useBackgroundGoroutine to asyncJobInfoQuery

Signed-off-by: fscnick <[email protected]>

* [RayJob] use ray job info in logger

Signed-off-by: fscnick <[email protected]>

* [RayJob] remove cacheStorage nil check

Signed-off-by: fscnick <[email protected]>

* [RayJob] bg goroutine uses operator context instead

Signed-off-by: fscnick <[email protected]>

* [RayJob] bg goroutine handle task queue full

Signed-off-by: fscnick <[email protected]>

* [RayJob] correct the comment

Signed-off-by: fscnick <[email protected]>

* [RayJob] refactor initialize dashboard client for background goroutine

Signed-off-by: fscnick <[email protected]>

* [RayJob] worker handle ctx.Done correctly

Signed-off-by: fscnick <[email protected]>

* [RayJob] remove unnecessary putting task into queue

* [RayJob] if queue is full, retry again

* [RayJob] make cache immutable to avoid data race

Signed-off-by: fscnick <[email protected]>

* [RayJob] remove unused function

Signed-off-by: fscnick <[email protected]>

* [RayJob] remove cacheStorage lock

Signed-off-by: fscnick <[email protected]>

* [RayJob] update cache error

Signed-off-by: fscnick <[email protected]>

* [RayJob] If error on fetching job info, it removes from loop

Signed-off-by: fscnick <[email protected]>

* [RayJob] task queue is extendable

Signed-off-by: fscnick <[email protected]>

* [RayJob] change slice to ring buffer

Signed-off-by: fscnick <[email protected]>

* [RayJob] rename PutTask to AddTask

* [RayJob] extendable channel use open source library

Signed-off-by: fscnick <[email protected]>

* [RayJob] async job info query use feature gate instead

Signed-off-by: fscnick <[email protected]>

* [RayJob] add comment for task

Signed-off-by: fscnick <[email protected]>

* [RayJob] rename function signature of worker pool init function

Signed-off-by: fscnick <[email protected]>

* [RayJob] change ErrAgain error message

Signed-off-by: fscnick <[email protected]>

* [RayJob] fix lint error

Signed-off-by: fscnick <[email protected]>

* [RayJob] change back to EAGAIN

Signed-off-by: fscnick <[email protected]>

* [RayJob] remove queue size from todo comment

Signed-off-by: fscnick <[email protected]>

* [RayJob] rename queue full error

Signed-off-by: fscnick <[email protected]>

* [RayJob] add lock to avoid data race

Signed-off-by: fscnick <[email protected]>

* [RayJob] requeue check context has canceled or not

Signed-off-by: fscnick <[email protected]>

* [RayJob] add cluster name on the cache key

Signed-off-by: fscnick <[email protected]>

* [RayJob] check raycluster is nil or not when initializing the dashboard client

Signed-off-by: fscnick <[email protected]>

* [RayJob] avoid send to a block channel when graceful shutdown

Signed-off-by: fscnick <[email protected]>

* [RayJob] use contain to check the placeholder at the beginning of task

Signed-off-by: fscnick <[email protected]>

* [RayJob] graceful shutdown avoid panic from a nil task

* [RayJob] fix channel receive condition

Signed-off-by: fscnick <[email protected]>

* [RayJob] fix nil rayCluster in dashboard cache client

Signed-off-by: fscnick <[email protected]>

* [RayJob] remove with name from log for sharing purpose

Signed-off-by: fscnick <[email protected]>

* [RayJob] remove checkname to avoid collision

Signed-off-by: fscnick <[email protected]>

* [RayJob] add task with blocking send

Signed-off-by: fscnick <[email protected]>

* [RayJob] remove unused error

Signed-off-by: fscnick <[email protected]>

* [RayJob] provide raycluster name if it is absent for removing cache

Signed-off-by: fscnick <[email protected]>

---------

Signed-off-by: fscnick <[email protected]>
2026-01-11 21:08:46 +08:00

88 lines
2.4 KiB
Go

package sampleyaml
import (
"fmt"
"maps"
"os"
"path/filepath"
"runtime"
. "github.com/onsi/gomega"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
. "github.com/ray-project/kuberay/ray-operator/test/support"
)
func GetSampleYAMLDir(t Test) string {
t.T().Helper()
_, b, _, _ := runtime.Caller(0)
sampleYAMLDir := filepath.Join(filepath.Dir(b), "../../config/samples")
info, err := os.Stat(sampleYAMLDir)
require.NoError(t.T(), err)
assert.True(t.T(), info.IsDir())
return sampleYAMLDir
}
func SubmitJobsToAllPods(t Test, rayCluster *rayv1.RayCluster) func(Gomega) {
return func(g Gomega) {
pods, err := GetAllPods(t, rayCluster)
g.Expect(err).NotTo(HaveOccurred())
cmd := []string{
"python",
"-c",
"import ray; ray.init(); print(ray.cluster_resources())",
}
for _, pod := range pods {
container := pod.Spec.Containers[utils.RayContainerIndex] // Directly access the Ray container
ExecPodCmd(t, &pod, container.Name, cmd)
}
}
}
func getApps(rayService *rayv1.RayService) map[string]rayv1.AppStatus {
apps := make(map[string]rayv1.AppStatus)
maps.Copy(apps, rayService.Status.ActiveServiceStatus.Applications)
return apps
}
func AllAppsRunning(rayService *rayv1.RayService) bool {
appStatuses := getApps(rayService)
if len(appStatuses) == 0 {
return false
}
for _, appStatus := range appStatuses {
if appStatus.Status != rayv1.ApplicationStatusEnum.RUNNING {
return false
}
}
return true
}
func QueryDashboardGetAppStatus(t Test, rayCluster *rayv1.RayCluster) func(Gomega) {
return func(g Gomega) {
pod, err := GetHeadPod(t, rayCluster)
g.Expect(err).ToNot(HaveOccurred())
localPort := 8265
remotePort := 8265
stopChan, err := SetupPortForward(t, pod.Name, pod.Namespace, localPort, remotePort)
defer close(stopChan)
g.Expect(err).ToNot(HaveOccurred())
url := fmt.Sprintf("127.0.0.1:%d", localPort)
rayDashboardClientFunc := utils.GetRayDashboardClientFunc(t.Ctx(), nil, false)
rayDashboardClient, err := rayDashboardClientFunc(rayCluster, url)
g.Expect(err).ToNot(HaveOccurred())
serveDetails, err := rayDashboardClient.GetServeDetails(t.Ctx())
g.Expect(err).ToNot(HaveOccurred())
for _, value := range serveDetails.Applications {
g.Expect(value.ServeApplicationStatus.Status).To(Equal(rayv1.ApplicationStatusEnum.RUNNING))
}
}
}