mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
* [RayJob] background job info poc * [RayJob] add implement some methods * [RayJob] encapsulate the worker pool * [RayJob] replace concurrency map with lru cache * [RayJob] remove cache on stop and config flag * [RayJob] expiry cache cleanup goroutine Signed-off-by: fscnick <[email protected]> * [RayJob] code and comment minor fix Signed-off-by: fscnick <[email protected]> * [RayJob] task check contain or not befor add Signed-off-by: fscnick <[email protected]> * [RayJob] remove delete cache from deleteClusterResources and add lock for cache Signed-off-by: fscnick <[email protected]> * [Helm] add argument for useBackgroundGoroutine Signed-off-by: fscnick <[email protected]> * [RayJob] repeated error did not update Signed-off-by: fscnick <[email protected]> * [RayJob] remove unused function and background goroutine observability Signed-off-by: fscnick <[email protected]> * [RayJob] cache client support graceful shutdown Signed-off-by: fscnick <[email protected]> * [RayJob] rename useBackgroundGoroutine to asyncJobInfoQuery Signed-off-by: fscnick <[email protected]> * [RayJob] use ray job info in logger Signed-off-by: fscnick <[email protected]> * [RayJob] remove cacheStorage nil check Signed-off-by: fscnick <[email protected]> * [RayJob] bg goroutine uses operator context instead Signed-off-by: fscnick <[email protected]> * [RayJob] bg goroutine handle task queue full Signed-off-by: fscnick <[email protected]> * [RayJob] correct the comment Signed-off-by: fscnick <[email protected]> * [RayJob] refactor initialize dashboard client for background goroutine Signed-off-by: fscnick <[email protected]> * [RayJob] worker handle ctx.Done correctly Signed-off-by: fscnick <[email protected]> * [RayJob] remove unnecessary putting task into queue * [RayJob] if queue is full, retry again * [RayJob] make cache immutable to avoid data race Signed-off-by: fscnick <[email protected]> * [RayJob] remove unused function Signed-off-by: fscnick <[email protected]> * [RayJob] remove cacheStorage lock Signed-off-by: fscnick <[email protected]> * [RayJob] update cache error Signed-off-by: fscnick <[email protected]> * [RayJob] If error on fetching job info, it removes from loop Signed-off-by: fscnick <[email protected]> * [RayJob] task queue is extendable Signed-off-by: fscnick <[email protected]> * [RayJob] change slice to ring buffer Signed-off-by: fscnick <[email protected]> * [RayJob] rename PutTask to AddTask * [RayJob] extendable channel use open source library Signed-off-by: fscnick <[email protected]> * [RayJob] async job info query use feature gate instead Signed-off-by: fscnick <[email protected]> * [RayJob] add comment for task Signed-off-by: fscnick <[email protected]> * [RayJob] rename function signature of worker pool init function Signed-off-by: fscnick <[email protected]> * [RayJob] change ErrAgain error message Signed-off-by: fscnick <[email protected]> * [RayJob] fix lint error Signed-off-by: fscnick <[email protected]> * [RayJob] change back to EAGAIN Signed-off-by: fscnick <[email protected]> * [RayJob] remove queue size from todo comment Signed-off-by: fscnick <[email protected]> * [RayJob] rename queue full error Signed-off-by: fscnick <[email protected]> * [RayJob] add lock to avoid data race Signed-off-by: fscnick <[email protected]> * [RayJob] requeue check context has canceled or not Signed-off-by: fscnick <[email protected]> * [RayJob] add cluster name on the cache key Signed-off-by: fscnick <[email protected]> * [RayJob] check raycluster is nil or not when initializing the dashboard client Signed-off-by: fscnick <[email protected]> * [RayJob] avoid send to a block channel when graceful shutdown Signed-off-by: fscnick <[email protected]> * [RayJob] use contain to check the placeholder at the beginning of task Signed-off-by: fscnick <[email protected]> * [RayJob] graceful shutdown avoid panic from a nil task * [RayJob] fix channel receive condition Signed-off-by: fscnick <[email protected]> * [RayJob] fix nil rayCluster in dashboard cache client Signed-off-by: fscnick <[email protected]> * [RayJob] remove with name from log for sharing purpose Signed-off-by: fscnick <[email protected]> * [RayJob] remove checkname to avoid collision Signed-off-by: fscnick <[email protected]> * [RayJob] add task with blocking send Signed-off-by: fscnick <[email protected]> * [RayJob] remove unused error Signed-off-by: fscnick <[email protected]> * [RayJob] provide raycluster name if it is absent for removing cache Signed-off-by: fscnick <[email protected]> --------- Signed-off-by: fscnick <[email protected]>
88 lines
2.4 KiB
Go
88 lines
2.4 KiB
Go
package sampleyaml
|
|
|
|
import (
|
|
"fmt"
|
|
"maps"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
|
|
. "github.com/onsi/gomega"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
|
|
. "github.com/ray-project/kuberay/ray-operator/test/support"
|
|
)
|
|
|
|
func GetSampleYAMLDir(t Test) string {
|
|
t.T().Helper()
|
|
_, b, _, _ := runtime.Caller(0)
|
|
sampleYAMLDir := filepath.Join(filepath.Dir(b), "../../config/samples")
|
|
info, err := os.Stat(sampleYAMLDir)
|
|
require.NoError(t.T(), err)
|
|
assert.True(t.T(), info.IsDir())
|
|
return sampleYAMLDir
|
|
}
|
|
|
|
func SubmitJobsToAllPods(t Test, rayCluster *rayv1.RayCluster) func(Gomega) {
|
|
return func(g Gomega) {
|
|
pods, err := GetAllPods(t, rayCluster)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
cmd := []string{
|
|
"python",
|
|
"-c",
|
|
"import ray; ray.init(); print(ray.cluster_resources())",
|
|
}
|
|
for _, pod := range pods {
|
|
container := pod.Spec.Containers[utils.RayContainerIndex] // Directly access the Ray container
|
|
ExecPodCmd(t, &pod, container.Name, cmd)
|
|
}
|
|
}
|
|
}
|
|
|
|
func getApps(rayService *rayv1.RayService) map[string]rayv1.AppStatus {
|
|
apps := make(map[string]rayv1.AppStatus)
|
|
maps.Copy(apps, rayService.Status.ActiveServiceStatus.Applications)
|
|
return apps
|
|
}
|
|
|
|
func AllAppsRunning(rayService *rayv1.RayService) bool {
|
|
appStatuses := getApps(rayService)
|
|
if len(appStatuses) == 0 {
|
|
return false
|
|
}
|
|
|
|
for _, appStatus := range appStatuses {
|
|
if appStatus.Status != rayv1.ApplicationStatusEnum.RUNNING {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func QueryDashboardGetAppStatus(t Test, rayCluster *rayv1.RayCluster) func(Gomega) {
|
|
return func(g Gomega) {
|
|
pod, err := GetHeadPod(t, rayCluster)
|
|
g.Expect(err).ToNot(HaveOccurred())
|
|
|
|
localPort := 8265
|
|
remotePort := 8265
|
|
stopChan, err := SetupPortForward(t, pod.Name, pod.Namespace, localPort, remotePort)
|
|
defer close(stopChan)
|
|
|
|
g.Expect(err).ToNot(HaveOccurred())
|
|
url := fmt.Sprintf("127.0.0.1:%d", localPort)
|
|
rayDashboardClientFunc := utils.GetRayDashboardClientFunc(t.Ctx(), nil, false)
|
|
rayDashboardClient, err := rayDashboardClientFunc(rayCluster, url)
|
|
g.Expect(err).ToNot(HaveOccurred())
|
|
serveDetails, err := rayDashboardClient.GetServeDetails(t.Ctx())
|
|
g.Expect(err).ToNot(HaveOccurred())
|
|
|
|
for _, value := range serveDetails.Applications {
|
|
g.Expect(value.ServeApplicationStatus.Status).To(Equal(rayv1.ApplicationStatusEnum.RUNNING))
|
|
}
|
|
}
|
|
}
|