mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
* [Fix][1/N] defer sidecar submitter failure until dashboard check Signed-off-by: justinyeh1995 <[email protected]> * [Feature][SidecarMode] Add SidecarSubmitterRestart feature gate for per-container restart Signed-off-by: justinyeh1995 <[email protected]> * [Feature][SidecarMode] Add SidecarSubmitterRestart feature gate unit test Signed-off-by: justinyeh1995 <[email protected]> * [Feature][SidecarMode] Add K8s version check if SidecarSubmitterRestart feature gate is enabled at operator startup Signed-off-by: justinyeh1995 <[email protected]> * [Chore] better explainations Signed-off-by: justinyeh1995 <[email protected]> * [Chore] remove incorrect nil check Signed-off-by: justinyeh1995 <[email protected]> * [Fix] SidecarMode should not fail job based on container exit status Signed-off-by: justinyeh1995 <[email protected]> * Trigger CI Signed-off-by: justinyeh1995 <[email protected]> * [Fix][SidecarMode] add e2e tests Signed-off-by: justinyeh1995 <[email protected]> * [Fix][SidecarMode] clarify RestartPolicyRules exit code rationale Signed-off-by: justinyeh1995 <[email protected]> * [Fix][Sidecar Mode] Use ParseGeneric + GitVersion, and add patch version for more robust k8s version comparison Signed-off-by: justinyeh1995 <[email protected]> * Apply suggestions Co-authored-by: Jia-Wei Jiang <[email protected]> Signed-off-by: JustinYeh <[email protected]> * [Fix] use ContainerRestartPolicyOnFailure to express the intention better; update the comment to explain the intention better Signed-off-by: justinyeh1995 <[email protected]> * [Fix] bump version check to v1.35 and update sidecar mode test to check onFailure restart policy Signed-off-by: justinyeh1995 <[email protected]> * [Fix] check the existence of the feature gate instead of relying on the derivate result Signed-off-by: justinyeh1995 <[email protected]> * improve the why for using K8s v1.35 Signed-off-by: justinyeh1995 <[email protected]> * [Fix] Revert checkSidecarContainerStatus removal; scope PR to K8s v1.35+ use case Signed-off-by: justinyeh1995 <[email protected]> * [Fix] use correct health command constant and argument order in the feature gate test. Signed-off-by: justinyeh1995 <[email protected]> * [Fix] remove leftover changes Signed-off-by: justinyeh1995 <[email protected]> * [Fix] prevent panicing due to failing to fetch head pod Signed-off-by: justinyeh1995 <[email protected]> * add warning about the risk of having kubelet version skew Signed-off-by: justinyeh1995 <[email protected]> * document ray v2.54.0+ is needed Co-authored-by: Jun-Hao Wan <[email protected]> Signed-off-by: JustinYeh <[email protected]> * add max restart count annotaion, envVar, and default max restart count Signed-off-by: justinyeh1995 <[email protected]> * do not fail the job if the submitter container exited successfully Signed-off-by: justinyeh1995 <[email protected]> * fix BasePythonHealthCommand argument Signed-off-by: justinyeh1995 <[email protected]> * trigger ci Signed-off-by: justinyeh1995 <[email protected]> * address suggestion part 1 Signed-off-by: justinyeh1995 <[email protected]> * k8s 1.35 version Signed-off-by: Future-Outlier <[email protected]> * add override Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * upd for helm and kustomization Signed-off-by: Future-Outlier <[email protected]> * update Signed-off-by: Future-Outlier <[email protected]> * fix typo Signed-off-by: justinyeh1995 <[email protected]> * address comment, assert no resubmission Signed-off-by: justinyeh1995 <[email protected]> * bump kind version to the latest version which supports v1.35.0 by default Signed-off-by: justinyeh1995 <[email protected]> * incremental upgrade use kind-config-buildkite.yml (v1.35) Signed-off-by: justinyeh1995 <[email protected]> * replace hardcoded value to use rayJob.Spec.SubmitterConfig.BackoffLimit Signed-off-by: justinyeh1995 <[email protected]> --------- Signed-off-by: justinyeh1995 <[email protected]> Signed-off-by: JustinYeh <[email protected]> Signed-off-by: Future-Outlier <[email protected]> Co-authored-by: Jia-Wei Jiang <[email protected]> Co-authored-by: Jun-Hao Wan <[email protected]> Co-authored-by: Future-Outlier <[email protected]>
480 lines
16 KiB
Go
480 lines
16 KiB
Go
package common
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strconv"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
corev1 "k8s.io/api/core/v1"
|
|
|
|
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
|
|
"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
|
|
"github.com/ray-project/kuberay/ray-operator/pkg/features"
|
|
)
|
|
|
|
func rayJobTemplate() *rayv1.RayJob {
|
|
return &rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
RuntimeEnvYAML: "test: test",
|
|
Metadata: map[string]string{
|
|
"testKey": "testValue",
|
|
},
|
|
RayClusterSpec: &rayv1.RayClusterSpec{
|
|
RayVersion: "2.6.0",
|
|
},
|
|
Entrypoint: "echo no quote 'single quote' \"double quote\"",
|
|
EntrypointNumCpus: 1,
|
|
EntrypointNumGpus: 0.5,
|
|
EntrypointResources: `{"Custom_1": 1, "Custom_2": 5.5}`,
|
|
},
|
|
Status: rayv1.RayJobStatus{
|
|
DashboardURL: "http://127.0.0.1:8265",
|
|
JobId: "testJobId",
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestGetRuntimeEnvJsonFromBase64(t *testing.T) {
|
|
testRayJob := rayJobTemplate()
|
|
expected := `{"test":"test"}`
|
|
jsonOutput, err := getRuntimeEnvJson(testRayJob)
|
|
require.NoError(t, err)
|
|
assert.JSONEq(t, expected, jsonOutput)
|
|
}
|
|
|
|
func TestGetRuntimeEnvJsonFromYAML(t *testing.T) {
|
|
rayJobWithYAML := &rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
RuntimeEnvYAML: `
|
|
working_dir: "https://github.com/ray-project/serve_config_examples/archive/b393e77bbd6aba0881e3d94c05f968f05a387b96.zip"
|
|
pip: ["python-multipart==0.0.6"]
|
|
`,
|
|
},
|
|
}
|
|
expectedJSON := `{"working_dir":"https://github.com/ray-project/serve_config_examples/archive/b393e77bbd6aba0881e3d94c05f968f05a387b96.zip","pip":["python-multipart==0.0.6"]}`
|
|
jsonOutput, err := getRuntimeEnvJson(rayJobWithYAML)
|
|
require.NoError(t, err)
|
|
|
|
var expectedMap map[string]any
|
|
var actualMap map[string]any
|
|
|
|
// Convert the JSON strings into map types to avoid errors due to ordering
|
|
require.NoError(t, json.Unmarshal([]byte(expectedJSON), &expectedMap))
|
|
require.NoError(t, json.Unmarshal([]byte(jsonOutput), &actualMap))
|
|
|
|
// Now compare the maps
|
|
assert.Equal(t, expectedMap, actualMap)
|
|
}
|
|
|
|
func TestBuildJobSubmitCommandWithK8sJobMode(t *testing.T) {
|
|
testRayJob := rayJobTemplate()
|
|
expected := []string{
|
|
"until",
|
|
fmt.Sprintf(utils.BasePythonHealthCommand, "http://127.0.0.1:8265/"+utils.RayDashboardGCSHealthPath, utils.RayDashboardGCSHealthCheckTimeoutSeconds),
|
|
">/dev/null", "2>&1", ";",
|
|
"do", "echo", strconv.Quote("Waiting for Ray Dashboard GCS to become healthy at http://127.0.0.1:8265 ..."), ";", "sleep", "2", ";", "done", ";",
|
|
"if",
|
|
"!", "ray", "job", "status", "--address", "http://127.0.0.1:8265", "testJobId", ">/dev/null", "2>&1",
|
|
";", "then",
|
|
"ray", "job", "submit", "--address", "http://127.0.0.1:8265", "--no-wait",
|
|
"--runtime-env-json", strconv.Quote(`{"test":"test"}`),
|
|
"--metadata-json", strconv.Quote(`{"testKey":"testValue"}`),
|
|
"--submission-id", "testJobId",
|
|
"--entrypoint-num-cpus", "1.000000",
|
|
"--entrypoint-num-gpus", "0.500000",
|
|
"--entrypoint-resources", strconv.Quote(`{"Custom_1": 1, "Custom_2": 5.5}`),
|
|
"--",
|
|
"echo no quote 'single quote' \"double quote\"",
|
|
";", "fi", ";",
|
|
"ray", "job", "logs", "--address", "http://127.0.0.1:8265", "--follow", "testJobId",
|
|
}
|
|
command, err := BuildJobSubmitCommand(testRayJob, rayv1.K8sJobMode)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, expected, command)
|
|
}
|
|
|
|
func TestBuildJobSubmitCommandWithSidecarMode(t *testing.T) {
|
|
testRayJob := rayJobTemplate()
|
|
testRayJob.Spec.RayClusterSpec.HeadGroupSpec.Template.Spec.Containers = []corev1.Container{
|
|
{
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
Name: utils.DashboardPortName,
|
|
ContainerPort: utils.DefaultDashboardPort,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
expected := []string{
|
|
"until",
|
|
fmt.Sprintf(
|
|
utils.BasePythonHealthCommand,
|
|
fmt.Sprintf("http://localhost:%d/%s", utils.DefaultDashboardPort, utils.RayDashboardGCSHealthPath),
|
|
utils.RayDashboardGCSHealthCheckTimeoutSeconds,
|
|
),
|
|
">/dev/null", "2>&1", ";",
|
|
"do", "echo", strconv.Quote("Waiting for Ray Dashboard GCS to become healthy at http://127.0.0.1:8265 ..."), ";", "sleep", "2", ";", "done", ";",
|
|
"ray", "job", "submit", "--address", "http://127.0.0.1:8265",
|
|
"--runtime-env-json", strconv.Quote(`{"test":"test"}`),
|
|
"--metadata-json", strconv.Quote(`{"testKey":"testValue"}`),
|
|
"--submission-id", "testJobId",
|
|
"--entrypoint-num-cpus", "1.000000",
|
|
"--entrypoint-num-gpus", "0.500000",
|
|
"--entrypoint-resources", strconv.Quote(`{"Custom_1": 1, "Custom_2": 5.5}`),
|
|
"--",
|
|
"echo no quote 'single quote' \"double quote\"",
|
|
";",
|
|
}
|
|
command, err := BuildJobSubmitCommand(testRayJob, rayv1.SidecarMode)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, expected, command)
|
|
}
|
|
|
|
func TestBuildJobSubmitCommandWithSidecarModeVersionSwitch(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
rayVersion string
|
|
}{
|
|
{
|
|
name: "uses python health command for ray >= 2.53",
|
|
rayVersion: "2.53.0",
|
|
},
|
|
{
|
|
name: "uses python health command for ray < 2.53",
|
|
rayVersion: "2.52.1",
|
|
},
|
|
{
|
|
name: "uses python health command when rayVersion is invalid",
|
|
rayVersion: "invalid-version",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
testRayJob := rayJobTemplate()
|
|
testRayJob.Spec.RayClusterSpec.RayVersion = tt.rayVersion
|
|
// Avoid metadata-json version parsing failure; this test only checks health command selection.
|
|
testRayJob.Spec.Metadata = nil
|
|
testRayJob.Spec.RayClusterSpec.HeadGroupSpec.Template.Spec.Containers = []corev1.Container{
|
|
{
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
Name: utils.DashboardPortName,
|
|
ContainerPort: utils.DefaultDashboardPort,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
command, err := BuildJobSubmitCommand(testRayJob, rayv1.SidecarMode)
|
|
require.NoError(t, err)
|
|
require.GreaterOrEqual(t, len(command), 2)
|
|
assert.Equal(t, "until", command[0])
|
|
assert.Contains(t, command[1], "python -c")
|
|
assert.Contains(t, command[1], utils.RayDashboardGCSHealthPath)
|
|
assert.NotContains(t, command[1], "wget")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBuildJobSubmitCommandWithSidecarModeCustomDashboardPort(t *testing.T) {
|
|
testRayJob := rayJobTemplate()
|
|
const customPort = 9000
|
|
testRayJob.Spec.RayClusterSpec.HeadGroupSpec.Template.Spec.Containers = []corev1.Container{
|
|
{
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
Name: utils.DashboardPortName,
|
|
ContainerPort: customPort,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
command, err := BuildJobSubmitCommand(testRayJob, rayv1.SidecarMode)
|
|
require.NoError(t, err)
|
|
require.GreaterOrEqual(t, len(command), 2)
|
|
assert.Equal(t, "until", command[0])
|
|
assert.Contains(t, command[1], fmt.Sprintf("localhost:%d/%s", customPort, utils.RayDashboardGCSHealthPath))
|
|
assert.Contains(t, command[1], "python -c")
|
|
assert.NotContains(t, command[1], "wget")
|
|
}
|
|
|
|
func TestBuildJobSubmitCommandWithK8sJobModeHealthWaitLoop(t *testing.T) {
|
|
testRayJob := rayJobTemplate()
|
|
command, err := BuildJobSubmitCommand(testRayJob, rayv1.K8sJobMode)
|
|
require.NoError(t, err)
|
|
require.GreaterOrEqual(t, len(command), 2)
|
|
assert.Equal(t, "until", command[0])
|
|
assert.Contains(t, command[1], "python -c")
|
|
assert.Contains(t, command[1], utils.RayDashboardGCSHealthPath)
|
|
assert.Contains(t, command[1], "127.0.0.1:8265")
|
|
assert.NotContains(t, command[1], "wget")
|
|
}
|
|
|
|
func TestBuildJobSubmitCommandWithSidecarModeAndFeatureGate(t *testing.T) {
|
|
// Enable the SidecarSubmitterRestart feature gate for this test
|
|
features.SetFeatureGateDuringTest(t, features.SidecarSubmitterRestart, true)
|
|
|
|
testRayJob := rayJobTemplate()
|
|
testRayJob.Spec.RayClusterSpec.HeadGroupSpec.Template.Spec.Containers = []corev1.Container{
|
|
{
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
Name: utils.DashboardPortName,
|
|
ContainerPort: utils.DefaultDashboardPort,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
// With SidecarSubmitterRestart enabled, the command should include:
|
|
// - status check (if ! ray job status ...)
|
|
// - --no-wait flag
|
|
// - job logs follow at the end
|
|
healthURL := fmt.Sprintf("http://localhost:%d/%s", utils.DefaultDashboardPort, utils.RayDashboardGCSHealthPath)
|
|
expected := []string{
|
|
"until",
|
|
fmt.Sprintf(
|
|
utils.BasePythonHealthCommand,
|
|
healthURL,
|
|
utils.RayDashboardGCSHealthCheckTimeoutSeconds,
|
|
),
|
|
">/dev/null", "2>&1", ";",
|
|
"do", "echo", strconv.Quote("Waiting for Ray Dashboard GCS to become healthy at http://127.0.0.1:8265 ..."), ";", "sleep", "2", ";", "done", ";",
|
|
"if", "!", "ray", "job", "status", "--address", "http://127.0.0.1:8265", "testJobId", ">/dev/null", "2>&1", ";", "then",
|
|
"ray", "job", "submit", "--address", "http://127.0.0.1:8265", "--no-wait",
|
|
"--runtime-env-json", strconv.Quote(`{"test":"test"}`),
|
|
"--metadata-json", strconv.Quote(`{"testKey":"testValue"}`),
|
|
"--submission-id", "testJobId",
|
|
"--entrypoint-num-cpus", "1.000000",
|
|
"--entrypoint-num-gpus", "0.500000",
|
|
"--entrypoint-resources", strconv.Quote(`{"Custom_1": 1, "Custom_2": 5.5}`),
|
|
"--",
|
|
"echo no quote 'single quote' \"double quote\"",
|
|
";", "fi", ";",
|
|
"ray", "job", "logs", "--address", "http://127.0.0.1:8265", "--follow", "testJobId",
|
|
}
|
|
command, err := BuildJobSubmitCommand(testRayJob, rayv1.SidecarMode)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, expected, command)
|
|
}
|
|
|
|
func TestBuildJobSubmitCommandWithK8sJobModeAndYAML(t *testing.T) {
|
|
rayJobWithYAML := &rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
RuntimeEnvYAML: `
|
|
working_dir: "https://github.com/ray-project/serve_config_examples/archive/b393e77bbd6aba0881e3d94c05f968f05a387b96.zip"
|
|
pip: ["python-multipart==0.0.6"]
|
|
`,
|
|
Metadata: map[string]string{
|
|
"testKey": "testValue",
|
|
},
|
|
RayClusterSpec: &rayv1.RayClusterSpec{
|
|
RayVersion: "2.6.0",
|
|
},
|
|
Entrypoint: "echo no quote 'single quote' \"double quote\"",
|
|
},
|
|
Status: rayv1.RayJobStatus{
|
|
DashboardURL: "http://127.0.0.1:8265",
|
|
JobId: "testJobId",
|
|
},
|
|
}
|
|
expected := []string{
|
|
"until",
|
|
fmt.Sprintf(utils.BasePythonHealthCommand, "http://127.0.0.1:8265/"+utils.RayDashboardGCSHealthPath, utils.RayDashboardGCSHealthCheckTimeoutSeconds),
|
|
">/dev/null", "2>&1", ";",
|
|
"do", "echo", strconv.Quote("Waiting for Ray Dashboard GCS to become healthy at http://127.0.0.1:8265 ..."), ";", "sleep", "2", ";", "done", ";",
|
|
"if",
|
|
"!", "ray", "job", "status", "--address", "http://127.0.0.1:8265", "testJobId", ">/dev/null", "2>&1",
|
|
";", "then",
|
|
"ray", "job", "submit", "--address", "http://127.0.0.1:8265", "--no-wait",
|
|
"--runtime-env-json", strconv.Quote(`{"working_dir":"https://github.com/ray-project/serve_config_examples/archive/b393e77bbd6aba0881e3d94c05f968f05a387b96.zip","pip":["python-multipart==0.0.6"]}`),
|
|
"--metadata-json", strconv.Quote(`{"testKey":"testValue"}`),
|
|
"--submission-id", "testJobId",
|
|
"--",
|
|
"echo no quote 'single quote' \"double quote\"",
|
|
";", "fi", ";",
|
|
"ray", "job", "logs", "--address", "http://127.0.0.1:8265", "--follow", "testJobId",
|
|
}
|
|
command, err := BuildJobSubmitCommand(rayJobWithYAML, rayv1.K8sJobMode)
|
|
require.NoError(t, err)
|
|
|
|
// Ensure the slices are the same length.
|
|
assert.Len(t, command, len(expected))
|
|
|
|
for i := 0; i < len(expected); i++ {
|
|
// For non-JSON elements, compare them directly.
|
|
assert.Equal(t, expected[i], command[i])
|
|
if expected[i] == "--runtime-env-json" {
|
|
// Decode the JSON string from the next element.
|
|
var expectedMap, actualMap map[string]any
|
|
//nolint:gosec // G602: test invariant guarantees "--runtime-env-json" is followed by a value.
|
|
unquoteExpected, err1 := strconv.Unquote(expected[i+1])
|
|
require.NoError(t, err1)
|
|
|
|
unquotedCommand, err2 := strconv.Unquote(command[i+1])
|
|
require.NoError(t, err2)
|
|
err1 = json.Unmarshal([]byte(unquoteExpected), &expectedMap)
|
|
err2 = json.Unmarshal([]byte(unquotedCommand), &actualMap)
|
|
|
|
// If there's an error decoding either JSON string, it's an error in the test.
|
|
require.NoError(t, err1)
|
|
require.NoError(t, err2)
|
|
|
|
// Compare the maps directly to avoid errors due to ordering.
|
|
assert.Equal(t, expectedMap, actualMap)
|
|
|
|
// Skip the next element because we've just checked it.
|
|
i++
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestBuildJobSubmitCommandWithClusterSelector verifies that metadata is included
|
|
// when submitting a RayJob to an existing cluster via clusterSelector (no RayClusterSpec,
|
|
// so we can't know the Ray version without looking up the RayCluster; we assume >= 2.6).
|
|
func TestBuildJobSubmitCommandWithClusterSelector(t *testing.T) {
|
|
rayJobWithClusterSelector := &rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
ClusterSelector: map[string]string{
|
|
"ray.io/cluster": "existing-cluster",
|
|
},
|
|
Metadata: map[string]string{
|
|
"tenant": "tenant1",
|
|
"team": "ml-platform",
|
|
},
|
|
Entrypoint: "python /app/batch_inference.py",
|
|
},
|
|
Status: rayv1.RayJobStatus{
|
|
DashboardURL: "http://existing-cluster-head-svc:8265",
|
|
JobId: "cluster-selector-job-id",
|
|
},
|
|
}
|
|
|
|
command, err := BuildJobSubmitCommand(rayJobWithClusterSelector, rayv1.K8sJobMode)
|
|
require.NoError(t, err)
|
|
|
|
hasMetadataFlag := false
|
|
for i, arg := range command {
|
|
if arg == "--metadata-json" {
|
|
hasMetadataFlag = true
|
|
require.Greater(t, len(command), i+1)
|
|
unquoted, err := strconv.Unquote(command[i+1])
|
|
require.NoError(t, err)
|
|
var metadata map[string]string
|
|
require.NoError(t, json.Unmarshal([]byte(unquoted), &metadata))
|
|
assert.Equal(t, "tenant1", metadata["tenant"])
|
|
assert.Equal(t, "ml-platform", metadata["team"])
|
|
break
|
|
}
|
|
}
|
|
assert.True(t, hasMetadataFlag, "metadata-json flag should be present when using clusterSelector with metadata")
|
|
}
|
|
|
|
// TestBuildJobSubmitCommandWithUnparseableRayVersion verifies that metadata is
|
|
// rejected when RayJob.Spec.RayClusterSpec.RayVersion is set to a non-semver string. A user
|
|
// who went to the trouble of typing a version should fail fast rather than silently proceed.
|
|
func TestBuildJobSubmitCommandWithUnparseableRayVersion(t *testing.T) {
|
|
rayJob := &rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
RayClusterSpec: &rayv1.RayClusterSpec{
|
|
RayVersion: "not-a-version",
|
|
},
|
|
Metadata: map[string]string{
|
|
"testKey": "testValue",
|
|
},
|
|
Entrypoint: "echo hello",
|
|
},
|
|
Status: rayv1.RayJobStatus{
|
|
DashboardURL: "http://127.0.0.1:8265",
|
|
JobId: "testJobId",
|
|
},
|
|
}
|
|
_, err := BuildJobSubmitCommand(rayJob, rayv1.K8sJobMode)
|
|
require.Error(t, err)
|
|
}
|
|
|
|
// TestBuildJobSubmitCommandWithOldRayVersion verifies that metadata is
|
|
// rejected when RayJob.Spec.RayClusterSpec.RayVersion is explicitly set below 2.6.0.
|
|
func TestBuildJobSubmitCommandWithOldRayVersion(t *testing.T) {
|
|
rayJob := &rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
RayClusterSpec: &rayv1.RayClusterSpec{
|
|
RayVersion: "2.5.0",
|
|
},
|
|
Metadata: map[string]string{
|
|
"testKey": "testValue",
|
|
},
|
|
Entrypoint: "echo hello",
|
|
},
|
|
Status: rayv1.RayJobStatus{
|
|
DashboardURL: "http://127.0.0.1:8265",
|
|
JobId: "testJobId",
|
|
},
|
|
}
|
|
_, err := BuildJobSubmitCommand(rayJob, rayv1.K8sJobMode)
|
|
require.Error(t, err)
|
|
}
|
|
|
|
// TestBuildJobSubmitCommandWithUnsetRayVersion verifies that
|
|
// metadata is still included when RayClusterSpec is present but RayVersion is empty. We assume
|
|
// the cluster is >= 2.6.0 unless the user explicitly sets a lower version.
|
|
func TestBuildJobSubmitCommandWithUnsetRayVersion(t *testing.T) {
|
|
rayJob := &rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{
|
|
RayClusterSpec: &rayv1.RayClusterSpec{},
|
|
Metadata: map[string]string{
|
|
"testKey": "testValue",
|
|
},
|
|
Entrypoint: "echo hello",
|
|
},
|
|
Status: rayv1.RayJobStatus{
|
|
DashboardURL: "http://127.0.0.1:8265",
|
|
JobId: "testJobId",
|
|
},
|
|
}
|
|
command, err := BuildJobSubmitCommand(rayJob, rayv1.K8sJobMode)
|
|
require.NoError(t, err)
|
|
|
|
hasMetadataFlag := false
|
|
for i, arg := range command {
|
|
if arg == "--metadata-json" {
|
|
hasMetadataFlag = true
|
|
require.Greater(t, len(command), i+1)
|
|
unquoted, err := strconv.Unquote(command[i+1])
|
|
require.NoError(t, err)
|
|
var metadata map[string]string
|
|
require.NoError(t, json.Unmarshal([]byte(unquoted), &metadata))
|
|
assert.Equal(t, "testValue", metadata["testKey"])
|
|
break
|
|
}
|
|
}
|
|
assert.True(t, hasMetadataFlag, "metadata-json flag should be present when RayVersion is unset")
|
|
}
|
|
|
|
func TestGetSubmitterTemplate(t *testing.T) {
|
|
rayJob := &rayv1.RayJob{
|
|
Spec: rayv1.RayJobSpec{},
|
|
}
|
|
rayCluster := &rayv1.RayCluster{
|
|
Spec: rayv1.RayClusterSpec{
|
|
HeadGroupSpec: rayv1.HeadGroupSpec{
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Image: "rayproject/ray:test-submitter-template",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
template := GetSubmitterTemplate(&rayJob.Spec, &rayCluster.Spec)
|
|
assert.Equal(t, template.Spec.Containers[0].Image, rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Image)
|
|
}
|