Files

601 lines
16 KiB
Go

package job
import (
"fmt"
"os"
"path/filepath"
"reflect"
"testing"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/cli-runtime/pkg/genericclioptions"
cmdutil "k8s.io/kubectl/pkg/cmd/util"
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
)
func TestRayJobSubmitComplete(t *testing.T) {
testStreams, _, _, _ := genericclioptions.NewTestIOStreams()
configFlags := genericclioptions.NewConfigFlags(true)
cmdFactory := cmdutil.NewFactory(configFlags)
fakeSubmitJobOptions := NewJobSubmitOptions(cmdFactory, testStreams)
fakeSubmitJobOptions.runtimeEnv = "././fake/path/to/env/yaml"
fakeSubmitJobOptions.fileName = "fake/path/to/rayjob.yaml"
cmd := &cobra.Command{}
configFlags.AddFlags(cmd.Flags())
err := fakeSubmitJobOptions.Complete()
require.NoError(t, err)
assert.Equal(t, "default", fakeSubmitJobOptions.namespace)
assert.Equal(t, "fake/path/to/env/yaml", fakeSubmitJobOptions.runtimeEnv)
}
func TestRayJobSubmitWithYamlValidate(t *testing.T) {
testStreams, _, _, _ := genericclioptions.NewTestIOStreams()
cmdFactory := cmdutil.NewFactory(genericclioptions.NewConfigFlags(true))
fakeDir := t.TempDir()
tests := []struct {
name string
yamlContent string
expectError string
}{
{
name: "Successful submit job validation with RayJob",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
submissionMode: 'InteractiveMode'`,
},
{
name: "BackoffLimit co-exist with InteractiveMode",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
submissionMode: 'InteractiveMode'
backoffLimit: 1`,
expectError: "BackoffLimit is incompatible with InteractiveMode",
},
{
name: "BackoffLimit is set to 0 with InteractiveMode",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
submissionMode: 'InteractiveMode'
backoffLimit: 0`,
},
{
name: "shutdownAfterJobFinishes is false and ttlSecondsAfterFinished is not zero",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
shutdownAfterJobFinishes: false
ttlSecondsAfterFinished: 10
submissionMode: 'InteractiveMode'`,
expectError: "ttlSecondsAfterFinished is only supported when shutdownAfterJobFinishes is set to true",
},
{
name: "shutdownAfterJobFinishes is true and ttlSecondsAfterFinished is not zero",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
shutdownAfterJobFinishes: true
ttlSecondsAfterFinished: 10
submissionMode: 'InteractiveMode'`,
},
{
name: "ttlSecondsAfterFinished is less than zero",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
shutdownAfterJobFinishes: true
ttlSecondsAfterFinished: -10
submissionMode: 'InteractiveMode'`,
expectError: "ttlSecondsAfterFinished must be greater than or equal to 0",
},
{
name: "ttlSecondsAfterFinished is less than zero",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
shutdownAfterJobFinishes: true
ttlSecondsAfterFinished: 0
submissionMode: 'InteractiveMode'`,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
rayJobYamlPath := filepath.Join(fakeDir, "rayjob-temp-*.yaml")
file, err := os.Create(rayJobYamlPath)
require.NoError(t, err)
_, err = file.Write([]byte(tc.yamlContent))
require.NoError(t, err)
opts := &SubmitJobOptions{
cmdFactory: cmdFactory,
ioStreams: &testStreams,
fileName: rayJobYamlPath,
workingDir: "Fake/File/Path",
}
err = opts.Validate(&cobra.Command{})
if tc.expectError != "" {
require.EqualError(t, err, tc.expectError)
} else {
require.NoError(t, err)
}
})
}
}
func TestRayJobSubmitWithoutYamlValidate(t *testing.T) {
testStreams, _, _, _ := genericclioptions.NewTestIOStreams()
cmdFactory := cmdutil.NewFactory(genericclioptions.NewConfigFlags(true))
test := []struct {
name string
rayjobName string
expectError string
ttlSecondsAfterFinished int32
shutdownAfterJobFinishes bool
}{
{
name: "ttlSecondsAfterFinished is validate value",
rayjobName: "rayjob-sample",
ttlSecondsAfterFinished: 10,
},
{
name: "ttlSecondsAfterFinished is less than zero",
rayjobName: "rayjob-sample",
ttlSecondsAfterFinished: -10,
expectError: "--ttl-seconds-after-finished must be greater than or equal to 0",
},
{
name: "ttlSecondsAfterFinished is equal to zero",
rayjobName: "rayjob-sample",
ttlSecondsAfterFinished: 0,
},
}
for _, tc := range test {
t.Run(tc.name, func(t *testing.T) {
opts := &SubmitJobOptions{
cmdFactory: cmdFactory,
ioStreams: &testStreams,
rayjobName: tc.rayjobName,
workingDir: "Fake/File/Path",
ttlSecondsAfterFinished: tc.ttlSecondsAfterFinished,
}
err := opts.Validate(&cobra.Command{})
if tc.expectError != "" {
require.EqualError(t, err, tc.expectError)
} else {
require.NoError(t, err)
}
})
}
}
func TestRayJobSubmit_AddressValidation(t *testing.T) {
testStreams, _, _, _ := genericclioptions.NewTestIOStreams()
cmdFactory := cmdutil.NewFactory(genericclioptions.NewConfigFlags(true))
test := []struct {
name string
address string
expectError string
expectNormalized string
flagChanged bool
}{
{
name: "address flag not set: port-forward mode",
address: "",
flagChanged: false,
},
{
name: "address flag set but empty: error",
address: "",
flagChanged: true,
expectError: "--address was provided but is empty",
},
{
name: "valid https address: OK",
address: "https://ingress.example.com",
flagChanged: true,
},
}
for _, tc := range test {
t.Run(tc.name, func(t *testing.T) {
opts := &SubmitJobOptions{
cmdFactory: cmdFactory,
ioStreams: &testStreams,
rayjobName: "rayjob-sample",
workingDir: "Fake/File/Path",
}
cmd := &cobra.Command{}
cmd.Flags().StringVar(&opts.address, "address", "", "Ray Dashboard base URL")
if tc.flagChanged {
require.NoError(t, cmd.Flags().Set("address", tc.address))
} else {
opts.address = tc.address
}
err := opts.Validate(cmd)
if tc.expectError != "" {
require.EqualError(t, err, tc.expectError)
} else {
require.NoError(t, err)
if tc.expectNormalized != "" {
require.Equal(t, tc.expectNormalized, opts.address)
}
}
})
}
}
func TestRayJobSubmitCmdFlagsOverrideYaml(t *testing.T) {
testStreams, _, _, _ := genericclioptions.NewTestIOStreams()
cmdFactory := cmdutil.NewFactory(genericclioptions.NewConfigFlags(true))
fakeDir := t.TempDir()
tests := []struct {
name string
yamlContent string
flagMap map[string]any
expectSpec map[string]any
expectError string
}{
{
name: "Both shutdownAfterJobFinishes and ttlSecondsAfterFinished are not set in yaml",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
submissionMode: 'InteractiveMode'`,
flagMap: map[string]any{
"ttl-seconds-after-finished": 20,
},
expectSpec: map[string]any{
"ShutdownAfterJobFinishes": true,
"TTLSecondsAfterFinished": int32(20),
},
},
{
name: "Both shutdownAfterJobFinishes and ttlSecondsAfterFinished are set in yaml with wrong values",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
shutdownAfterJobFinishes: false
ttlSecondsAfterFinished: -10
submissionMode: 'InteractiveMode'`,
flagMap: map[string]any{
"ttl-seconds-after-finished": 20,
},
expectSpec: map[string]any{
"ShutdownAfterJobFinishes": true,
"TTLSecondsAfterFinished": int32(20),
},
},
{
name: "Only shutdownAfterJobFinishes is set in yaml",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
shutdownAfterJobFinishes: false
submissionMode: 'InteractiveMode'`,
flagMap: map[string]any{
"ttl-seconds-after-finished": 20,
},
expectSpec: map[string]any{
"ShutdownAfterJobFinishes": true,
"TTLSecondsAfterFinished": int32(20),
},
},
{
name: "Only ttlSecondsAfterFinished is set in yaml",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
ttlSecondsAfterFinished: 10
submissionMode: 'InteractiveMode'`,
flagMap: map[string]any{
"ttl-seconds-after-finished": 20,
},
expectSpec: map[string]any{
"ShutdownAfterJobFinishes": true,
"TTLSecondsAfterFinished": int32(20),
},
},
{
name: "Override only ttlSecondsAfterFinished",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
shutdownAfterJobFinishes: true
ttlSecondsAfterFinished: 100
submissionMode: 'InteractiveMode'`,
flagMap: map[string]any{
"ttl-seconds-after-finished": 200,
},
expectSpec: map[string]any{
"ShutdownAfterJobFinishes": true,
"TTLSecondsAfterFinished": int32(200),
},
},
{
name: "Set ttl-seconds-after-finished to zero",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
submissionMode: 'InteractiveMode'`,
flagMap: map[string]any{
"ttl-seconds-after-finished": 0,
},
expectSpec: map[string]any{
"ShutdownAfterJobFinishes": true,
"TTLSecondsAfterFinished": int32(0),
},
},
{
name: "Override only ttlSecondsAfterFinished to negative and cause error",
yamlContent: `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
shutdownAfterJobFinishes: true
ttlSecondsAfterFinished: 10
submissionMode: 'InteractiveMode'`,
flagMap: map[string]any{
"ttl-seconds-after-finished": -10,
},
expectError: "--ttl-seconds-after-finished must be greater than or equal to 0",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
rayJobYamlPath := filepath.Join(fakeDir, "rayjob-temp-*.yaml")
file, err := os.Create(rayJobYamlPath)
require.NoError(t, err)
_, err = file.Write([]byte(tc.yamlContent))
require.NoError(t, err)
opts := &SubmitJobOptions{
cmdFactory: cmdFactory,
ioStreams: &testStreams,
fileName: rayJobYamlPath,
workingDir: "Fake/File/Path",
}
cmd := &cobra.Command{}
cmd.Flags().Int32Var(&opts.ttlSecondsAfterFinished, "ttl-seconds-after-finished", 0, "")
args := []string{}
for flag, value := range tc.flagMap {
if v, ok := value.(bool); ok && v {
args = append(args, fmt.Sprintf("--%s", flag))
} else {
args = append(args, fmt.Sprintf("--%s=%v", flag, value))
}
}
cmd.SetArgs(args)
err = cmd.ParseFlags(args)
require.NoError(t, err)
err = opts.Validate(cmd)
if tc.expectError != "" {
require.EqualError(t, err, tc.expectError)
} else {
require.NoError(t, err)
}
if tc.expectSpec != nil {
for field, expected := range tc.expectSpec {
actual := reflect.ValueOf(opts.RayJob.Spec).FieldByName(field).Interface()
require.Equal(t, expected, actual)
}
}
})
}
}
func TestDecodeRayJobYaml(t *testing.T) {
rayjobtmpfile, err := os.CreateTemp("./", "rayjob-temp-*.yaml")
require.NoError(t, err)
defer os.Remove(rayjobtmpfile.Name())
rayYaml := `apiVersion: ray.io/v1
kind: RayJob
metadata:
name: rayjob-sample
spec:
submissionMode: 'InteractiveMode'`
_, err = rayjobtmpfile.Write([]byte(rayYaml))
require.NoError(t, err)
rayJobYamlActual, err := decodeRayJobYaml(filepath.Join("./", rayjobtmpfile.Name()))
require.NoError(t, err)
assert.Equal(t, "rayjob-sample", rayJobYamlActual.GetName())
submissionMode := rayJobYamlActual.Spec.SubmissionMode
assert.Equal(t, rayv1.InteractiveMode, submissionMode)
}
func TestRuntimeEnvHasWorkingDir(t *testing.T) {
runtimeEnvFile, err := os.CreateTemp("./", "runtime-env-*.yaml")
require.NoError(t, err)
defer os.Remove(runtimeEnvFile.Name())
runTimeEnv := `pip:
- requests==2.26.0
- pendulum==2.1.2
env_vars:
counter_name: "test_counter"
working_dir: /fake/dir/ray_working_dir/
`
_, err = runtimeEnvFile.Write([]byte(runTimeEnv))
require.NoError(t, err)
runtimeEnvActual, err := runtimeEnvHasWorkingDir(filepath.Join("./", runtimeEnvFile.Name()))
require.NoError(t, err)
assert.NotEmpty(t, runtimeEnvActual)
assert.Equal(t, "/fake/dir/ray_working_dir/", runtimeEnvActual)
}
func TestRaySubmitCmd(t *testing.T) {
testStreams, _, _, _ := genericclioptions.NewTestIOStreams()
cmdFactory := cmdutil.NewFactory(genericclioptions.NewConfigFlags(true))
fakeSubmitJobOptions := NewJobSubmitOptions(cmdFactory, testStreams)
fakeSubmitJobOptions.runtimeEnv = "/fake-runtime/path"
fakeSubmitJobOptions.runtimeEnvJson = "{\"env_vars\":{\"counter_name\":\"test_counter\"}"
fakeSubmitJobOptions.submissionID = "fake-submission-id12345"
fakeSubmitJobOptions.entryPointCPU = 2.0
fakeSubmitJobOptions.entryPointGPU = 1.0
fakeSubmitJobOptions.entryPointMemory = 600
fakeSubmitJobOptions.entryPointResource = "{\"fake-resource\":{\"the-fake-resource\"}"
fakeSubmitJobOptions.noWait = true
fakeSubmitJobOptions.headers = "{\"requestHeaders\": {\"header\": \"header\"}}"
fakeSubmitJobOptions.verify = "True"
fakeSubmitJobOptions.workingDir = "/fake/working/dir"
fakeSubmitJobOptions.entryPoint = "python fake_python_script.py"
fakeSubmitJobOptions.address = dashboardAddr
actualCmd, err := fakeSubmitJobOptions.raySubmitCmd()
require.NoError(t, err)
expectedCmd := []string{
"ray",
"job",
"submit",
"--address",
dashboardAddr,
"--runtime-env",
"/fake-runtime/path",
"--runtime-env-json",
"{\"env_vars\":{\"counter_name\":\"test_counter\"}",
"--submission-id",
"fake-submission-id12345",
"--entrypoint-num-cpus",
"2.000000",
"--entrypoint-num-gpus",
"1.000000",
"--entrypoint-memory",
"600",
"--entrypoint-resource",
"{\"fake-resource\":{\"the-fake-resource\"}",
"--no-wait",
"--headers",
"{\"requestHeaders\": {\"header\": \"header\"}}",
"--verify",
"True",
"--working-dir",
"/fake/working/dir",
"--",
"python",
"fake_python_script.py",
}
assert.Equal(t, expectedCmd, actualCmd)
}
func TestRayJobSubmit_FlagsHaveDefaults(t *testing.T) {
streams, _, _, _ := genericclioptions.NewTestIOStreams()
factory := cmdutil.NewFactory(genericclioptions.NewConfigFlags(true))
opts := NewJobSubmitOptions(factory, streams)
cmd := NewJobSubmitCommand(factory, streams)
require.NoError(t, cmd.ParseFlags([]string{}))
assert.InDelta(t, float32(0), opts.entryPointCPU, 1e-6, "default entrypoint-num-cpus should be 0")
assert.InDelta(t, float32(0), opts.entryPointGPU, 1e-6, "default entrypoint-num-gpus should be 0")
assert.Equal(t, 0, opts.entryPointMemory, "default entrypoint-memory should be 0")
assert.False(t, opts.noWait, "default no-wait should be false")
}
func TestRaySubmitCmd_AddressSelection(t *testing.T) {
streams, _, _, _ := genericclioptions.NewTestIOStreams()
factory := cmdutil.NewFactory(genericclioptions.NewConfigFlags(true))
makeCmd := func(addr string) ([]string, error) {
opts := NewJobSubmitOptions(factory, streams)
opts.workingDir = "/fake/working/dir"
opts.entryPoint = "python fake.py"
opts.address = addr
return opts.raySubmitCmd()
}
tests := []struct {
name string
address string
expectedAddr string
}{
{
name: "no address provided: falls back to dashboardAddr",
address: "",
expectedAddr: dashboardAddr,
},
{
name: "custom address provided: uses custom",
address: "https://ingress.example.com",
expectedAddr: "https://ingress.example.com",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
cmd, err := makeCmd(tc.address)
require.NoError(t, err)
require.GreaterOrEqual(t, len(cmd), 5, "command too short")
assert.Equal(t, "ray", cmd[0])
assert.Equal(t, "job", cmd[1])
assert.Equal(t, "submit", cmd[2])
assert.Equal(t, "--address", cmd[3])
assert.Equal(t, tc.expectedAddr, cmd[4])
require.Contains(t, cmd, "--working-dir")
require.Contains(t, cmd, "--")
})
}
}