mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
+6








a5adbbe18b
* [RayService] More envtests that follow the most common scenario in the RayService code path (#2880) Signed-off-by: Rueian <[email protected]> * [RayService] Remove outdated env tests (#2886) Signed-off-by: kaihsun <[email protected]> * [RayService] Refactor envtests (#2888) * [docs][ray-operator] fix typo in Golang version (#2893) Project uses Golang 1.22 not 1.20. Signed-off-by: David Xia <[email protected]> * [Fix][kubectl-plugin] Fix no context nil error SIGSEGV in tests (#2892) Signed-off-by: Chi-Sheng Liu <[email protected]> * [CI] Enable testifylint rule (#2896) Signed-off-by: Chi-Sheng Liu <[email protected]> * [CI] Enable `testifylint` `error-nil` rule (#2907) * [kubectl-plugin][feat] support specifying number of head GPUs (#2895) when creating a RayCluster with `kubectl ray create cluster NAME --head-gpu N`. Similar to the `--worker-gpu` switch. Signed-off-by: David Xia <[email protected]> * Use webhook.CustomValidator instead of deprecated webhook.Validator. (#2803) * Use webhook.CustomValidator instead of deprecated webhook.Validator. * Move RayClusterWebhook to pkg/webhook/v1. * [kubectl-plugin] update context error messages (#2891) to tell user they can use `--context` to set the K8s context. Add tests Signed-off-by: David Xia <[email protected]> Co-authored-by: Chi-Sheng Liu <[email protected]> * [Fix][kubectl-plugin] make tests use a temporary kube config (#2894) file that's not at the default path. Right now tests fail if there's a K8s current context set with a command like `kubectl config use-context my-context`. This change allows tests to pass regardless of current context. Signed-off-by: David Xia <[email protected]> * [CI] Enable testifylint formatter rule (#2915) Signed-off-by: 400Ping <[email protected]> * [CI] Enable `testifylint` `empty` rule (#2908) * [CI] Enable testifylint bool-compare rule (#2911) Signed-off-by: 400Ping <[email protected]> * [CI] Auto download golang tools in pre-commit (#2917) Signed-off-by: Chi-Sheng Liu <[email protected]> * [RayService] a safeguard for preventing overriding the pending cluster during a upgrade (#2887) Signed-off-by: Rueian <[email protected]> * [RayService] Refactor unit tests for ShouldPrepareNewCluster (#2928) Signed-off-by: kaihsun <[email protected]> * [chore][kubectl-plugin] use consistent capitalization (#2922) in comments and console messages. * capitalize "Ray" when used as a proper noun * capitalize Ray K8s CRDs like "RayJob" and "RayCluster" * capitalize acronyms like "YAML" * fix some minor typos No functional changes. Signed-off-by: David Xia <[email protected]> * [CI] Enable `testifylint` `require-error` rule (#2909) * [kubectl-plugin] support general `kubectl` switches like `--context` (#2883) * [CI] Fix lint error (require-error) (#2931) Signed-off-by: Chi-Sheng Liu <[email protected]> * [CI] Enable `testifylint` `float-compare` rule (#2910) * [docs] move pre-commit instructions to main dev docs (#2921) * [CI] Enable `testifylint` `expected-actual` rule (#2914) * [CI] Generate CRD json schema separately in pre-commit (#2930) * [release][1/N] Update YAMLs from Ray 2.9 to Ray 2.41 (#2934) * Delete `[raycluster|rayjob|rayservice]_types_test.go` unnecessary tests (#2935) * [release][2/N] Update RayCluster Helm chart from Ray 2.9 to Ray 2.41 (#2936) Signed-off-by: kaihsun <[email protected]> * [release][3/N] Update RayService e2e tests YAML files from Ray 2.9 to Ray 2.41 (#2937) * [release][4/N] Update Ray images / versions in kubectl plugin (#2938) Signed-off-by: kaihsun <[email protected]> * [release][5/N] Update some RayJob YAMLs from Ray 2.9 to Ray 2.41 (#2941) Signed-off-by: kaihsun <[email protected]> * [release][6/N] Remove unnecessary YAMLs (#2946) Signed-off-by: kaihsun <[email protected]> * [docs][kubectl-plugin] add dev docs (#2912) Signed-off-by: David Xia <[email protected]> Co-authored-by: Chi-Sheng Liu <[email protected]> * [CI] Add shellcheck and fix error of it (#2933) * [chore][kubectl-plugin] use better test assertions (#2955) Fix two places. * use `assert.EqualError()` instead of `assert.Error()` to check the error string is what we expect * use testify assertion instead of string checking for better error output Signed-off-by: David Xia <[email protected]> * [chore] add Markdown linting pre-commit hook (#2953) Ignore most rules we violate for now. Fix these two violations. ``` pre-commit run markdownlint --all-files --show-diff-on-failure ... clients/python-client/README.md:106 MD001/heading-increment Heading levels should only increment by one level at a time [Expected: h3; Actual: h4] CHANGELOG.md:420:1 MD030/list-marker-space Spaces after list markers [Expected: 1; Actual: 2] ``` Signed-off-by: David Xia <[email protected]> * [chore][kubectl-plugin] use consistent capitalization (#2950) in comments and console messages. capitalize "Ray" when used as a proper noun. Similar to #2922. Signed-off-by: David Xia <[email protected]> * [Fix][RayJob] Invalid quote for RayJob submitter (#2949) Closes: ray-project/kuberay#2943 Signed-off-by: Chi-Sheng Liu <[email protected]> * [release][7/N] Update RayService YAMLs (#2956) * [CI] Enable testifylint len rule (#2945) Signed-off-by: LeoLiao123 <[email protected]> * [docs][kubectl-plugin] improve help messages (#2952) Signed-off-by: David Xia <[email protected]> * [kubectl-plugin] Fix panic when GPU resource is not set (#2954) Signed-off-by: win5923 <[email protected]> * [release][8/N] Upgrade Stable Diffusion RayService to Ray 2.41 (#2960) * [docs][kubectl-plugin] fix incorrect example commands (#2951) Signed-off-by: David Xia <[email protected]> Co-authored-by: Chi-Sheng Liu <[email protected]> * [fix][kubectl-plugin] set worker group CPU limit (#2958) when creating a new worker group with `kubectl ray create workergroup`. Write a unit test. I noticed we are setting resource limits equal to resource requests everywhere else but in this command. I have a K8s [LimitRange] that prevented the creation of these worker Pods that had CPU limit defaulting to less than their CPU requests. Describing the RayCluster showed this warning event. `Failed to create worker Pod hyperkube/, Pod "dxia-test-other-group-worker-pm2sh" is invalid: spec.containers[0].resources.requests: Invalid value: "2": must be less than or equal to cpu limit of 250m` Signed-off-by: David Xia <[email protected]> [LimitRange]: https://kubernetes.io/docs/concepts/policy/limit-range/ * [RayJob] Deflaky RayJob e2e tests (#2963) Signed-off-by: kaihsun <[email protected]> * [RayService] Deflaky RayService envtest (#2962) Signed-off-by: kaihsun <[email protected]> * [release][9/N] Update text summarizer RayService to Ray 2.41 (#2961) * [RayService] adapter vllm 0.6.1.post2 (#2823) * adapter vllm 0.6.1.post2 * fix var define * Unify the cpu Settings in serve.py in service.yaml, all set to 1 * Maintain the configuration with vllm0.5x * [Release] Upgrade ray-job.batch-inference.yaml image to 2.41 (#2971) Signed-off-by: Chi-Sheng Liu <[email protected]> * [chore][docs] enable Markdownlint rule MD010 (#2975) * [CI] Change Pre-commit-shellcheck-to-shellcheck-py (#2974) Signed-off-by: owenowenisme <[email protected]> * [release] Update Yunikorn YAML file to Ray 2.41 (#2969) Signed-off-by: Cheng-Yeh Chung <[email protected]> * [release] Update YuniKorn YAML files to Ray 2.41 (#2976) Signed-off-by: win5923 <[email protected]> * [chore][docs] enable Markdownlint rule MD004 (#2973) [Unordered list style][1] [1]: https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md004---unordered-list-style * [Test] Use GcsFaultToleranceOptions in test and backward compatibility (#2972) * Update samples to use Ray 2.41.0 images (#2964) Signed-off-by: Andrew Sy Kim <[email protected]> * Update TPU Ray CR manifests to use Ray 2.41.0 (#2965) * [Refactor] Use constants for image tag, image repo, and versions in golang to avoid hard-coded strings (#2978) --------- Signed-off-by: Rueian <[email protected]> Signed-off-by: kaihsun <[email protected]> Signed-off-by: David Xia <[email protected]> Signed-off-by: Chi-Sheng Liu <[email protected]> Signed-off-by: 400Ping <[email protected]> Signed-off-by: LeoLiao123 <[email protected]> Signed-off-by: win5923 <[email protected]> Signed-off-by: owenowenisme <[email protected]> Signed-off-by: Cheng-Yeh Chung <[email protected]> Signed-off-by: Andrew Sy Kim <[email protected]> Co-authored-by: Rueian <[email protected]> Co-authored-by: Kai-Hsun Chen <[email protected]> Co-authored-by: David Xia <[email protected]> Co-authored-by: Chi-Sheng Liu <[email protected]> Co-authored-by: David Xia <[email protected]> Co-authored-by: Mykhailo Bobrovskyi <[email protected]> Co-authored-by: Ping <[email protected]> Co-authored-by: Owen Lin <[email protected]> Co-authored-by: Leo Liao <[email protected]> Co-authored-by: Blocka <[email protected]> Co-authored-by: zrant <[email protected]> Co-authored-by: kenchung285 <[email protected]> Co-authored-by: fscnick <[email protected]> Co-authored-by: ryanaoleary <[email protected]>
170 lines
4.5 KiB
Go
170 lines
4.5 KiB
Go
package support
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
"github.com/onsi/gomega"
|
|
"github.com/stretchr/testify/require"
|
|
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
|
"k8s.io/client-go/tools/remotecommand"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/client-go/tools/portforward"
|
|
"k8s.io/client-go/transport/spdy"
|
|
)
|
|
|
|
func Pods(t Test, namespace string, options ...Option[*metav1.ListOptions]) func(g gomega.Gomega) []corev1.Pod {
|
|
return func(g gomega.Gomega) []corev1.Pod {
|
|
listOptions := &metav1.ListOptions{}
|
|
|
|
for _, option := range options {
|
|
g.Expect(option.applyTo(listOptions)).To(gomega.Succeed())
|
|
}
|
|
|
|
pods, err := t.Client().Core().CoreV1().Pods(namespace).List(t.Ctx(), *listOptions)
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
return pods.Items
|
|
}
|
|
}
|
|
|
|
func storeAllPodLogs(t Test, namespace *corev1.Namespace) {
|
|
t.T().Helper()
|
|
|
|
pods, err := t.Client().Core().CoreV1().Pods(namespace.Name).List(t.Ctx(), metav1.ListOptions{})
|
|
require.NoError(t.T(), err)
|
|
|
|
for _, pod := range pods.Items {
|
|
for _, container := range pod.Spec.Containers {
|
|
t.T().Logf("Retrieving Pod Container %s/%s/%s logs", pod.Namespace, pod.Name, container.Name)
|
|
storeContainerLog(t, namespace, pod.Name, container.Name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func storeContainerLog(t Test, namespace *corev1.Namespace, podName, containerName string) {
|
|
t.T().Helper()
|
|
|
|
options := corev1.PodLogOptions{Container: containerName}
|
|
stream, err := t.Client().Core().CoreV1().Pods(namespace.Name).GetLogs(podName, &options).Stream(t.Ctx())
|
|
if err != nil {
|
|
t.T().Logf("Error getting logs from container %s/%s/%s", namespace.Name, podName, containerName)
|
|
return
|
|
}
|
|
require.NoError(t.T(), err)
|
|
|
|
defer func() {
|
|
require.NoError(t.T(), stream.Close())
|
|
}()
|
|
|
|
bytes, err := io.ReadAll(stream)
|
|
require.NoError(t.T(), err)
|
|
|
|
containerLogFileName := "pod-" + podName + "-" + containerName
|
|
WriteToOutputDir(t, containerLogFileName, Log, bytes)
|
|
}
|
|
|
|
func ExecPodCmd(t Test, pod *corev1.Pod, containerName string, cmd []string) (bytes.Buffer, bytes.Buffer) {
|
|
req := t.Client().Core().CoreV1().RESTClient().
|
|
Post().
|
|
Resource("pods").
|
|
Name(pod.Name).
|
|
Namespace(pod.Namespace).
|
|
SubResource("exec").
|
|
VersionedParams(&corev1.PodExecOptions{
|
|
Command: cmd,
|
|
Container: containerName,
|
|
Stdin: false,
|
|
Stdout: true,
|
|
Stderr: true,
|
|
TTY: false,
|
|
}, clientgoscheme.ParameterCodec)
|
|
|
|
t.T().Logf("Executing command: %s", cmd)
|
|
cfg := t.Client().Config()
|
|
exec, err := remotecommand.NewSPDYExecutor(&cfg, "POST", req.URL())
|
|
require.NoError(t.T(), err)
|
|
// Capture the output streams
|
|
var stdout, stderr bytes.Buffer
|
|
// Execute the command in the pod
|
|
err = exec.StreamWithContext(t.Ctx(), remotecommand.StreamOptions{
|
|
Stdin: nil,
|
|
Stdout: &stdout,
|
|
Stderr: &stderr,
|
|
Tty: false,
|
|
})
|
|
t.T().Logf("Command stdout: %s", stdout.String())
|
|
t.T().Logf("Command stderr: %s", stderr.String())
|
|
require.NoError(t.T(), err)
|
|
return stdout, stderr
|
|
}
|
|
|
|
func SetupPortForward(t Test, podName, namespace string, localPort, remotePort int) (chan struct{}, error) {
|
|
cfg := t.Client().Config()
|
|
|
|
req := t.Client().Core().CoreV1().RESTClient().
|
|
Post().
|
|
Resource("pods").
|
|
Namespace(namespace).
|
|
Name(podName).
|
|
SubResource("portforward")
|
|
|
|
transport, upgrader, err := spdy.RoundTripperFor(&cfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
stopChan := make(chan struct{}, 1)
|
|
readyChan := make(chan struct{}, 1)
|
|
out := new(strings.Builder)
|
|
errOut := new(strings.Builder)
|
|
|
|
// create port forward
|
|
forwarder, err := portforward.New(
|
|
spdy.NewDialer(upgrader, &http.Client{Transport: transport}, http.MethodPost, req.URL()),
|
|
[]string{fmt.Sprintf("%d:%d", localPort, remotePort)},
|
|
stopChan,
|
|
readyChan,
|
|
out,
|
|
errOut,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// launch Port Forward
|
|
go func() {
|
|
defer GinkgoRecover()
|
|
err := forwarder.ForwardPorts()
|
|
require.NoError(t.T(), err)
|
|
}()
|
|
<-readyChan // wait for port forward to finish
|
|
|
|
return stopChan, nil
|
|
}
|
|
|
|
func CreateCurlPod(t Test, podName, containerName, namespace string) (*corev1.Pod, error) {
|
|
// Define the podSpec spec
|
|
podSpec := &corev1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
Namespace: namespace,
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: containerName,
|
|
Image: "rancher/curl",
|
|
Command: []string{"/bin/sh", "-c", "tail -f /dev/null"},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
return t.Client().Core().CoreV1().Pods(namespace).Create(t.Ctx(), podSpec, metav1.CreateOptions{})
|
|
}
|