Files
fscnickandGitHub b404142118 [historyserver] implement grafana health for live session (#4425)
* [historyserver] implement grafana health for live session

Signed-off-by: fscnick <[email protected]>

* [historyserver] move getGrafanaHealth to router.go

Signed-off-by: fscnick <[email protected]>

* [Doc][historyserver] add setup for grafana and RayCluster with history server

Signed-off-by: fscnick <[email protected]>

* [Test][historyserver] add e2e test for grafana health

Signed-off-by: fscnick <[email protected]>

* [Test][historyserver] use CombinedOutput to ensure cleanup exits after deletion finished

Signed-off-by: fscnick <[email protected]>

* [Test][historyserver] test grafana health response body

Signed-off-by: fscnick <[email protected]>

* [Test][historyserver] add missing injectCollectorRayClusterID

Signed-off-by: fscnick <[email protected]>

---------

Signed-off-by: fscnick <[email protected]>
2026-02-02 08:07:32 -08:00

79 lines
2.5 KiB
Go

package support
import (
"context"
"fmt"
"net/http"
"net/http/cookiejar"
"os/exec"
"time"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
. "github.com/ray-project/kuberay/ray-operator/test/support"
)
// CreateHTTPClientWithCookieJar creates an HTTP client with cookie jar to maintain session.
func CreateHTTPClientWithCookieJar(g *WithT) *http.Client {
jar, err := cookiejar.New(nil)
g.Expect(err).NotTo(HaveOccurred())
return &http.Client{
Jar: jar,
Timeout: 30 * time.Second,
}
}
// GetContainerStatusByName retrieves the container status by container name.
// NOTE: ContainerStatuses order doesn't guarantee to match Spec.Containers order.
// For more details, please refer to the following link:
// https://github.com/ray-project/kuberay/blob/7791a8786861818f0cebcce381ef221436a0fa4d/ray-operator/controllers/ray/raycluster_controller.go#L1160C1-L1171C2
func GetContainerStatusByName(pod *corev1.Pod, containerName string) (*corev1.ContainerStatus, error) {
for _, containerStatus := range pod.Status.ContainerStatuses {
if containerStatus.Name == containerName {
return &containerStatus, nil
}
}
return nil, fmt.Errorf("container %s not found in pod %s/%s", containerName, pod.Namespace, pod.Name)
}
func PortForwardService(test Test, g *WithT, namespace, serviceName string, port int) {
ctx, cancel := context.WithCancel(context.Background())
test.T().Cleanup(cancel)
kubectlCmd := exec.CommandContext(
ctx,
"kubectl",
"-n", namespace,
"port-forward",
fmt.Sprintf("svc/%s", serviceName),
fmt.Sprintf("%d:%d", port, port),
)
err := kubectlCmd.Start()
g.Expect(err).NotTo(HaveOccurred())
}
// InstallGrafanaAndPrometheus installs Grafana and Prometheus in the cluster for testing.
func InstallGrafanaAndPrometheus(test Test, g *WithT) {
test.T().Cleanup(func() {
cleanCMD := exec.Command("kubectl", "delete", "ns", "prometheus-system")
output, err := cleanCMD.CombinedOutput()
if err != nil {
LogWithTimestamp(test.T(), "Failed to clean up Prometheus and Grafana installation with %v because %s",
err, string(output))
return
}
LogWithTimestamp(test.T(), "Clean up Prometheus and Grafana installation.")
})
installGrafanaCMD := exec.CommandContext(
test.Ctx(),
"../../../install/prometheus/install.sh",
"--auto-load-dashboard",
"true",
)
output, err := installGrafanaCMD.CombinedOutput()
g.Expect(err).NotTo(HaveOccurred(), "fail to install because %s", string(output))
LogWithTimestamp(test.T(), "Install Grafana and Prometheus successfully.")
}