mirror of
https://github.com/hanzo-ml/kuberay.git
synced 2026-10-30 19:48:47 +00:00
* [historyserver] implement grafana health for live session Signed-off-by: fscnick <[email protected]> * [historyserver] move getGrafanaHealth to router.go Signed-off-by: fscnick <[email protected]> * [Doc][historyserver] add setup for grafana and RayCluster with history server Signed-off-by: fscnick <[email protected]> * [Test][historyserver] add e2e test for grafana health Signed-off-by: fscnick <[email protected]> * [Test][historyserver] use CombinedOutput to ensure cleanup exits after deletion finished Signed-off-by: fscnick <[email protected]> * [Test][historyserver] test grafana health response body Signed-off-by: fscnick <[email protected]> * [Test][historyserver] add missing injectCollectorRayClusterID Signed-off-by: fscnick <[email protected]> --------- Signed-off-by: fscnick <[email protected]>
79 lines
2.5 KiB
Go
79 lines
2.5 KiB
Go
package support
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/cookiejar"
|
|
"os/exec"
|
|
"time"
|
|
|
|
. "github.com/onsi/gomega"
|
|
corev1 "k8s.io/api/core/v1"
|
|
|
|
. "github.com/ray-project/kuberay/ray-operator/test/support"
|
|
)
|
|
|
|
// CreateHTTPClientWithCookieJar creates an HTTP client with cookie jar to maintain session.
|
|
func CreateHTTPClientWithCookieJar(g *WithT) *http.Client {
|
|
jar, err := cookiejar.New(nil)
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
return &http.Client{
|
|
Jar: jar,
|
|
Timeout: 30 * time.Second,
|
|
}
|
|
}
|
|
|
|
// GetContainerStatusByName retrieves the container status by container name.
|
|
// NOTE: ContainerStatuses order doesn't guarantee to match Spec.Containers order.
|
|
// For more details, please refer to the following link:
|
|
// https://github.com/ray-project/kuberay/blob/7791a8786861818f0cebcce381ef221436a0fa4d/ray-operator/controllers/ray/raycluster_controller.go#L1160C1-L1171C2
|
|
func GetContainerStatusByName(pod *corev1.Pod, containerName string) (*corev1.ContainerStatus, error) {
|
|
for _, containerStatus := range pod.Status.ContainerStatuses {
|
|
if containerStatus.Name == containerName {
|
|
return &containerStatus, nil
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("container %s not found in pod %s/%s", containerName, pod.Namespace, pod.Name)
|
|
}
|
|
|
|
func PortForwardService(test Test, g *WithT, namespace, serviceName string, port int) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
test.T().Cleanup(cancel)
|
|
|
|
kubectlCmd := exec.CommandContext(
|
|
ctx,
|
|
"kubectl",
|
|
"-n", namespace,
|
|
"port-forward",
|
|
fmt.Sprintf("svc/%s", serviceName),
|
|
fmt.Sprintf("%d:%d", port, port),
|
|
)
|
|
err := kubectlCmd.Start()
|
|
g.Expect(err).NotTo(HaveOccurred())
|
|
}
|
|
|
|
// InstallGrafanaAndPrometheus installs Grafana and Prometheus in the cluster for testing.
|
|
func InstallGrafanaAndPrometheus(test Test, g *WithT) {
|
|
test.T().Cleanup(func() {
|
|
cleanCMD := exec.Command("kubectl", "delete", "ns", "prometheus-system")
|
|
output, err := cleanCMD.CombinedOutput()
|
|
if err != nil {
|
|
LogWithTimestamp(test.T(), "Failed to clean up Prometheus and Grafana installation with %v because %s",
|
|
err, string(output))
|
|
return
|
|
}
|
|
LogWithTimestamp(test.T(), "Clean up Prometheus and Grafana installation.")
|
|
})
|
|
|
|
installGrafanaCMD := exec.CommandContext(
|
|
test.Ctx(),
|
|
"../../../install/prometheus/install.sh",
|
|
"--auto-load-dashboard",
|
|
"true",
|
|
)
|
|
output, err := installGrafanaCMD.CombinedOutput()
|
|
g.Expect(err).NotTo(HaveOccurred(), "fail to install because %s", string(output))
|
|
LogWithTimestamp(test.T(), "Install Grafana and Prometheus successfully.")
|
|
}
|