mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
github.com/hanzoai/ai/object is the record layer. Seven of its files were not records: they built client-go clients, ran an informer cache, polled the metrics API, rendered manifests through kustomize and applied them. Importing an Application row therefore linked a Kubernetes control plane. They move to github.com/hanzoai/ai/cluster, which owns everything the product does against a cluster: render, apply, observe. object keeps the values — ApplicationView and its nested types are plain data with no k8s types in them, so Application.Details keeps its exact static type and the /v1/get-application wire shape is unchanged. The Status constants become their own string literals rather than aliases of core/v1 pod phases. Two braids come apart with the move: UpdateApplication recomputed the manifest from the template on every write, so a status or URL update re-ran kustomize. It now writes the record; cluster.Manifest renders, and the four callers that want a fresh manifest render first. DeleteApplication raced a fire-and-forget undeploy goroutine against its own row delete. The controllers now tear down first, best-effort, then drop the row — an org with no Kubernetes provider still deletes its applications. RegisterServedModel, which writes the Provider and ModelRoute for a served fine-tune, is a record write and stays in object; it was the one place reaching adapter.db from the moved files. No route, opcode, swagger path or admin page changes. Names lose the stutter the package qualifier now carries: GetK8sStatus is cluster.Status, AddDetails is cluster.Describe, DeployApplicationSync is cluster.DeploySync, GetFinetuneTrainJobStatus is cluster.TrainJobStatus. object drops 538 packages, 1799 to 1261: all 412 k8s.io, all 84 sigs.k8s.io, and with kustomize gone the `plugin` package, which forces --export-dynamic on every binary that links it. Co-authored-by: Hanzo Dev <[email protected]>
512 lines
14 KiB
Go
512 lines
14 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
// Portions Copyright 2025 The OpenAgent Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
package cluster
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
networkingv1 "k8s.io/api/networking/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
"k8s.io/client-go/informers"
|
|
coreinformers "k8s.io/client-go/informers/core/v1"
|
|
"k8s.io/client-go/tools/cache"
|
|
)
|
|
|
|
type cachedMetrics struct {
|
|
TotalCPU resource.Quantity `json:"totalCPU"`
|
|
TotalMemory resource.Quantity `json:"totalMemory"`
|
|
PodCount int `json:"podCount"`
|
|
CPUPercentage float64 `json:"cpuPercentage"`
|
|
MemoryPercentage float64 `json:"memoryPercentage"`
|
|
LastUpdated time.Time `json:"lastUpdated"`
|
|
}
|
|
|
|
// eventCache represents namespace-level event cache
|
|
type eventCache struct {
|
|
Namespace string `json:"namespace"` // Namespace name
|
|
Events map[string]*v1.Event `json:"events"` // Event name -> Event object
|
|
}
|
|
type cacheManager struct {
|
|
factory informers.SharedInformerFactory
|
|
nsInformer coreinformers.NamespaceInformer
|
|
svcCache map[string]map[string]*v1.Service // namespace -> name -> service
|
|
deployCache map[string]map[string]*appsv1.Deployment // namespace -> name -> deployment
|
|
ingressCache map[string]map[string]*networkingv1.Ingress // namespace -> name -> ingress
|
|
eventCaches map[string]*eventCache // namespace -> eventCache
|
|
nsCache map[string]*v1.Namespace // name -> namespace
|
|
nodeCache map[string]*v1.Node // name -> node
|
|
metricsCache map[string]*cachedMetrics // namespace -> metrics
|
|
mu sync.RWMutex
|
|
stopCh chan struct{}
|
|
started bool
|
|
// debug counters
|
|
svcHits int
|
|
deployHits int
|
|
nodeHits int
|
|
nsHits int
|
|
ingressHits int
|
|
eventHits int
|
|
metricsHits int
|
|
}
|
|
|
|
var (
|
|
mirror *cacheManager
|
|
cachedHost string
|
|
)
|
|
|
|
// initMirror initializes the cache manager
|
|
func initMirror() error {
|
|
if mirror != nil {
|
|
return nil
|
|
}
|
|
factory := informers.NewSharedInformerFactory(client.clientSet, 30*time.Second)
|
|
mgr := &cacheManager{
|
|
factory: factory,
|
|
nsInformer: factory.Core().V1().Namespaces(),
|
|
svcCache: make(map[string]map[string]*v1.Service),
|
|
deployCache: make(map[string]map[string]*appsv1.Deployment),
|
|
ingressCache: make(map[string]map[string]*networkingv1.Ingress),
|
|
eventCaches: make(map[string]*eventCache),
|
|
nsCache: make(map[string]*v1.Namespace),
|
|
nodeCache: make(map[string]*v1.Node),
|
|
metricsCache: make(map[string]*cachedMetrics),
|
|
stopCh: make(chan struct{}),
|
|
}
|
|
if err := mgr.setupInformers(); err != nil {
|
|
return fmt.Errorf("failed to setup informers: %v", err)
|
|
}
|
|
mirror = mgr
|
|
return nil
|
|
}
|
|
|
|
// startMirror starts the cache manager if it exists
|
|
func startMirror(lang string) error {
|
|
if mirror == nil {
|
|
return fmt.Errorf("cache manager not initialized")
|
|
}
|
|
if mirror.started {
|
|
return nil
|
|
}
|
|
if err := mirror.Start(lang); err != nil {
|
|
return fmt.Errorf("failed to start cache manager: %v", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// setupInformers configures all required informers
|
|
func (cm *cacheManager) setupInformers() error {
|
|
// Namespace informer
|
|
cm.nsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
|
|
AddFunc: cm.onNamespaceAdd,
|
|
UpdateFunc: cm.onNamespaceUpdate,
|
|
DeleteFunc: cm.onNamespaceDelete,
|
|
})
|
|
// Service informer for all namespaces
|
|
svcInformer := cm.factory.Core().V1().Services()
|
|
svcInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
|
|
AddFunc: cm.onServiceAdd,
|
|
UpdateFunc: cm.onServiceUpdate,
|
|
DeleteFunc: cm.onServiceDelete,
|
|
})
|
|
// Deployment informer for all namespaces
|
|
deployInformer := cm.factory.Apps().V1().Deployments()
|
|
deployInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
|
|
AddFunc: cm.onDeploymentAdd,
|
|
UpdateFunc: cm.onDeploymentUpdate,
|
|
DeleteFunc: cm.onDeploymentDelete,
|
|
})
|
|
// Node informer for external IP resolution
|
|
nodeInformer := cm.factory.Core().V1().Nodes()
|
|
nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
|
|
AddFunc: cm.onNodeAdd,
|
|
UpdateFunc: cm.onNodeUpdate,
|
|
DeleteFunc: cm.onNodeDelete,
|
|
})
|
|
// Ingress informer for all namespaces
|
|
ingressInformer := cm.factory.Networking().V1().Ingresses()
|
|
ingressInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
|
|
AddFunc: cm.onIngressAdd,
|
|
UpdateFunc: cm.onIngressUpdate,
|
|
DeleteFunc: cm.onIngressDelete,
|
|
})
|
|
// Event informer
|
|
eventInformer := cm.factory.Core().V1().Events()
|
|
eventInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
|
|
AddFunc: cm.onEventAdd,
|
|
UpdateFunc: cm.onEventUpdate,
|
|
DeleteFunc: cm.onEventDelete,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// Event-related event handlers
|
|
func (cm *cacheManager) onEventAdd(obj interface{}) {
|
|
event := obj.(*v1.Event)
|
|
cm.updateEventCache(event)
|
|
}
|
|
|
|
func (cm *cacheManager) onEventUpdate(oldObj, newObj interface{}) {
|
|
event := newObj.(*v1.Event)
|
|
cm.updateEventCache(event)
|
|
}
|
|
|
|
func (cm *cacheManager) onEventDelete(obj interface{}) {
|
|
event := obj.(*v1.Event)
|
|
cm.deleteEventCache(event)
|
|
}
|
|
|
|
// updateEventCache updates the event cache
|
|
func (cm *cacheManager) updateEventCache(event *v1.Event) {
|
|
cm.mu.Lock()
|
|
defer cm.mu.Unlock()
|
|
ns := event.Namespace
|
|
if cm.eventCaches[ns] == nil {
|
|
cm.eventCaches[ns] = &eventCache{
|
|
Namespace: ns,
|
|
Events: make(map[string]*v1.Event),
|
|
}
|
|
}
|
|
cm.eventCaches[ns].Events[event.Name] = event
|
|
}
|
|
|
|
// deleteEventCache deletes from event cache
|
|
func (cm *cacheManager) deleteEventCache(event *v1.Event) {
|
|
cm.mu.Lock()
|
|
defer cm.mu.Unlock()
|
|
if nsEventCache, exists := cm.eventCaches[event.Namespace]; exists {
|
|
delete(nsEventCache.Events, event.Name)
|
|
if len(nsEventCache.Events) == 0 {
|
|
delete(cm.eventCaches, event.Namespace)
|
|
}
|
|
}
|
|
}
|
|
|
|
// getEvents retrieves all events for the specified namespace
|
|
func (cm *cacheManager) getEvents(namespace string) []*v1.Event {
|
|
cm.mu.RLock()
|
|
defer cm.mu.RUnlock()
|
|
cm.eventHits++
|
|
var events []*v1.Event
|
|
if nsEventCache, exists := cm.eventCaches[namespace]; exists {
|
|
for _, event := range nsEventCache.Events {
|
|
events = append(events, event)
|
|
}
|
|
}
|
|
return events
|
|
}
|
|
|
|
// Start begins the cache manager
|
|
func (cm *cacheManager) Start(lang string) error {
|
|
if cm.started {
|
|
return nil
|
|
}
|
|
cm.factory.Start(cm.stopCh)
|
|
// Wait for initial sync
|
|
synced := cm.factory.WaitForCacheSync(cm.stopCh)
|
|
for informer, synced := range synced {
|
|
if !synced {
|
|
return fmt.Errorf("failed to sync informer: %v", informer)
|
|
}
|
|
}
|
|
if metricsClient != nil {
|
|
go cm.startMetricsPoller(60*time.Second, lang)
|
|
}
|
|
cm.started = true
|
|
return nil
|
|
}
|
|
|
|
// Stop stops the cache manager
|
|
func (cm *cacheManager) Stop() {
|
|
if cm.started {
|
|
close(cm.stopCh)
|
|
cm.started = false
|
|
}
|
|
}
|
|
|
|
func (cm *cacheManager) startMetricsPoller(interval time.Duration, lang string) {
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
cm.updateAllMetrics(lang)
|
|
for {
|
|
select {
|
|
case <-ticker.C:
|
|
cm.updateAllMetrics(lang)
|
|
case <-cm.stopCh:
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (cm *cacheManager) updateAllMetrics(lang string) {
|
|
cm.mu.RLock()
|
|
namespaces := make([]string, 0, len(cm.nsCache))
|
|
for name := range cm.nsCache {
|
|
namespaces = append(namespaces, name)
|
|
}
|
|
cm.mu.RUnlock()
|
|
for _, namespace := range namespaces {
|
|
cm.updateNamespaceMetrics(namespace, lang)
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// updateNamespaceMetrics updates cached metrics for a namespace
|
|
func (cm *cacheManager) updateNamespaceMetrics(namespace string, lang string) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
metrics, err := calculateNamespaceMetrics(ctx, metricsClient, namespace, cm.deployCache, &cm.mu, lang)
|
|
if err != nil {
|
|
return
|
|
}
|
|
cm.mu.Lock()
|
|
cm.metricsCache[namespace] = metrics
|
|
cm.mu.Unlock()
|
|
}
|
|
|
|
func (cm *cacheManager) getNamespaceMetricsFromCache(namespace string) (*cachedMetrics, bool) {
|
|
cm.mu.RLock()
|
|
defer cm.mu.RUnlock()
|
|
cm.metricsHits++
|
|
metrics, found := cm.metricsCache[namespace]
|
|
return metrics, found
|
|
}
|
|
|
|
// Namespace handlers
|
|
func (cm *cacheManager) onNamespaceAdd(obj interface{}) {
|
|
ns := obj.(*v1.Namespace)
|
|
cm.updateNamespaceCache(ns)
|
|
}
|
|
|
|
func (cm *cacheManager) onNamespaceUpdate(oldObj, newObj interface{}) {
|
|
ns := newObj.(*v1.Namespace)
|
|
cm.updateNamespaceCache(ns)
|
|
}
|
|
|
|
func (cm *cacheManager) onNamespaceDelete(obj interface{}) {
|
|
ns := obj.(*v1.Namespace)
|
|
cm.deleteNamespaceCache(ns)
|
|
}
|
|
|
|
func (cm *cacheManager) updateNamespaceCache(ns *v1.Namespace) {
|
|
cm.mu.Lock()
|
|
defer cm.mu.Unlock()
|
|
cm.nsCache[ns.Name] = ns
|
|
}
|
|
|
|
func (cm *cacheManager) deleteNamespaceCache(ns *v1.Namespace) {
|
|
cm.mu.Lock()
|
|
defer cm.mu.Unlock()
|
|
delete(cm.nsCache, ns.Name)
|
|
delete(cm.metricsCache, ns.Name)
|
|
delete(cm.eventCaches, ns.Name)
|
|
}
|
|
|
|
// Service event handlers
|
|
func (cm *cacheManager) onServiceAdd(obj interface{}) {
|
|
svc := obj.(*v1.Service)
|
|
cm.updateServiceCache(svc)
|
|
}
|
|
|
|
func (cm *cacheManager) onServiceUpdate(oldObj, newObj interface{}) {
|
|
svc := newObj.(*v1.Service)
|
|
cm.updateServiceCache(svc)
|
|
}
|
|
|
|
func (cm *cacheManager) onServiceDelete(obj interface{}) {
|
|
svc := obj.(*v1.Service)
|
|
cm.deleteServiceCache(svc)
|
|
}
|
|
|
|
func (cm *cacheManager) updateServiceCache(svc *v1.Service) {
|
|
cm.mu.Lock()
|
|
defer cm.mu.Unlock()
|
|
ns := svc.Namespace
|
|
if cm.svcCache[ns] == nil {
|
|
cm.svcCache[ns] = make(map[string]*v1.Service)
|
|
}
|
|
cm.svcCache[ns][svc.Name] = svc
|
|
}
|
|
|
|
func (cm *cacheManager) deleteServiceCache(svc *v1.Service) {
|
|
cm.mu.Lock()
|
|
defer cm.mu.Unlock()
|
|
if nsCache, exists := cm.svcCache[svc.Namespace]; exists {
|
|
delete(nsCache, svc.Name)
|
|
}
|
|
}
|
|
|
|
// Deployment event handlers
|
|
func (cm *cacheManager) onDeploymentAdd(obj interface{}) {
|
|
deploy := obj.(*appsv1.Deployment)
|
|
cm.updateDeploymentCache(deploy)
|
|
}
|
|
|
|
func (cm *cacheManager) onDeploymentUpdate(oldObj, newObj interface{}) {
|
|
deploy := newObj.(*appsv1.Deployment)
|
|
cm.updateDeploymentCache(deploy)
|
|
}
|
|
|
|
func (cm *cacheManager) onDeploymentDelete(obj interface{}) {
|
|
deploy := obj.(*appsv1.Deployment)
|
|
cm.deleteDeploymentCache(deploy)
|
|
}
|
|
|
|
func (cm *cacheManager) updateDeploymentCache(deploy *appsv1.Deployment) {
|
|
cm.mu.Lock()
|
|
defer cm.mu.Unlock()
|
|
ns := deploy.Namespace
|
|
if cm.deployCache[ns] == nil {
|
|
cm.deployCache[ns] = make(map[string]*appsv1.Deployment)
|
|
}
|
|
cm.deployCache[ns][deploy.Name] = deploy
|
|
}
|
|
|
|
func (cm *cacheManager) deleteDeploymentCache(deploy *appsv1.Deployment) {
|
|
cm.mu.Lock()
|
|
defer cm.mu.Unlock()
|
|
if nsCache, exists := cm.deployCache[deploy.Namespace]; exists {
|
|
delete(nsCache, deploy.Name)
|
|
}
|
|
}
|
|
|
|
// Node event handlers
|
|
func (cm *cacheManager) onNodeAdd(obj interface{}) {
|
|
node := obj.(*v1.Node)
|
|
cm.updateNodeCache(node)
|
|
}
|
|
|
|
func (cm *cacheManager) onNodeUpdate(oldObj, newObj interface{}) {
|
|
node := newObj.(*v1.Node)
|
|
cm.updateNodeCache(node)
|
|
}
|
|
|
|
func (cm *cacheManager) onNodeDelete(obj interface{}) {
|
|
node := obj.(*v1.Node)
|
|
cm.deleteNodeCache(node)
|
|
}
|
|
|
|
func (cm *cacheManager) updateNodeCache(node *v1.Node) {
|
|
cm.mu.Lock()
|
|
defer cm.mu.Unlock()
|
|
cm.nodeCache[node.Name] = node
|
|
}
|
|
|
|
func (cm *cacheManager) deleteNodeCache(node *v1.Node) {
|
|
cm.mu.Lock()
|
|
defer cm.mu.Unlock()
|
|
delete(cm.nodeCache, node.Name)
|
|
}
|
|
|
|
func (cm *cacheManager) onIngressAdd(obj interface{}) {
|
|
ingress := obj.(*networkingv1.Ingress)
|
|
cm.updateIngressCache(ingress)
|
|
}
|
|
|
|
func (cm *cacheManager) onIngressUpdate(oldObj, newObj interface{}) {
|
|
ingress := newObj.(*networkingv1.Ingress)
|
|
cm.updateIngressCache(ingress)
|
|
}
|
|
|
|
func (cm *cacheManager) onIngressDelete(obj interface{}) {
|
|
ingress := obj.(*networkingv1.Ingress)
|
|
cm.deleteIngressCache(ingress)
|
|
}
|
|
|
|
func (cm *cacheManager) updateIngressCache(ingress *networkingv1.Ingress) {
|
|
cm.mu.Lock()
|
|
defer cm.mu.Unlock()
|
|
ns := ingress.Namespace
|
|
if cm.ingressCache[ns] == nil {
|
|
cm.ingressCache[ns] = make(map[string]*networkingv1.Ingress)
|
|
}
|
|
cm.ingressCache[ns][ingress.Name] = ingress
|
|
}
|
|
|
|
func (cm *cacheManager) deleteIngressCache(ingress *networkingv1.Ingress) {
|
|
cm.mu.Lock()
|
|
defer cm.mu.Unlock()
|
|
if nsCache, exists := cm.ingressCache[ingress.Namespace]; exists {
|
|
delete(nsCache, ingress.Name)
|
|
}
|
|
}
|
|
|
|
func (cm *cacheManager) getNamespace(name string) *v1.Namespace {
|
|
cm.mu.RLock()
|
|
defer cm.mu.RUnlock()
|
|
cm.nsHits++
|
|
if ns, exists := cm.nsCache[name]; exists {
|
|
return ns
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Cache access methods with hit counting
|
|
func (cm *cacheManager) getServices(namespace string) []*v1.Service {
|
|
cm.mu.RLock()
|
|
defer cm.mu.RUnlock()
|
|
cm.svcHits++
|
|
var services []*v1.Service
|
|
if nsCache, exists := cm.svcCache[namespace]; exists {
|
|
for _, svc := range nsCache {
|
|
if svc.Name != "kubernetes" { // Skip default service
|
|
services = append(services, svc)
|
|
}
|
|
}
|
|
}
|
|
return services
|
|
}
|
|
|
|
func (cm *cacheManager) getDeployments(namespace string) []*appsv1.Deployment {
|
|
cm.mu.RLock()
|
|
defer cm.mu.RUnlock()
|
|
cm.deployHits++
|
|
var deployments []*appsv1.Deployment
|
|
if nsCache, exists := cm.deployCache[namespace]; exists {
|
|
for _, deploy := range nsCache {
|
|
deployments = append(deployments, deploy)
|
|
}
|
|
}
|
|
return deployments
|
|
}
|
|
|
|
func (cm *cacheManager) getNodes() []*v1.Node {
|
|
cm.mu.RLock()
|
|
defer cm.mu.RUnlock()
|
|
cm.nodeHits++
|
|
var nodes []*v1.Node
|
|
for _, node := range cm.nodeCache {
|
|
nodes = append(nodes, node)
|
|
}
|
|
return nodes
|
|
}
|
|
|
|
func (cm *cacheManager) getIngresses(namespace string) []*networkingv1.Ingress {
|
|
cm.mu.RLock()
|
|
defer cm.mu.RUnlock()
|
|
cm.ingressHits++
|
|
var ingresses []*networkingv1.Ingress
|
|
if nsCache, exists := cm.ingressCache[namespace]; exists {
|
|
for _, ingress := range nsCache {
|
|
ingresses = append(ingresses, ingress)
|
|
}
|
|
}
|
|
return ingresses
|
|
}
|