mirror of
https://github.com/hanzoai/metrics.git
synced 2026-10-30 20:03:39 +00:00
Pablo (Tomas) FernandezandGitHub
1dee3d5e16
### Describe Your Changes - Add an introduction with a brief explanation of the operator and its benefits as an intro - Make some steps more explicit, instead of just linking to the VM cluster guide - Separate config/chart values files from kubectl apply (instead of using heredoc and in-line yaml) - Update screenshots and add figcaptions where needed - Update Kubernetes and tools versions to newer releases - Remove revision numbers from the Grafana config to install the latest revision - Added a section to configure scraping of Kubernetes resources (nodes, pods, etc.) - Tested updated instructions on GKE 1.33 and 1.34 (and a local k3s instance) successfully - Added and updated expected outputs. Some were missing and others were outdated - Updated Grafana dashboards screenshots since they changed from the last revision - Minor corrections and typo fixes. Improved flow - Added a section at the end pointing readers to where they can go next. ### Checklist The following checks are **mandatory**: - [X] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist). - [X] My change adheres to [VictoriaMetrics development goals](https://docs.victoriametrics.com/victoriametrics/goals/).
244 lines
6.7 KiB
Plaintext
Executable File
244 lines
6.7 KiB
Plaintext
Executable File
# vmnodescrape cadvisor
|
|
apiVersion: operator.victoriametrics.com/v1beta1
|
|
kind: VMNodeScrape
|
|
metadata:
|
|
name: kubelet-cadvisor
|
|
spec:
|
|
scheme: https
|
|
path: /metrics/cadvisor
|
|
tlsConfig:
|
|
insecureSkipVerify: true
|
|
caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
|
|
# Override job label
|
|
relabelConfigs:
|
|
- targetLabel: job
|
|
replacement: "kubernetes-nodes-cadvisor"
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_node_label_(.+)
|
|
|
|
# relabel metrics to work with Grafana dashboard
|
|
metricRelabelConfigs:
|
|
- action: replace
|
|
sourceLabels: [pod]
|
|
regex: "(.+)"
|
|
targetLabel: pod_name
|
|
replacement: "$1"
|
|
- action: replace
|
|
sourceLabels: [container]
|
|
regex: "(.+)"
|
|
targetLabel: container_name
|
|
replacement: "$1"
|
|
- action: replace
|
|
targetLabel: name
|
|
replacement: "k8s_stub"
|
|
- action: replace
|
|
sourceLabels: [id]
|
|
regex: "^/system\\.slice/(.+)\\.service$"
|
|
targetLabel: systemd_service_name
|
|
replacement: "$1"
|
|
|
|
---
|
|
|
|
# vmnodescrape kubelet
|
|
apiVersion: operator.victoriametrics.com/v1beta1
|
|
kind: VMNodeScrape
|
|
metadata:
|
|
name: kubelet-metrics
|
|
spec:
|
|
scheme: https
|
|
tlsConfig:
|
|
insecureSkipVerify: true
|
|
caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
|
|
# relabel metrics to work with Grafana dashboard
|
|
relabelConfigs:
|
|
- targetLabel: job
|
|
replacement: "kubernetes-nodes"
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_node_label_(.+)
|
|
|
|
---
|
|
|
|
# vmscrapeconfig apiservers
|
|
apiVersion: operator.victoriametrics.com/v1beta1
|
|
kind: VMScrapeConfig
|
|
metadata:
|
|
name: kubernetes-apiservers
|
|
spec:
|
|
kubernetesSDConfigs:
|
|
- role: endpoints
|
|
|
|
scheme: https
|
|
tlsConfig:
|
|
caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
insecureSkipVerify: true
|
|
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
|
|
relabelConfigs:
|
|
# Keep only the kubernetes service on https port
|
|
- sourceLabels:
|
|
- __meta_kubernetes_namespace
|
|
- __meta_kubernetes_service_name
|
|
- __meta_kubernetes_endpoint_port_name
|
|
action: keep
|
|
regex: "default;kubernetes;https"
|
|
|
|
# relabel metrics to work with Grafana dashboard
|
|
- targetLabel: job
|
|
replacement: "kubernetes-apiservers"
|
|
|
|
---
|
|
|
|
# vmscrapeconfig pods
|
|
apiVersion: operator.victoriametrics.com/v1beta1
|
|
kind: VMScrapeConfig
|
|
metadata:
|
|
name: kubernetes-pods
|
|
spec:
|
|
kubernetesSDConfigs:
|
|
- role: pod
|
|
|
|
relabelConfigs:
|
|
# Skip init containers
|
|
- action: drop
|
|
sourceLabels: [__meta_kubernetes_pod_container_init]
|
|
regex: "true"
|
|
|
|
# Ensure port annotation matches container port
|
|
- action: keep_if_equal
|
|
sourceLabels:
|
|
- __meta_kubernetes_pod_annotation_prometheus_io_port
|
|
- __meta_kubernetes_pod_container_port_number
|
|
|
|
# Only pods with prometheus.io/scrape="true"
|
|
- sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
|
action: keep
|
|
regex: "true"
|
|
|
|
# Exclude pods marked as "slow"
|
|
- sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow]
|
|
action: drop
|
|
regex: "true"
|
|
|
|
# Scheme override
|
|
- sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
|
|
action: replace
|
|
targetLabel: __scheme__
|
|
regex: "(https?)"
|
|
|
|
# Path override
|
|
- sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
|
action: replace
|
|
targetLabel: __metrics_path__
|
|
regex: "(.+)"
|
|
|
|
# Address override using prometheus.io/port
|
|
- sourceLabels:
|
|
- __address__
|
|
- __meta_kubernetes_pod_annotation_prometheus_io_port
|
|
action: replace
|
|
targetLabel: __address__
|
|
regex: "([^:]+)(?::\\d+)?;(\\d+)"
|
|
replacement: "$1:$2"
|
|
|
|
# Copy pod labels
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_pod_label_(.+)
|
|
|
|
# Use kubernetes_namespace, kubernetes_pod_name, kubernetes_node labels
|
|
- sourceLabels: [__meta_kubernetes_namespace]
|
|
action: replace
|
|
targetLabel: kubernetes_namespace
|
|
- sourceLabels: [__meta_kubernetes_pod_name]
|
|
action: replace
|
|
targetLabel: kubernetes_pod_name
|
|
- sourceLabels: [__meta_kubernetes_pod_node_name]
|
|
action: replace
|
|
targetLabel: kubernetes_node
|
|
|
|
# Drop non-running pods
|
|
- sourceLabels: [__meta_kubernetes_pod_phase]
|
|
action: drop
|
|
regex: "Pending|Succeeded|Failed|Completed"
|
|
|
|
# Override job label
|
|
- targetLabel: job
|
|
replacement: "kubernetes-pods"
|
|
|
|
---
|
|
|
|
# vmscrapeconfig service endpoints
|
|
apiVersion: operator.victoriametrics.com/v1beta1
|
|
kind: VMScrapeConfig
|
|
metadata:
|
|
name: kubernetes-service-endpoints
|
|
spec:
|
|
kubernetesSDConfigs:
|
|
- role: endpoints
|
|
|
|
relabelConfigs:
|
|
# Skip init containers
|
|
- action: drop
|
|
sourceLabels: [__meta_kubernetes_pod_container_init]
|
|
regex: "true"
|
|
|
|
# Ensure port annotation matches container port
|
|
- action: keep_if_equal
|
|
sourceLabels:
|
|
- __meta_kubernetes_pod_annotation_prometheus_io_port
|
|
- __meta_kubernetes_pod_container_port_number
|
|
|
|
# Only services with prometheus.io/scrape="true"
|
|
- sourceLabels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
|
|
action: keep
|
|
regex: "true"
|
|
|
|
# Exclude "slow" services
|
|
- sourceLabels: [__meta_kubernetes_service_annotation_prometheus_io_scrape_slow]
|
|
action: drop
|
|
regex: "true"
|
|
|
|
# Scheme override
|
|
- sourceLabels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
|
|
action: replace
|
|
targetLabel: __scheme__
|
|
regex: "(https?)"
|
|
|
|
# Path override
|
|
- sourceLabels: [__meta_kubernetes_service_annotation_prometheus_io_path]
|
|
action: replace
|
|
targetLabel: __metrics_path__
|
|
regex: "(.+)"
|
|
|
|
# Address override using prometheus.io/port
|
|
- sourceLabels:
|
|
- __address__
|
|
- __meta_kubernetes_service_annotation_prometheus_io_port
|
|
action: replace
|
|
targetLabel: __address__
|
|
regex: "([^:]+)(?::\\d+)?;(\\d+)"
|
|
replacement: "$1:$2"
|
|
|
|
# Copy service labels
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_service_label_(.+)
|
|
|
|
# relabel metrics to work with Grafana dashboard
|
|
- sourceLabels: [__meta_kubernetes_namespace]
|
|
action: replace
|
|
targetLabel: kubernetes_namespace
|
|
- sourceLabels: [__meta_kubernetes_service_name]
|
|
action: replace
|
|
targetLabel: kubernetes_name
|
|
- sourceLabels: [__meta_kubernetes_pod_node_name]
|
|
action: replace
|
|
targetLabel: kubernetes_node
|
|
|
|
# Override job label
|
|
- targetLabel: job
|
|
replacement: "kubernetes-service-endpoints"
|
|
|