version: '3.7'

services:
  victoriametrics:
    image: victoriametrics/victoria-metrics:v1.75.0
    command:
      - '--retentionPeriod=2'
    volumes:
    - ${PRJPATH-/data/docker/victoriametrics}/victoria-metrics-data:/victoria-metrics-data:rw
    ports:
      - 8428:8428
    restart: on-failure:3
    deploy:
      replicas: 1
      resources:
        limits:
          memory: 2g
      restart_policy:
        condition: any
    healthcheck:
      test: wget -nv --spider http://localhost:8428 > /dev/null || exit 1
      interval: 60s
      timeout: 5s
      retries: 3
      start_period: 15s

https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-k8s-stack/values.yaml

helm upgrade --install vm-stack vm/victoria-metrics-k8s-stack \
--namespace monitoring --create-namespace \
--version 0.91.2 \
--set vmsingle.enabled=false \
--set vmcluster.enabled=false \
--set vmalert.enabled=false \
--set alertmanager.enabled=false \
--set grafana.enabled=false \
--set vmagent.enabled=true \
--set vmagent.spec.scrapeInterval=30s \
--set vmagent.spec.externalLabels.cluster=$(kubectl config current-context) \
--set vmagent.spec.statefulMode=true \
--set vmagent.spec.statefulStorage.volumeClaimTemplate.spec.resources.requests.storage=5Gi \
--set-json vmagent.spec.remoteWrite='[{
  "url":"http://vmauth.monitoring.svc:8427/api/v1/write",
  "inlineUrlRelabelConfig":[
    {"action":"labeldrop","regex":"beta_kubernetes_io.*"},
    {"action":"labeldrop","regex":"kubernetes_azure_com.*"},
    {"action":"labeldrop","regex":"failure_domain_beta_kubernetes_io.*"},
    {"action":"labeldrop","regex":"node_kubernetes_io_instance_type"},
    {"action":"labeldrop","regex":"storageprofile"},
    {"action":"labeldrop","regex":"storagetier"},
    {"action":"labeldrop","regex":"agentpool"},
    {"action":"labeldrop","regex":"topology_kubernetes_io.*"},
    {"action":"labeldrop","regex":"feature_node_kubernetes_io.*"},
    {"action":"labeldrop","regex":"nvidia_com.*"}
  ]
}]' \
--set vmagent.spec.extraArgs.loggerFormat=json \
--set vmagent.spec.resources.requests.cpu=100m --set vmagent.spec.resources.requests.memory=256Mi \
--set vmagent.spec.resources.limits.cpu=500m --set vmagent.spec.resources.limits.memory=512Mi \
--set-json vmagent.spec.tolerations='[{"key":"sku","operator":"Equal","value":"monitoring","effect":"NoSchedule"}]' \
--set prometheus-node-exporter.resources.requests.cpu=50m --set prometheus-node-exporter.resources.requests.memory=32Mi \
--set prometheus-node-exporter.resources.limits.cpu=200m --set prometheus-node-exporter.resources.limits.memory=64Mi \
--set-json prometheus-node-exporter.tolerations='[{"operator":"Exists","effect":"NoSchedule"}]' \
--set kube-state-metrics.resources.requests.cpu=50m --set kube-state-metrics.resources.requests.memory=64Mi \
--set kube-state-metrics.resources.limits.cpu=200m --set kube-state-metrics.resources.limits.memory=128Mi \
--set-json kube-state-metrics.tolerations='[{"key":"sku","operator":"Equal","value":"monitoring","effect":"NoSchedule"}]' \
--set victoria-metrics-operator.crds.upgrade.enabled=true \
--set victoria-metrics-operator.crds.upgrade.forceConflicts=true \
--set victoria-metrics-operator.resources.requests.cpu=50m --set victoria-metrics-operator.resources.requests.memory=64Mi \
--set victoria-metrics-operator.resources.limits.cpu=200m --set victoria-metrics-operator.resources.limits.memory=256Mi \
--set-json victoria-metrics-operator.tolerations='[{"key":"sku","operator":"Equal","value":"monitoring","effect":"NoSchedule"}]' \
--atomic --timeout 2m0s

Debug
Проверить версию victoria-metrics-operator в чарте:

helm show chart vm/victoria-metrics-k8s-stack --version ${VMSTACK_CHART_VERSION} | grep -A4 "victoria-metrics-operator"

Проверить доступные параметры апгрэйда:

helm show values vm/victoria-metrics-operator --version ${VMOPERATOR_CHART_VERSION} | grep -A8 "  upgrade:"

Создать rules из чарта:

helm template vm-stack vm/victoria-metrics-k8s-stack   --version 0.79.0   --set vmalert.enabled=true   --namespace monitoring | yq e 'select(.kind == "VMRule")' | yq e '.spec' - > k8s-rules.yaml

Проверить:

cat k8s-rules.yaml | yq e '.groups[].name'

Пример вывода:

alertmanager.rules
---
etcd
---
general.rules
---
k8s.rules.container_cpu_limits
---
k8s.rules.container_cpu_requests
---
k8s.rules.container_cpu_usage_seconds_total
---
k8s.rules.container_memory_cache
---
k8s.rules.container_memory_limits
---
k8s.rules.container_memory_requests
---
k8s.rules.container_memory_rss
---
k8s.rules.container_memory_swap
---
k8s.rules.container_memory_working_set_bytes
---
k8s.rules.pod_owner
---
kube-apiserver-availability.rules
---
kube-apiserver-burnrate.rules
---
kube-apiserver-histogram.rules
---
kube-apiserver-slos
---
kube-prometheus-node-recording.rules
---
kube-scheduler.rules
---
kube-state-metrics
---
kubelet.rules
---
kubernetes-apps
---
kubernetes-resources
---
kubernetes-storage
---
kubernetes-system-apiserver
---
kubernetes-system-controller-manager
---
kubernetes-system-kubelet
---
kubernetes-system-scheduler
---
kubernetes-system
---
node-exporter.rules
---
node-exporter
---
node-network
---
node.rules
---
vm-health
---
vmagent
---
vmalert
---
vmcluster
---
vmoperator
---
vmsingle

services:
  vmalert-infrastructure:
    image: victoriametrics/vmalert:${TAG-latest}
    command:
      - -rule=/rules/k8s-rules.yaml
      - -datasource.url=${VM_URL-http://victoriametrics:8428}?extra_label=cluster=Infrastructure
      - -remoteWrite.url=${VM_URL-http://victoriametrics:8428}
      - -remoteRead.url=${VM_URL-http://victoriametrics:8428}
      - -evaluationInterval=30s
      - -external.label=cluster=Infrastructure
      - -notifier.blackhole
      - -loggerFormat=json
    volumes:
      - ./k8s-rules.yaml:/rules/k8s-rules.yaml

Пробросить порт vmagent:

kubectl --context=${CLUSTER} -n monitoring port-forward svc/vmagent-vm-stack-victoria-metrics-k8s-stack 8429:8429

Показать кол-во лэблов после drop правил:

curl -s "http://localhost:8429/api/v1/targets" | jq '[.data.activeTargets[] | select(.labels.job == "kubelet") | "\(.labels.node): \(.labels | with_entries(select(.key | test("beta_kubernetes_io|kubernetes_azure_com|failure_domain_beta_kubernetes_io|node_kubernetes_io_instance_type|storageprofile|storagetier|agentpool|topology_kubernetes_io|feature_node_kubernetes_io") | not)) | keys | length)"].[]'

Показать максимальное кол-во лэйблов в кластере до drop:

curl -s "http://localhost:8429/api/v1/targets" | jq '.data.activeTargets[] | .labels | keys | length' | sort -rn | head -1