SundayHK

VictoriaLogs Cluster采集Kubernetes Pod日志

2026-01-27 14 0

VictoriaLogs 介绍

VictoriaLogs 是 VictoriaMetrics 团队推出的开源日志解决方案，旨在替代笨重的 Elasticsearch (ELK) 和架构复杂的 Loki

组件介绍

  • vlinsert 接收来自 Fluent-bit、Vector 或 Logstash 的日志数据
  • vlstorage 核心存储，将日志压缩并持久化到磁盘，响应查询请求。
  • vlselect 查询网关，处理Grafana或CLI的搜索请求。
  • vmauth 认证网关 - 可选，提供统一入口、负载均衡和身份验证。

部署VictoriaLogs Cluster

这里使用从 Kubernetes 中运行的所有容器收集日志，并将数据发送到已安装的集群版 VictoriaLogs。

vlstorage部署了两个实例，每个实例的保留期为180天，PVC 为200Gi。

Helm部署

helm repo add vm https://victoriametrics.github.io/helm-charts/
helm repo update
export RETENTION=180d
export PVC_SIZE=200Gi
export VLSTORAGE_REPLICAS=2
export NAMESPACE=logging

# Install victoria-logs-cluster chart
helm install vlc vm/victoria-logs-cluster --namespace $NAMESPACE --wait \
    --set "vlstorage.retentionPeriod=$RETENTION" \
    --set "vlstorage.persistentVolume.size=$PVC_SIZE" \
    --set vmauth.enabled=true \
    --set vlstorage.replicaCount=$VLSTORAGE_REPLICAS

# Install victoria-logs-collector chart
helm install collector vm/victoria-logs-collector --namespace $NAMESPACE \
    --set "remoteWrite[0].url=http://vlc-victoria-logs-cluster-vmauth:8427"

离线部署

export RETENTION=180d
export PVC_SIZE=200Gi
export VLSTORAGE_REPLICAS=2
export NAMESPACE=logging

# 基于本地tgz包安装
helm install vlc ./victoria-logs-cluster-0.0.20.tgz \
  --namespace $NAMESPACE --wait \
  --set "vlstorage.retentionPeriod=$RETENTION" \
  --set "vlstorage.persistentVolume.size=$PVC_SIZE" \
  --set vmauth.enabled=true \
  --set vlstorage.replicaCount=$VLSTORAGE_REPLICAS

# 或基于解压后的目录安装
helm install vlc ./victoria-logs-cluster \
  --namespace $NAMESPACE --wait \
  --set "vlstorage.retentionPeriod=$RETENTION" \
  --set "vlstorage.persistentVolume.size=$PVC_SIZE" \
  --set vmauth.enabled=true \
  --set vlstorage.replicaCount=$VLSTORAGE_REPLICAS

服务暴露，如果没有ingress，可以修改服务类型为NodePort

# 读取
kubectl patch svc vlc-victoria-logs-cluster-vlselect -n logging -p '{"spec": {"type": "NodePort"}}'

# 写入 (可选，暂时不调整)
kubectl patch svc vlc-victoria-logs-cluster-vlinsert -n logging -p '{"spec": {"type": "NodePort"}}'
root@arm-us1:~# kubectl get svc -n logging
NAME                                  TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
vlc-victoria-logs-cluster-vlinsert    NodePort    10.230.57.167   <none>        9481:30863/TCP   50m
vlc-victoria-logs-cluster-vlselect    NodePort    10.230.38.52    <none>        9471:31368/TCP   50m
vlc-victoria-logs-cluster-vlstorage   ClusterIP   None            <none>        9491/TCP         50m
vlc-victoria-logs-cluster-vmauth      ClusterIP   10.230.15.166   <none>        8427/TCP         50m

fluent-bit 采集Kubernetes日志

configMap.yaml

  • logging 部署命名空间
  • 采集Pod日志/var/log/containers/*.log路径
  • 写入日志地址 vlc-victoria-logs-cluster-vlinsert
  • 写入日志端口9481
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  namespace: logging
  labels:
    app.kubernetes.io/name: fluent-bit
data:
  fluent-bit.conf: |
    [SERVICE]
        Flush        1
        Log_Level    info
        HTTP_Server  On
        HTTP_Listen  0.0.0.0
        HTTP_Port    2020
        Health_Check On
    [INPUT]
        Name            tail
        Tag             kubernetes.*
        Path            /var/log/containers/*.log
        # --- 关键修改在这里 ---
        Exclude_Path    /var/log/containers/fluent-bit-*.log
        Parser          cri
        DB              /var/log/flb_kube.db
        Mem_Buf_Limit   5MB
        Skip_Long_Lines On
        Refresh_Interval 10
    [FILTER]
        Name            kubernetes
        Match           kubernetes.*
        Kube_URL        https://kubernetes.default.svc:443
        Kube_CA_File    /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix kubernetes.var.log.containers.
        Merge_Log       On
        Merge_Log_Key   log_processed
        K8S-Logging.Parser On
        K8S-Logging.Exclude Off
    [OUTPUT]
        Name            http
        Match           *
        Host            vlc-victoria-logs-cluster-vlinsert
        Port            9481
        URI             /insert/jsonline?_msg_field=log&_stream_fields=kubernetes.namespace_name,kubernetes.pod_name,kubernetes.container_name,stream&_time_field=time
        Format          json_lines
        Json_date_key   time
        Json_date_format iso8601
        Header          X-Scope-OrgID 1
        tls             Off
        tls.verify      Off
  parsers.conf: |
    [PARSER]
        Name   cri
        Format regex
        Regex  ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
        Time_Key    time

rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluent-bit
  namespace: logging # 命名空间
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluent-bit-read
rules:
- apiGroups: [""]
  resources:
  - namespaces
  - pods
  - pods/logs
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: fluent-bit-read
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: fluent-bit-read
subjects:
- kind: ServiceAccount
  name: fluent-bit
  namespace: logging # 命名空间

daemonSet.yaml

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluent-bit
  namespace: logging # 命名空间
  labels:
    app.kubernetes.io/name: fluent-bit
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: fluent-bit
  template:
    metadata:
      labels:
        app.kubernetes.io/name: fluent-bit
    spec:
      containers:
      - name: fluent-bit
        image: fluent/fluent-bit:3.0.1
        imagePullPolicy: Always
        ports:
        - containerPort: 2020
        env:
        - name: FLB_PROCESSOR
          value: "OFF"
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: fluent-bit-config
          mountPath: /fluent-bit/etc/
        - name: etcmachineid
          mountPath: /etc/machine-id
          readOnly: true
        resources:
          limits:
            memory: 100Mi
          requests:
            cpu: 10m
            memory: 20Mi
        securityContext:
          runAsUser: 0
          privileged: true
      serviceAccountName: fluent-bit
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: fluent-bit-config
        configMap:
          name: fluent-bit-config
      - name: etcmachineid
        hostPath:
          path: /etc/machine-id
          type: File
      tolerations:
      - key: node-role.kubernetes.io/master
        operator: Exists
        effect: NoSchedule
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule
root@arm-us1:/data/kube_yaml/logging# kubectl get pod -n logging | grep fluent-bit
fluent-bit-2ths8                                      1/1     Running   0          50m
fluent-bit-s979h                                      1/1     Running   0          50m
root@arm-us1:~# kubectl get svc -n logging | grep vlselect
vlc-victoria-logs-cluster-vlselect    NodePort    10.230.38.52    <none>        9471:31368/TCP   76m

image.png

image.png

prometheus监控

https://grafana.com/orgs/victoriametrics/dashboards
https://grafana.com/grafana/dashboards/23274-victorialogs-cluster/

serviceMonitor

vlc-servicemonitor.yaml

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: victoria-logs-cluster-monitor
  # 建议放在监控组件所在的命名空间
  namespace: kube-prometheus
  labels:
    # 关键：必须包含这个标签，Prometheus 才会发现它
    app.kubernetes.io/instance: prometheus
spec:
  # 匹配 VictoriaLogs 的 Service
  selector:
    matchLabels:
      app.kubernetes.io/instance: vlc
      app.kubernetes.io/name: victoria-logs-cluster
  # 允许跨命名空间抓取 logging 下的指标
  namespaceSelector:
    matchNames:
      - logging
  endpoints:
  - port: http
    path: /metrics
    interval: 30s
    # 自动将不同的 Service Name 映射为 component 标签，方便 Grafana 区分
    relabelings:
    - sourceLabels: [__meta_kubernetes_service_name]
      targetLabel: component
$ kubectl get servicemonitor -n kube-prometheus | grep victoria-logs-cluster-monitor
victoria-logs-cluster-monitor                        17h

image.png

grafana

victoria-logs-cluster dashboard

导入ID： 23274

若使用Prometheus，Component versions ($job) 则会没有数据，因为label_set是MetricsQL方法，原生 Prometheus 没有这个函数，需调整为 label_replace

sum by (shortVersion, tier) (
  # 1. 处理 select + insert 混合节点
  label_replace(
    label_replace(
      (vm_app_version{job=~"$job", instance=~"$instance_select_all"} and on(instance, version) vm_app_version{job=~"$job", instance=~"$instance_insert_all"}),
      "tier", "select,insert", "", ""
    ),
    "shortVersion", "$1", "version", ".*tags-(v[0-9]+\\.[0-9]+\\.[0-9]+).*"
  )
  or
  # 2. 纯 select 节点
  label_replace(
    label_replace(
      (vm_app_version{job=~"$job", instance=~"$instance_select_all"} unless on(instance, version) vm_app_version{job=~"$job", instance=~"$instance_insert_all"}),
      "tier", "select", "", ""
    ),
    "shortVersion", "$1", "version", ".*tags-(v[0-9]+\\.[0-9]+\\.[0-9]+).*"
  )
  or
  # 3. 纯 insert 节点
  label_replace(
    label_replace(
      (vm_app_version{job=~"$job", instance=~"$instance_insert_all"} unless on(instance, version) vm_app_version{job=~"$job", instance=~"$instance_select_all"}),
      "tier", "insert", "", ""
    ),
    "shortVersion", "$1", "version", ".*tags-(v[0-9]+\\.[0-9]+\\.[0-9]+).*"
  )
  or
  # 4. storage 节点
  label_replace(
    label_replace(
      vm_app_version{job=~"$job", instance=~"$instance_storage_all"},
      "tier", "storage", "", ""
    ),
    "shortVersion", "$1", "version", ".*tags-(v[0-9]+\\.[0-9]+\\.[0-9]+).*"
  )
  or
  # 5. 其他未知节点 (Unknown)
  label_replace(
    label_replace(
      (vm_app_version{job=~"$job", instance=~"$instance"} unless on(instance, version) (vm_app_version{job=~"$job", instance=~"$instance_select_all"} or vm_app_version{job=~"$job", instance=~"$instance_insert_all"} or vm_app_version{job=~"$job", instance=~"$instance_storage_all"})),
      "tier", "unknown", "", ""
    ),
    "shortVersion", "$1", "version", ".*tags-(v[0-9]+\\.[0-9]+\\.[0-9]+).*"
  )
)

image.png

grafana victoriametrics-log插件

支持对victoriametrics-logs数据查询及图表展示

grafana cli plugins install victoriametrics-logs-datasource

image.png

URL: http://vlc-victoria-logs-cluster-vlselect.logging.svc.cluster.local:9471

{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "grafana",
          "uid": "-- Grafana --"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 0,
  "id": null,
  "links": [],
  "liveNow": true,
  "panels": [
    {
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "${DS_VICTORIALOGS}"
      },
      "description": "显示当前筛选条件下的日志流量，用于确认是否有数据",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "bars",
            "fillOpacity": 80,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "normal"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 1,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "${DS_VICTORIALOGS}"
          },
          "expr": "_stream:{kubernetes.namespace_name=~\"$namespace\", kubernetes.pod_name=~\"$pod\"} AND $search | stats by (_time:1m) count()",
          "refId": "A"
        }
      ],
      "title": "🟢 实时流量脉冲 (Live Volume)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "${DS_VICTORIALOGS}"
      },
      "gridPos": {
        "h": 21,
        "w": 24,
        "x": 0,
        "y": 3
      },
      "id": 2,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": false,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": false
      },
      "targets": [
        {
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "${DS_VICTORIALOGS}"
          },
          "expr": "_stream:{kubernetes.namespace_name=~\"$namespace\", kubernetes.pod_name=~\"$pod\"} AND $search",
          "refId": "A"
        }
      ],
      "title": "📜 实时日志流 (Real-time Logs)",
      "type": "logs"
    }
  ],
  "refresh": "5s",
  "schemaVersion": 38,
  "style": "dark",
  "tags": [
    "realtime",
    "logs",
    "simple"
  ],
  "templating": {
    "list": [
      {
        "current": {
          "selected": true,
          "text": ".*",
          "value": ".*"
        },
        "description": "输入 .* 查看所有，或输入具体 Namespace",
        "hide": 0,
        "label": "Namespace",
        "name": "namespace",
        "options": [
          {
            "selected": true,
            "text": ".*",
            "value": ".*"
          }
        ],
        "query": ".*",
        "skipUrlSync": false,
        "type": "textbox"
      },
      {
        "current": {
          "selected": true,
          "text": ".*",
          "value": ".*"
        },
        "description": "输入 .* 查看所有，或输入 Pod 关键词",
        "hide": 0,
        "label": "Pod Name",
        "name": "pod",
        "options": [
          {
            "selected": true,
            "text": ".*",
            "value": ".*"
          }
        ],
        "query": ".*",
        "skipUrlSync": false,
        "type": "textbox"
      },
      {
        "current": {
          "selected": false,
          "text": "*",
          "value": "*"
        },
        "description": "全文搜索，输入 * 查看所有",
        "hide": 0,
        "label": "🔍 Search",
        "name": "search",
        "options": [
          {
            "selected": true,
            "text": "*",
            "value": "*"
          }
        ],
        "query": "*",
        "skipUrlSync": false,
        "type": "textbox"
      }
    ]
  },
  "time": {
    "from": "now-15m",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "1s",
      "5s",
      "10s",
      "30s",
      "1m"
    ]
  },
  "timezone": "",
  "title": "Pure Live Logs (Real-time)",
  "uid": null,
  "version": 0,
  "weekStart": "",
  "__inputs": [
    {
      "name": "DS_VICTORIALOGS",
      "label": "VictoriaLogs Data Source",
      "description": "Select your VictoriaMetrics Logs datasource",
      "type": "datasource",
      "pluginId": "victoriametrics-logs-datasource",
      "pluginName": "VictoriaMetrics Logs"
    }
  ]
}

https://i4t.com/25770.html

sunday
Grafana, Kubernetes, Prometheus
0 0

相关文章

发布评论

sunday

文章181 评论0