feat(vector): setup new log aggregator

kubernetes/talos-flux/apps/observability/vector/agent/config/vector.yaml

@@ -0,0 +1,36 @@
+---
+data_dir: /vector-data-dir
+
+sources:
+  journald_source:
+    type: journald
+    journal_directory: /var/log/journal
+
+  kubernetes_source:
+    type: kubernetes_logs
+    use_apiserver_cache: true
+    pod_annotation_fields:
+      container_image: container_image
+      container_name: container_name
+      pod_labels: pod_labels
+      pod_name: pod_name
+      pod_annotations: ""
+    namespace_annotation_fields:
+      namespace_labels: ""
+    node_annotation_fields:
+      node_labels: ""
+
+sinks:
+  journald:
+    type: vector
+    compression: true
+    version: "2"
+    address: vector-aggregator.observability.svc.cluster.local:6000
+    inputs: ["journald_source"]
+
+  kubernetes:
+    type: vector
+    compression: true
+    version: "2"
+    address: vector-aggregator.observability.svc.cluster.local:6010
+    inputs: ["kubernetes_source"]

kubernetes/talos-flux/apps/observability/vector/agent/helm-release.yaml

@@ -0,0 +1,121 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: &app vector-agent
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: app-template
+      version: 3.4.0
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s-charts
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  values:
+    controllers:
+      vector-agent:
+        type: daemonset
+        strategy: RollingUpdate
+        annotations:
+          reloader.stakater.com/auto: "true"
+        # this is not required in my setup as master nodes are not tainted
+        pod:
+          tolerations:
+            - key: node-role.kubernetes.io/master
+              effect: NoSchedule
+        containers:
+          app:
+            image:
+              repository: docker.io/timberio/vector
+              tag: 0.40.1-debian@sha256:881a3728091fab7a39dade6ca70f1e76e14e0a308bdb57a00009dc3d0b54e74d
+            args:
+              - --config
+              - /etc/vector/vector.yaml
+            env:
+              PROCFS_ROOT: /host/proc
+              SYSFS_ROOT: /host/sys
+              VECTOR_SELF_NODE_NAME:
+                valueFrom:
+                  fieldRef:
+                    apiVersion: v1
+                    fieldPath: spec.nodeName
+              VECTOR_SELF_POD_NAME:
+                valueFrom:
+                  fieldRef:
+                    apiVersion: v1
+                    fieldPath: metadata.name
+              VECTOR_SELF_POD_NAMESPACE:
+                valueFrom:
+                  fieldRef:
+                    apiVersion: v1
+                    fieldPath: metadata.namespace
+            securityContext:
+              privileged: true
+            probes:
+              liveness:
+                enabled: true
+              readiness:
+                enabled: true
+              startup:
+                enabled: true
+                spec:
+                  failureThreshold: 30
+                  periodSeconds: 5
+            resources:
+              requests:
+                cpu: 23m
+                memory: 249M
+    serviceAccount:
+      create: true
+      name: *app
+    persistence:
+      config:
+        type: configMap
+        name: vector-agent-configmap
+        globalMounts:
+          - path: /etc/vector/vector.yaml
+            subPath: vector.yaml
+            readOnly: true
+      data:
+        type: emptyDir
+        globalMounts:
+          - path: /vector-data-dir
+      var-log:
+        type: hostPath
+        hostPath: /var/log
+        hostPathType: Directory
+        globalMounts:
+          - path: /var/log
+            readOnly: true
+      var-lib:
+        type: hostPath
+        hostPath: /var/lib
+        hostPathType: Directory
+        globalMounts:
+          - path: /var/lib
+            readOnly: true
+      procfs:
+        type: hostPath
+        hostPath: /proc
+        hostPathType: Directory
+        globalMounts:
+          - path: /host/proc
+            readOnly: true
+      sysfs:
+        type: hostPath
+        hostPath: /sys
+        hostPathType: Directory
+        globalMounts:
+          - path: /host/sys
+            readOnly: true

kubernetes/talos-flux/apps/observability/vector/agent/kustomization.yaml

@@ -0,0 +1,13 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helm-release.yaml
+  - ./rbac.yaml
+configMapGenerator:
+  - name: vector-agent-configmap
+    files:
+      - vector.yaml=./config/vector.yaml
+generatorOptions:
+  disableNameSuffixHash: true

kubernetes/talos-flux/apps/observability/vector/agent/rbac.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: vector-agent
+rules:
+  - apiGroups: [""]
+    resources: ["namespaces", "nodes", "pods"]
+    verbs: ["list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: vector-agent
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: vector-agent
+subjects:
+  - kind: ServiceAccount
+    name: vector-agent
+    namespace: observability

kubernetes/talos-flux/apps/observability/vector/flux-sync.yaml

@@ -0,0 +1,24 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &appname vector-agent
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: vector-aggregator
+  targetNamespace: observability
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: vector
+      app.kubernetes.io/instance: *appname
+  interval: 10m
+  path: ./kubernetes/talos-flux/apps/observability/vector/agent
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops
+  wait: true
+  retryInterval: 1m
+  timeout: 5m