Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(vector): setup new log aggregator #3554

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
---
data_dir: /vector-data-dir

sources:
journald_source:
type: journald
journal_directory: /var/log/journal

kubernetes_source:
type: kubernetes_logs
use_apiserver_cache: true
pod_annotation_fields:
container_image: container_image
container_name: container_name
pod_labels: pod_labels
pod_name: pod_name
pod_annotations: ""
namespace_annotation_fields:
namespace_labels: ""
node_annotation_fields:
node_labels: ""

sinks:
journald:
type: vector
compression: true
version: "2"
address: vector-aggregator.observability.svc.cluster.local:6000
inputs: ["journald_source"]

kubernetes:
type: vector
compression: true
version: "2"
address: vector-aggregator.observability.svc.cluster.local:6010
inputs: ["kubernetes_source"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
---
# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: &app vector-agent
spec:
interval: 30m
chart:
spec:
chart: app-template
version: 3.4.0
sourceRef:
kind: HelmRepository
name: bjw-s-charts
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
strategy: rollback
retries: 3
values:
controllers:
vector-agent:
type: daemonset
strategy: RollingUpdate
annotations:
reloader.stakater.com/auto: "true"
# this is not required in my setup as master nodes are not tainted
pod:
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
containers:
app:
image:
repository: docker.io/timberio/vector
tag: 0.40.1-debian@sha256:881a3728091fab7a39dade6ca70f1e76e14e0a308bdb57a00009dc3d0b54e74d
args:
- --config
- /etc/vector/vector.yaml
env:
PROCFS_ROOT: /host/proc
SYSFS_ROOT: /host/sys
VECTOR_SELF_NODE_NAME:
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
VECTOR_SELF_POD_NAME:
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
VECTOR_SELF_POD_NAMESPACE:
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
securityContext:
privileged: true
probes:
liveness:
enabled: true
readiness:
enabled: true
startup:
enabled: true
spec:
failureThreshold: 30
periodSeconds: 5
resources:
requests:
cpu: 23m
memory: 249M
serviceAccount:
create: true
name: *app
persistence:
config:
type: configMap
name: vector-agent-configmap
globalMounts:
- path: /etc/vector/vector.yaml
subPath: vector.yaml
readOnly: true
data:
type: emptyDir
globalMounts:
- path: /vector-data-dir
var-log:
type: hostPath
hostPath: /var/log
hostPathType: Directory
globalMounts:
- path: /var/log
readOnly: true
var-lib:
type: hostPath
hostPath: /var/lib
hostPathType: Directory
globalMounts:
- path: /var/lib
readOnly: true
procfs:
type: hostPath
hostPath: /proc
hostPathType: Directory
globalMounts:
- path: /host/proc
readOnly: true
sysfs:
type: hostPath
hostPath: /sys
hostPathType: Directory
globalMounts:
- path: /host/sys
readOnly: true
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
---
# yaml-language-server: $schema=https://json.schemastore.org/kustomization
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helm-release.yaml
- ./rbac.yaml
configMapGenerator:
- name: vector-agent-configmap
files:
- vector.yaml=./config/vector.yaml
generatorOptions:
disableNameSuffixHash: true
22 changes: 22 additions & 0 deletions kubernetes/talos-flux/apps/observability/vector/agent/rbac.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: vector-agent
rules:
- apiGroups: [""]
resources: ["namespaces", "nodes", "pods"]
verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: vector-agent
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: vector-agent
subjects:
- kind: ServiceAccount
name: vector-agent
namespace: observability
24 changes: 24 additions & 0 deletions kubernetes/talos-flux/apps/observability/vector/flux-sync.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
---
# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: &appname vector-agent
namespace: flux-system
spec:
dependsOn:
- name: vector-aggregator
targetNamespace: observability
commonMetadata:
labels:
app.kubernetes.io/name: vector
app.kubernetes.io/instance: *appname
interval: 10m
path: ./kubernetes/talos-flux/apps/observability/vector/agent
prune: true
sourceRef:
kind: GitRepository
name: home-ops
wait: true
retryInterval: 1m
timeout: 5m