Skip to content

Commit

Permalink
[okd] working monitoring + x509-exporter
Browse files Browse the repository at this point in the history
  • Loading branch information
caruccio committed Jul 31, 2024
1 parent e279244 commit 5b06330
Show file tree
Hide file tree
Showing 10 changed files with 618 additions and 174 deletions.
169 changes: 84 additions & 85 deletions root/usr/local/bin/x509-exporter-config-builder.sh
Original file line number Diff line number Diff line change
@@ -1,97 +1,96 @@
if [ $# -ne 1 ]; then
echo "Usage: $0 [cp|node]"
exit 1
fi

SEARCH_DIRS_CRT_CP=(
/etc/kubernetes/pki
/etc/kubernetes/ssl
#!/bin/bash

/etc/kubernetes/static-pod-resources/etcd-certs/secrets/etcd-all-certs
/etc/kubernetes/static-pod-resources/etcd-certs/configmaps
/etc/kubernetes/static-pod-resources/etcd-certs/configmaps/etcd-peer-client-ca
/etc/kubernetes/static-pod-resources/etcd-certs/configmaps/etcd-serving-ca

/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/aggregator-client
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/check-endpoints-client-cert-key
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/control-plane-node-admin-client-cert-key
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/external-loadbalancer-serving-certkey
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/internal-loadbalancer-serving-certkey
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/kubelet-client
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/localhost-serving-cert-certkey
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/service-network-serving-certkey
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/configmaps/aggregator-client-ca
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/configmaps/client-ca
# /etc/kubernetes/static-pod-resources/kube-apiserver-certs/configmaps/trusted-ca-bundle
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/

/etc/kubernetes/static-pod-resources/kube-controller-manager-certs/secrets/csr-signer
/etc/kubernetes/static-pod-resources/kube-controller-manager-certs/secrets/kube-controller-manager-client-cert-key
/etc/kubernetes/static-pod-resources/kube-controller-manager-certs/configmaps/aggregator-client-ca
/etc/kubernetes/static-pod-resources/kube-controller-manager-certs/configmaps/client-ca
# /etc/kubernetes/static-pod-resources/kube-controller-manager-certs/configmaps/trusted-ca-bundle

/etc/kubernetes/static-pod-resources/kube-scheduler-certs/secrets/kube-scheduler-client-cert-key
)
declare -A CERTS=()
declare -A CONFS=()

SEARCH_DIRS_CRT_NODE=(
/var/lib/kubelet/pki
/var/lib/kubelet/ssl
OKD_ROOT_DIR=/etc/kubernetes/static-pod-resources
CERTS_DIRS=(
/etc/kubernetes
/etc/kubernetes/pki
/etc/kubernetes/ssl
/var/lib/kubelet/pki/kubelet-client-current.pem
/var/lib/kubelet/pki/kubelet-server-current.pem
)
PODS_DIRS=()

SEARCH_DIRS_KUBECFG=(
if [ -d "$OKD_ROOT_DIR" ]; then
CERTS_DIR+=(
$OKD_ROOT_DIR/configmaps
$OKD_ROOT_DIR/etcd-certs
$OKD_ROOT_DIR/kube-apiserver-certs
$OKD_ROOT_DIR/kube-controller-manager-certs
$OKD_ROOT_DIR/kube-scheduler-certs
)
for name in etcd kube-apiserver kube-controller-manager kube-scheduler; do
current_no=$(printf "%s\n" $OKD_ROOT_DIR/${name}-pod-*/ | awk -F- '{print $NF}' | sort -n | tail -n 1)
pod_dir="$OKD_ROOT_DIR/${name}-pod-$current_no"

if ! [ -d "$pod_dir" ]; then
continue
fi
PODS_DIRS+=( "$pod_dir" )
done
fi

CERTS_DIRS+=( ${PODS_DIRS[*]} )

for name in ${CERTS_DIRS[*]}; do
certs=( $(find -L $name -type f -regextype egrep -regex '.*\.(crt|cert|pem)$' -exec grep -q '^-----BEGIN CERTIFICATE-----' {} \; -print 2>/dev/null) )

if [ ${#certs[*]} -eq 0 ]; then
continue
fi

for cert in ${certs[*]}; do
hash=$(md5sum "$cert" | cut -f 1 -d ' ')
CERTS["$hash"]="$cert"
done
done

CONFIG_DIRS=(
/etc/kubernetes
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/node-kubeconfigs
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/configmaps/check-endpoints-kubeconfig
/etc/kubernetes/static-pod-resources/kube-apiserver-certs/configmaps/control-plane-node-kubeconfig
/var/lib/kubelet
)

if [ $1 == cp ]; then
cat <<EOF
controlplane:
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- effect: NoSchedule
operator: Exists
EOF
echo
echo ' watchFiles: []'
echo
echo ' watchDirectories:'
for dir in ${SEARCH_DIRS_CRT_CP[@]}; do
find ${dir} -type f -regextype egrep -regex '.*\.(crt|pem)$' -printf "- %h\n" 2>/dev/null
done | sort -u | sed -e 's/^/ /'
echo
echo ' watchKubeconfFiles:'
for dir in ${SEARCH_DIRS_KUBECFG[@]}; do
find ${dir} -maxdepth 1 -type f -regextype egrep -regex '.*(kubeconfig|kubelet.conf|controller-manager.conf|scheduler.conf|admin.conf)$' -printf "- %p\n" 2>/dev/null
done | sort -u | sed -e 's/^/ /'


elif [ $1 == node ]; then
cat <<EOF
nodes:
tolerations:
- effect: NoSchedule
operator: Exists
EOF
echo
echo ' watchFiles: []'
echo
echo ' watchDirectories:'
for dir in ${SEARCH_DIRS_CRT_NODE[@]}; do
find ${dir} -type f -regextype egrep -regex '.*\.(crt|pem)$' -printf "- %h\n" 2>/dev/null
done | sort -u | sed -e 's/^/ /'
echo
echo ' watchKubeconfFiles:'
for dir in ${SEARCH_DIRS_KUBECFG[@]}; do
find ${dir} -maxdepth 1 -type f -regextype egrep -regex '.*(kubeconfig|kubelet.conf)$' -printf "- %p\n" 2>/dev/null
done | sort -u | sed -e 's/^/ /'
for name in ${CONFIG_DIRS[*]}; do
confs=( $(find -L $name -maxdepth 1 -type f -exec grep -qE '^(kind: Config|contexts:|clusters:)$' {} \; -print 2>/dev/null) )

if [ ${#confs[*]} -eq 0 ]; then
continue
fi

for conf in ${confs[*]}; do
hash=$(md5sum "$conf" | cut -f 1 -d ' ')
CONFS["$hash"]="$conf"
done
done

if [ ${#PODS_DIRS[*]} -gt 0 ]; then
for name in ${PODS_DIRS[*]}; do
confs=( $(find -L $name -type f -exec grep -qE '^(kind: Config|contexts:|clusters:)$' {} \; -print 2>/dev/null) )

if [ ${#confs[*]} -eq 0 ]; then
continue
fi

for conf in ${confs[*]}; do
hash=$(md5sum "$conf" | cut -f 1 -d ' ')
CONFS["$hash"]="$conf"
done
done
fi

echo 'watchFiles:'
if [ ${#CERTS[*]} -gt 0 ]; then
printf -- "- %s\n" ${CERTS[@]} | sort -u
else
echo "[]"
fi

echo
echo 'watchKubeconfFiles:'
if [ ${#CONFS[*]} -gt 0 ]; then
printf -- "- %s\n" ${CONFS[@]} | sort -u
else
echo "Usage: $0 [cp|node]"
exit 1
echo "[]"
fi
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ spec:
targetNamespace: cert-manager
values:
acme_email: ${ modules.cert-manager-config.acme_email }
ingress_class: ${ modules.cert-manager-config.ingress_class }
ingress_class: ${ cluster_type == "okd" ? "openshift-default" : modules.cert-manager-config.ingress_class }

cluster_issuer_selfsigned:
enabled: true
Expand Down
78 changes: 7 additions & 71 deletions templates/manifests/base/helmrelease-x509-exporter.yaml.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,13 @@ spec:
storageNamespace: x509-exporter
targetNamespace: x509-exporter
releaseName: x509-exporter
valuesFrom:
- kind: ConfigMap
name: host-paths-exporter-values-controlplane
optional: true
- kind: ConfigMap
name: host-paths-exporter-values-node
optional: true
values:
# Monitors certificates from node's filesystem
# https://github.com/enix/x509-certificate-exporter/tree/main/deploy/charts/x509-certificate-exporter#metrics-for-node-certificates-hostpath
Expand All @@ -88,77 +95,6 @@ spec:
user: system_u
%{~ endif }

daemonSets:
controlplane:
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- effect: NoSchedule
operator: Exists

%{~ if cluster_type == "okd" }
watchDirectories:
- /etc/kubernetes/static-pod-resources/etcd-certs/configmaps/etcd-metrics-proxy-client-ca
- /etc/kubernetes/static-pod-resources/etcd-certs/configmaps/etcd-metrics-proxy-serving-ca
- /etc/kubernetes/static-pod-resources/etcd-certs/configmaps/etcd-peer-client-ca
- /etc/kubernetes/static-pod-resources/etcd-certs/configmaps/etcd-serving-ca
- /etc/kubernetes/static-pod-resources/etcd-certs/secrets/etcd-all-certs
- /etc/kubernetes/static-pod-resources/kube-apiserver-certs/configmaps/aggregator-client-ca
- /etc/kubernetes/static-pod-resources/kube-apiserver-certs/configmaps/client-ca
# - /etc/kubernetes/static-pod-resources/kube-apiserver-certs/configmaps/trusted-ca-bundle
- /etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/aggregator-client
- /etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/check-endpoints-client-cert-key
- /etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/control-plane-node-admin-client-cert-key
- /etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/external-loadbalancer-serving-certkey
- /etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/internal-loadbalancer-serving-certkey
- /etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/kubelet-client
- /etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/localhost-serving-cert-certkey
- /etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/service-network-serving-certkey
- /etc/kubernetes/static-pod-resources/kube-controller-manager-certs/configmaps/aggregator-client-ca
- /etc/kubernetes/static-pod-resources/kube-controller-manager-certs/configmaps/client-ca
# - /etc/kubernetes/static-pod-resources/kube-controller-manager-certs/configmaps/trusted-ca-bundle
- /etc/kubernetes/static-pod-resources/kube-controller-manager-certs/secrets/csr-signer
- /etc/kubernetes/static-pod-resources/kube-controller-manager-certs/secrets/kube-controller-manager-client-cert-key
- /etc/kubernetes/static-pod-resources/kube-scheduler-certs/secrets/kube-scheduler-client-cert-key
%{~ endif }
%{~ if cluster_type == "kubespray" }
watchDirectories:
- /etc/kubernetes/ssl
- /var/lib/kubelet/pki

watchKubeconfFiles:
- /etc/kubernetes/admin.conf
- /etc/kubernetes/controller-manager.conf
- /etc/kubernetes/kubelet.conf
- /etc/kubernetes/scheduler.conf
%{~ endif }

nodes:
tolerations:
- effect: NoSchedule
operator: Exists

%{~ if cluster_type == "okd" }
watchFiles:
- /var/lib/kubelet/pki/kubelet-server-current.pem
- /var/lib/kubelet/pki/kubelet-client-current.pem

watchKubeconfFiles:
- /etc/kubernetes/kubeconfig
- /etc/kubernetes/kubelet.conf
%{~ endif }
%{~ if cluster_type == "kubespray" }
watchFiles:
- /var/lib/kubelet/pki/kubelet-server-current.pem
- /var/lib/kubelet/pki/kubelet-client-current.pem

watchDirectories:
- /etc/kubernetes/ssl

watchKubeconfFiles:
- /etc/kubernetes/kubelet.conf
%{~ endif }

# Monitors certificates from secrets
# https://github.com/enix/x509-certificate-exporter/tree/main/deploy/charts/x509-certificate-exporter#metrics-for-tls-secrets
secretsExporter:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
%{~ if teleport_auth_token != "" }
%{~ if cluster_type == "okd" ~}
---
apiVersion: security.openshift.io/v1
kind: SecurityContextConstraints
metadata:
annotations:
kubernetes.io/description: anyuid provides all features of the restricted SCC
but allows users to run with any UID and any GID.
name: teleport-agent
allowHostDirVolumePlugin: false
allowHostIPC: false
allowHostNetwork: false
allowHostPID: false
allowHostPorts: false
allowPrivilegeEscalation: true
allowPrivilegedContainer: false
allowedCapabilities: null
defaultAddCapabilities: null
fsGroup:
type: RunAsAny
groups:
- system:cluster-admins
priority: 10
readOnlyRootFilesystem: false
requiredDropCapabilities:
- MKNOD
runAsUser:
type: RunAsAny
seLinuxContext:
type: MustRunAs
supplementalGroups:
type: RunAsAny
users:
- system:serviceaccount:getup:teleport-agent
- system:serviceaccount:getup:teleport-agent-updater
volumes:
- configMap
- downwardAPI
- emptyDir
- persistentVolumeClaim
- projected
- secret
%{~ endif }
---
apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
name: teleport-agent
namespace: flux-system
spec:
chart:
spec:
chart: teleport-kube-agent
version: "14.1.3"
sourceRef:
kind: HelmRepository
name: teleport
install:
createNamespace: true
disableWait: true
remediation:
retries: -1
upgrade:
disableWait: false
remediation:
retries: -1
interval: 5m
releaseName: teleport-agent
storageNamespace: getup
targetNamespace: getup
values:
proxyAddr: ${teleport_proxy_addr}
authToken: ${teleport_auth_token}
kubeClusterName: ${teleport_kube_cluster_name}

labels:
${indent(6, yamlencode(teleport_labels))}

tolerations:
- key: dedicated
value: infra
effect: NoSchedule

affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: node-role.kubernetes.io/infra
operator: Exists
- weight: 90
preference:
matchExpressions:
- key: role
operator: In
values:
- infra

podSecurityPolicy:
enabled: false
%{~ endif }
Loading

0 comments on commit 5b06330

Please sign in to comment.