Support PAAS

* Adjust Kyverno policies to account for 'paas' role
OSC · Mar 29, 2024 · 1180359 · 1180359
1 parent dfd8b1c
commit 1180359
Show file tree

Hide file tree

Showing 30 changed files with 1,333 additions and 80 deletions.
diff --git a/.github/config/kyverno-policies-values.yaml b/.github/config/kyverno-policies-values.yaml
@@ -25,3 +25,16 @@ webservices:
     - busybox
   validNodeSelector:
     - test
+paas:
+  allowHostPaths:
+    - /etc/slurm
+    - /var/run/munge/munge.socket.2
+    - /users/?*
+    - /fs/?*
+    - /tmp
+  authorizedRegistries:
+    - docker-registry.osc.edu
+    - quay.io/oauth2-proxy/oauth2-proxy
+    - busybox
+  validNodeSelector:
+    - test
diff --git a/.github/config/kyverno-values.yaml b/.github/config/kyverno-values.yaml
@@ -38,3 +38,4 @@ config:
           operator: In
           values:
             - webservice
+            - paas
diff --git a/charts/kyverno-policies/Chart.yaml b/charts/kyverno-policies/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: kyverno-policies
 description: OSC Kyverno policies deployment
 type: application
-version: 0.23.0
+version: 0.24.0
 appVersion: "v1.10.7"
 maintainers:
   - name: treydock

diff --git a/charts/kyverno-policies/templates/add-service-account.yaml b/charts/kyverno-policies/templates/add-service-account.yaml
@@ -6,7 +6,7 @@ spec:
   validationFailureAction: enforce
   background: true
   rules:
-  - name: webservice-service-account-run-as
+  - name: service-account-run-as
     match:
       any:
         - resources:
@@ -18,6 +18,7 @@ spec:
                 operator: In
                 values:
                 - webservice
+                - paas
     preconditions:
     - key: "{{`{{ request.object.metadata.labels.\"osc.edu/service-account\" || '' }}`}}"
       operator: NotEquals
@@ -43,7 +44,7 @@ spec:
             runAsGroup: "{{`{{ gidMap.data.\"user-{{ request.object.metadata.labels.\"osc.edu/service-account\" }}\".to_number(@) }}`}}"
             fsGroup: "{{`{{ gidMap.data.\"user-{{ request.object.metadata.labels.\"osc.edu/service-account\" }}\".to_number(@) }}`}}"
             supplementalGroups: "{{`{{ gidsMap.data.\"user-{{ request.object.metadata.labels.\"osc.edu/service-account\" }}\" | parse_json(@)[*].to_number(@) }}`}}"
-  - name: webservice-service-account-run-as-containers
+  - name: service-account-run-as-containers
     match:
       any:
         - resources:
@@ -55,6 +56,7 @@ spec:
                 operator: In
                 values:
                 - webservice
+                - paas
     preconditions:
     - key: "{{`{{ request.object.metadata.labels.\"osc.edu/service-account\" || '' }}`}}"
       operator: NotEquals

diff --git a/charts/kyverno-policies/templates/authorized-registries.yaml b/charts/kyverno-policies/templates/authorized-registries.yaml
@@ -52,3 +52,30 @@ spec:
           =(initContainers):
             - image: "{{ . }}/* | {{ . }}:*"
       {{- end }}
+  - name: authorized-registries-paas
+    match:
+      any:
+        - resources:
+            kinds:
+            - Pod
+            namespaceSelector:
+              matchExpressions:
+              - key: osc.edu/role
+                operator: In
+                values:
+                - paas
+    exclude:
+      any:
+        - resources:
+            name: "cm-acme-http-solver-*"
+    validate:
+      message: >-
+        Images must come from {{ join " or " .Values.paas.authorizedRegistries }}
+      anyPattern:
+      {{- range .Values.paas.authorizedRegistries }}
+      - spec:
+          containers:
+            - image: "{{ . }}/* | {{ . }}:*"
+          =(initContainers):
+            - image: "{{ . }}/* | {{ . }}:*"
+      {{- end }}
diff --git a/charts/kyverno-policies/templates/pod-nodeselector.yaml b/charts/kyverno-policies/templates/pod-nodeselector.yaml
@@ -50,3 +50,30 @@ spec:
             nodeSelector:
               node-role.kubernetes.io/{{ . }}: ''
       {{- end }}
+  - name: pod-nodeselector-paas
+    match:
+      any:
+        - resources:
+            kinds:
+            - Pod
+            namespaceSelector:
+              matchExpressions:
+              - key: osc.edu/role
+                operator: In
+                values:
+                - paas
+    exclude:
+      any:
+        - resources:
+            name: "cm-acme-http-solver-*"
+    validate:
+      message: "Node selector must be set"
+      anyPattern:
+        - spec:
+            nodeSelector:
+              osc.edu/role: "{{ join " | " .Values.paas.validNodeSelector }}"
+      {{- range .Values.paas.validNodeSelector }}
+        - spec:
+            nodeSelector:
+              node-role.kubernetes.io/{{ . }}: ''
+      {{- end }}
diff --git a/charts/kyverno-policies/templates/pod-resources.yaml b/charts/kyverno-policies/templates/pod-resources.yaml
@@ -23,6 +23,7 @@ spec:
                 operator: In
                 values:
                 - webservice
+                - paas
     validate:
       message: "CPU and memory resource requests and limits are required for pods"
       pattern:
@@ -60,6 +61,7 @@ spec:
                 operator: In
                 values:
                 - webservice
+                - paas
     validate:
       message: "CPU and memory limits exceed max 8 CPUs and 32GB of memory"
       pattern:

diff --git a/charts/kyverno-policies/templates/pod-service-account-validation.yaml b/charts/kyverno-policies/templates/pod-service-account-validation.yaml
@@ -18,8 +18,9 @@ spec:
                 operator: In
                 values:
                 - webservice
+                - paas
     validate:
-      message: "Webservice pods must include a service account for access"
+      message: "Webservice and PAAS pods must include a service account for access"
       pattern:
         metadata:
           labels:
@@ -36,6 +37,7 @@ spec:
                 operator: In
                 values:
                 - webservice
+                - paas
     preconditions:
     - key: "{{`{{ request.operation }}`}}"
       operator: In
@@ -82,6 +84,7 @@ spec:
                 operator: In
                 values:
                 - webservice
+                - paas
     preconditions:
     - key: "{{`{{ request.operation }}`}}"
       operator: In
@@ -128,6 +131,7 @@ spec:
                 operator: In
                 values:
                 - webservice
+                - paas
     preconditions:
     - key: "{{`{{ request.operation }}`}}"
       operator: In
@@ -159,6 +163,7 @@ spec:
                 operator: In
                 values:
                 - webservice
+                - paas
     preconditions:
     - key: "{{`{{ request.operation }}`}}"
       operator: In

diff --git a/charts/kyverno-policies/templates/restrict-host-path.yaml b/charts/kyverno-policies/templates/restrict-host-path.yaml
@@ -48,3 +48,23 @@ spec:
           =(volumes):
           - =(hostPath):
               path: "{{ join " | " .Values.webservices.allowHostPaths }}"
+  - name: paas-host-path
+    match:
+      any:
+        - resources:
+            kinds:
+            - Pod
+            namespaceSelector:
+              matchExpressions:
+              - key: osc.edu/role
+                operator: In
+                values:
+                - paas
+    validate:
+      message: >-
+        That HostPath volume is forbidden. The fields spec.volumes[*].hostPath must not be set to allowed paths.
+      pattern:
+        spec:
+          =(volumes):
+          - =(hostPath):
+              path: "{{ join " | " .Values.paas.allowHostPaths }}"
diff --git a/charts/kyverno-policies/values.yaml b/charts/kyverno-policies/values.yaml
@@ -29,6 +29,23 @@ webservices:
   validNodeSelector:
     - infrastructure
     - webservices
+paas:
+  allowHostPaths:
+    - /var/lib/sss/pipes
+    - /etc/sssd
+    - /etc/nsswitch.conf
+    - /etc/slurm
+    - /var/run/munge/munge.socket.2
+    - /users/?*
+    - /fs/?*
+  authorizedRegistries:
+    - docker-registry.osc.edu
+    - docker-registry-test.osc.edu
+    - quay.io/oauth2-proxy/oauth2-proxy
+  validNodeSelector:
+    - infrastructure
+    - webservices
+    - paas
 validationFailureAction: {}
 kyverno-policies:
   # Supported- baseline/restricted/privileged/custom
@@ -69,6 +86,7 @@ kyverno-policies:
                 operator: In
                 values:
                 - webservice
+                - paas
     restrict-seccomp-strict:
       any:
         # TODO: Remove once ood_core updated
@@ -90,6 +108,7 @@ kyverno-policies:
                 operator: In
                 values:
                 - webservice
+                - paas
   policyPreconditions:
     disallow-capabilities:
       all:

diff --git a/tests/kyverno-policies/add-service-account/kyverno-test.yaml b/tests/kyverno-policies/add-service-account/kyverno-test.yaml
@@ -7,42 +7,56 @@ resources:
 variables: variables.yaml
 results:
   - policy: add-service-account
-    rule: webservice-service-account-run-as
+    rule: service-account-run-as
     resources:
       - test-webservice-service-account
     patchedResource: webservice-service-account-mutated.yaml
     kind: Pod
     result: pass
   - policy: add-service-account
-    rule: webservice-service-account-run-as
+    rule: service-account-run-as
+    resources:
+      - test-paas-service-account
+    patchedResource: paas-service-account-mutated.yaml
+    kind: Pod
+    result: pass
+  - policy: add-service-account
+    rule: service-account-run-as
     resources:
       - test-no-service-account-skip
     patchedResource: no-service-account.yaml
     kind: Pod
     result: skip
   - policy: add-service-account
-    rule: webservice-service-account-run-as
+    rule: service-account-run-as
     resources:
       - test-webservice-service-account-mariadb
     patchedResource: webservice-service-account-mariadb-mutated.yaml
     kind: Pod
     result: pass
   - policy: add-service-account
-    rule: webservice-service-account-run-as
+    rule: service-account-run-as
     resources:
       - test-webservice-service-account-skip
     patchedResource: skip.yaml
     kind: Pod
     result: skip
   - policy: add-service-account
-    rule: webservice-service-account-run-as-containers
+    rule: service-account-run-as-containers
     resources:
       - test-webservice-service-account-containers
     patchedResource: webservice-service-account-mutated-containers.yaml
     kind: Pod
     result: pass
   - policy: add-service-account
-    rule: webservice-service-account-run-as-containers
+    rule: service-account-run-as-containers
+    resources:
+      - test-paas-service-account-containers
+    patchedResource: paas-service-account-mutated-containers.yaml
+    kind: Pod
+    result: pass
+  - policy: add-service-account
+    rule: service-account-run-as-containers
     resources:
       - test-webservice-service-account-mariadb-containers
     patchedResource: webservice-service-account-mariadb-mutated-containers.yaml

diff --git a/tests/kyverno-policies/add-service-account/paas-service-account-mutated-containers.yaml b/tests/kyverno-policies/add-service-account/paas-service-account-mutated-containers.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-paas-service-account-containers
+  namespace: paas
+  labels:
+    osc.edu/service-account: test
+spec:
+  containers:
+    - name: nginx
+      image: nginx:latest
+  initContainers:
+    - name: init
+      image: busybox
diff --git a/tests/kyverno-policies/add-service-account/paas-service-account-mutated.yaml b/tests/kyverno-policies/add-service-account/paas-service-account-mutated.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-paas-service-account
+  namespace: paas
+  labels:
+    osc.edu/service-account: test
+spec:
+  containers:
+    - name: nginx
+      image: nginx:latest
+  initContainers:
+    - name: init
+      image: busybox
+  securityContext:
+    runAsUser: 1000
+    runAsGroup: 1001
+    fsGroup: 1001
+    supplementalGroups:
+      - 1001
+      - 1002
diff --git a/tests/kyverno-policies/add-service-account/resources.yaml b/tests/kyverno-policies/add-service-account/resources.yaml
@@ -26,6 +26,21 @@ spec:
 ---
 apiVersion: v1
 kind: Pod
+metadata:
+  name: test-paas-service-account
+  namespace: paas
+  labels:
+    osc.edu/service-account: test
+spec:
+  containers:
+    - name: nginx
+      image: nginx:latest
+  initContainers:
+    - name: init
+      image: busybox
+---
+apiVersion: v1
+kind: Pod
 metadata:
   name: test-webservice-service-account-containers
   namespace: webservice
@@ -41,6 +56,21 @@ spec:
 ---
 apiVersion: v1
 kind: Pod
+metadata:
+  name: test-paas-service-account-containers
+  namespace: paas
+  labels:
+    osc.edu/service-account: test
+spec:
+  containers:
+    - name: nginx
+      image: nginx:latest
+  initContainers:
+    - name: init
+      image: busybox
+---
+apiVersion: v1
+kind: Pod
 metadata:
   name: test-webservice-service-account-skip
   namespace: user-test
-Original file line number
+Diff line change
@@ Expand Up / @@ -38,3 +38,4 @@ config: @@
               operator: In
               values:
                 - webservice
+                - paas