From e6b8cfd998c856be2400f7ac799d02c94194cd6b Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Mon, 27 Nov 2023 09:36:48 +0000 Subject: [PATCH 1/5] Retrigger Snyk From 650d129ba9c82e095afc7e73aad446a1a5821981 Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Tue, 28 Nov 2023 08:28:01 +0000 Subject: [PATCH 2/5] Add YAML linting workflow and fix YAML formatting (#163) * Add YAML linting workflow and fix YAML formatting issues * Update .yamllint config file * Add YAML lint badge to README.md --- .github/workflows/lint-yaml.yaml | 14 + .yamllint.yaml | 6 + README.md | 2 +- config/base/kustomization.yaml | 13 +- config/base/manager_auth_proxy_patch.yaml | 3 +- config/base/manager_config_patch.yaml | 3 +- config/base/params.yaml | 3 +- ...styai.opendatahub.io_trustyaiservices.yaml | 186 ++++----- config/crd/kustomization.yaml | 4 +- config/crd/kustomizeconfig.yaml | 24 +- .../patches/webhook_in_trustyaiservices.yaml | 2 +- config/manager/kustomization.yaml | 4 +- config/manager/manager.yaml | 66 ++-- config/manifests/kustomization.yaml | 8 +- config/prometheus/kustomization.yaml | 2 +- .../rbac/auth_proxy_client_clusterrole.yaml | 8 +- config/rbac/auth_proxy_role.yaml | 24 +- config/rbac/auth_proxy_role_binding.yaml | 6 +- config/rbac/auth_proxy_service.yaml | 8 +- config/rbac/kustomization.yaml | 22 +- config/rbac/leader_election_role.yaml | 62 +-- config/rbac/leader_election_role_binding.yaml | 6 +- config/rbac/role.yaml | 354 +++++++++--------- config/rbac/role_binding.yaml | 6 +- config/rbac/trustyaiservice_editor_role.yaml | 36 +- config/rbac/trustyaiservice_viewer_role.yaml | 28 +- config/scorecard/bases/config.yaml | 4 +- config/scorecard/kustomization.yaml | 26 +- config/scorecard/patches/basic.config.yaml | 4 +- config/scorecard/patches/olm.config.yaml | 20 +- 30 files changed, 487 insertions(+), 467 deletions(-) create mode 100644 .github/workflows/lint-yaml.yaml create mode 100644 .yamllint.yaml diff --git a/.github/workflows/lint-yaml.yaml b/.github/workflows/lint-yaml.yaml new file mode 100644 index 00000000..a61a0ef9 --- /dev/null +++ b/.github/workflows/lint-yaml.yaml @@ -0,0 +1,14 @@ +name: YAML lint + +on: [push, pull_request] + +jobs: + lintAllTheThings: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: yaml-lint + uses: ibiqlik/action-yamllint@v3 + with: + file_or_dir: config/**/*.yaml + config_file: .yamllint.yaml \ No newline at end of file diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 00000000..75a3f8b6 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,6 @@ +extends: default + +rules: + line-length: + max: 80 + level: warning \ No newline at end of file diff --git a/README.md b/README.md index edab78d2..98e2cf67 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Controller Tests](https://github.com/trustyai-explainability/trustyai-service-operator/actions/workflows/controller-tests.yaml/badge.svg)](https://github.com/trustyai-explainability/trustyai-service-operator/actions/workflows/controller-tests.yaml) +[![Controller Tests](https://github.com/trustyai-explainability/trustyai-service-operator/actions/workflows/controller-tests.yaml/badge.svg)](https://github.com/trustyai-explainability/trustyai-service-operator/actions/workflows/controller-tests.yaml)[![YAML lint](https://github.com/trustyai-explainability/trustyai-service-operator/actions/workflows/lint-yaml.yaml/badge.svg)](https://github.com/trustyai-explainability/trustyai-service-operator/actions/workflows/lint-yaml.yaml) # TrustyAI Kubernetes Operator ## Overview diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml index 7e488d23..74947cbe 100644 --- a/config/base/kustomization.yaml +++ b/config/base/kustomization.yaml @@ -1,17 +1,16 @@ -#namespace: trustyai-service-operator-system - +--- namePrefix: trustyai-service-operator- resources: -- ../crd -- ../rbac -- ../manager + - ../crd + - ../rbac + - ../manager commonLabels: app.kubernetes.io/part-of: trustyai patchesStrategicMerge: -- manager_auth_proxy_patch.yaml + - manager_auth_proxy_patch.yaml configMapGenerator: - env: params.env @@ -37,4 +36,4 @@ vars: name: config apiVersion: v1 fieldref: - fieldpath: data.trustyaiOperatorImage \ No newline at end of file + fieldpath: data.trustyaiOperatorImage diff --git a/config/base/manager_auth_proxy_patch.yaml b/config/base/manager_auth_proxy_patch.yaml index 07c9db6a..2935b633 100644 --- a/config/base/manager_auth_proxy_patch.yaml +++ b/config/base/manager_auth_proxy_patch.yaml @@ -1,5 +1,4 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +--- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/config/base/manager_config_patch.yaml b/config/base/manager_config_patch.yaml index f6f58916..ee54324f 100644 --- a/config/base/manager_config_patch.yaml +++ b/config/base/manager_config_patch.yaml @@ -1,3 +1,4 @@ +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -7,4 +8,4 @@ spec: template: spec: containers: - - name: manager + - name: manager diff --git a/config/base/params.yaml b/config/base/params.yaml index 01ed557c..190afea0 100644 --- a/config/base/params.yaml +++ b/config/base/params.yaml @@ -1,3 +1,4 @@ +--- varReference: - kind: Deployment - path: spec/template/spec/containers[]/image \ No newline at end of file + path: spec/template/spec/containers[]/image diff --git a/config/crd/bases/trustyai.opendatahub.io.trustyai.opendatahub.io_trustyaiservices.yaml b/config/crd/bases/trustyai.opendatahub.io.trustyai.opendatahub.io_trustyaiservices.yaml index e7ee6e7e..7a44aa99 100644 --- a/config/crd/bases/trustyai.opendatahub.io.trustyai.opendatahub.io_trustyaiservices.yaml +++ b/config/crd/bases/trustyai.opendatahub.io.trustyai.opendatahub.io_trustyaiservices.yaml @@ -15,108 +15,108 @@ spec: singular: trustyaiservice scope: Namespaced versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: TrustyAIService is the Schema for the trustyaiservices API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation + - name: v1alpha1 + schema: + openAPIV3Schema: + description: TrustyAIService is the Schema for the trustyaiservices API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: TrustyAIServiceSpec defines the desired state of TrustyAIService - properties: - data: - properties: - filename: - type: string - format: - type: string - required: - - filename - - format - type: object - metrics: - properties: - batchSize: - type: integer - schedule: - type: string - required: - - schedule - type: object - replicas: - description: Number of replicas - format: int32 - type: integer - storage: - properties: - folder: - type: string - format: - type: string - size: - type: string - required: - - folder - - format - - size - type: object - required: - - data - - metrics - - storage - type: object - status: - description: TrustyAIServiceStatus defines the observed state of TrustyAIService - properties: - conditions: - items: - description: Condition represents possible conditions of a TrustyAIServiceStatus + type: string + metadata: + type: object + spec: + description: TrustyAIServiceSpec defines the desired state of TrustyAIService + properties: + data: properties: - lastTransitionTime: - format: date-time + filename: type: string - message: + format: type: string - reason: + required: + - filename + - format + type: object + metrics: + properties: + batchSize: + type: integer + schedule: + type: string + required: + - schedule + type: object + replicas: + description: Number of replicas + format: int32 + type: integer + storage: + properties: + folder: type: string - status: + format: type: string - type: + size: type: string required: - - lastTransitionTime - - message - - reason - - status - - type + - folder + - format + - size type: object - type: array - phase: - description: Define your status fields here - type: string - ready: - type: string - replicas: - format: int32 - type: integer - required: - - conditions - - phase - - replicas - type: object - type: object - served: true - storage: true - subresources: - status: {} + required: + - data + - metrics + - storage + type: object + status: + description: TrustyAIServiceStatus defines the observed state of TrustyAIService + properties: + conditions: + items: + description: Condition represents possible conditions of a TrustyAIServiceStatus + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + phase: + description: Define your status fields here + type: string + ready: + type: string + replicas: + format: int32 + type: integer + required: + - conditions + - phase + - replicas + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 727bbeb4..9c272a2d 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -1,5 +1,5 @@ resources: -- bases/trustyai.opendatahub.io.trustyai.opendatahub.io_trustyaiservices.yaml + - bases/trustyai.opendatahub.io.trustyai.opendatahub.io_trustyaiservices.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -7,4 +7,4 @@ patchesStrategicMerge: #+kubebuilder:scaffold:crdkustomizecainjectionpatch configurations: -- kustomizeconfig.yaml + - kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml index e0dd2980..3eb36bdc 100644 --- a/config/crd/kustomizeconfig.yaml +++ b/config/crd/kustomizeconfig.yaml @@ -1,18 +1,18 @@ nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition + - kind: Service version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/name + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false varReference: -- path: metadata/annotations + - path: metadata/annotations diff --git a/config/crd/patches/webhook_in_trustyaiservices.yaml b/config/crd/patches/webhook_in_trustyaiservices.yaml index 3b364e68..9ee4f134 100644 --- a/config/crd/patches/webhook_in_trustyaiservices.yaml +++ b/config/crd/patches/webhook_in_trustyaiservices.yaml @@ -12,4 +12,4 @@ spec: name: webhook-service path: /convert conversionReviewVersions: - - v1 + - v1 diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 056cd6ec..be0410af 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,4 +1,4 @@ resources: -- manager.yaml + - manager.yaml apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization \ No newline at end of file +kind: Kustomization diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index c0d7c7b5..058fa9bc 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -28,38 +28,38 @@ spec: seccompProfile: type: RuntimeDefault containers: - - command: - - /manager - args: - - --leader-elect - image: $(trustyaiOperatorImage) - name: manager - securityContext: - runAsNonRoot: true - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - seccompProfile: - type: RuntimeDefault - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi + - command: + - /manager + args: + - --leader-elect + image: $(trustyaiOperatorImage) + name: manager + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + seccompProfile: + type: RuntimeDefault + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml index e2870a36..15b0ed36 100644 --- a/config/manifests/kustomization.yaml +++ b/config/manifests/kustomization.yaml @@ -1,5 +1,5 @@ resources: -- bases/trustyai-service-operator.clusterserviceversion.yaml -- ../default -- ../samples -- ../scorecard \ No newline at end of file + - bases/trustyai-service-operator.clusterserviceversion.yaml + - ../default + - ../samples + - ../scorecard diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml index ed137168..d556b996 100644 --- a/config/prometheus/kustomization.yaml +++ b/config/prometheus/kustomization.yaml @@ -1,2 +1,2 @@ resources: -- monitor.yaml + - monitor.yaml diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml index e0a7df64..f93de767 100644 --- a/config/rbac/auth_proxy_client_clusterrole.yaml +++ b/config/rbac/auth_proxy_client_clusterrole.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/managed-by: kustomize name: metrics-reader rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get + - nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml index 51b0ff9a..1ba1c596 100644 --- a/config/rbac/auth_proxy_role.yaml +++ b/config/rbac/auth_proxy_role.yaml @@ -10,15 +10,15 @@ metadata: app.kubernetes.io/managed-by: kustomize name: proxy-role rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml index 095756e6..b8d5e154 100644 --- a/config/rbac/auth_proxy_role_binding.yaml +++ b/config/rbac/auth_proxy_role_binding.yaml @@ -14,6 +14,6 @@ roleRef: kind: ClusterRole name: proxy-role subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system + - kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml index 2250889d..7741af7b 100644 --- a/config/rbac/auth_proxy_service.yaml +++ b/config/rbac/auth_proxy_service.yaml @@ -13,9 +13,9 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https + - name: https + port: 8443 + protocol: TCP + targetPort: https selector: control-plane: controller-manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 34645515..02b3212f 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -1,12 +1,12 @@ resources: -- service_account.yaml -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml -- trustyaiservice_editor_role.yaml -- trustyaiservice_viewer_role.yaml \ No newline at end of file + - service_account.yaml + - role.yaml + - role_binding.yaml + - leader_election_role.yaml + - leader_election_role_binding.yaml + - auth_proxy_service.yaml + - auth_proxy_role.yaml + - auth_proxy_role_binding.yaml + - auth_proxy_client_clusterrole.yaml + - trustyaiservice_editor_role.yaml + - trustyaiservice_viewer_role.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml index bde71fd2..66f5f64f 100644 --- a/config/rbac/leader_election_role.yaml +++ b/config/rbac/leader_election_role.yaml @@ -10,34 +10,34 @@ metadata: app.kubernetes.io/managed-by: kustomize name: leader-election-role rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml index 4736aefb..75689753 100644 --- a/config/rbac/leader_election_role_binding.yaml +++ b/config/rbac/leader_election_role_binding.yaml @@ -14,6 +14,6 @@ roleRef: kind: Role name: leader-election-role subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system + - kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f97deb4c..39497986 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,180 +5,180 @@ metadata: creationTimestamp: null name: manager-role rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - deployments - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - deployments/finalizers - verbs: - - update -- apiGroups: - - apps - resources: - - deployments/status - verbs: - - get - - patch - - update -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - create - - list - - watch -- apiGroups: - - route.openshift.io - resources: - - routes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - serving.kserve.io - resources: - - inferenceservices - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - serving.kserve.io - resources: - - inferenceservices/finalizers - verbs: - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - serving.kserve.io - resources: - - servingruntimes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - serving.kserve.io - resources: - - servingruntimes/status - verbs: - - get - - patch - - update -- apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io - resources: - - trustyaiservices - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io - resources: - - trustyaiservices/finalizers - verbs: - - update -- apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io - resources: - - trustyaiservices/status - verbs: - - get - - patch - - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - apps + resources: + - deployments/finalizers + verbs: + - update + - apiGroups: + - apps + resources: + - deployments/status + verbs: + - get + - patch + - update + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - create + - list + - watch + - apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - serving.kserve.io + resources: + - inferenceservices + verbs: + - get + - list + - patch + - update + - watch + - apiGroups: + - serving.kserve.io + resources: + - inferenceservices/finalizers + verbs: + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - serving.kserve.io + resources: + - servingruntimes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - serving.kserve.io + resources: + - servingruntimes/status + verbs: + - get + - patch + - update + - apiGroups: + - trustyai.opendatahub.io.trustyai.opendatahub.io + resources: + - trustyaiservices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - trustyai.opendatahub.io.trustyai.opendatahub.io + resources: + - trustyaiservices/finalizers + verbs: + - update + - apiGroups: + - trustyai.opendatahub.io.trustyai.opendatahub.io + resources: + - trustyaiservices/status + verbs: + - get + - patch + - update diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 2fefba7c..bf595d68 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -14,6 +14,6 @@ roleRef: kind: ClusterRole name: manager-role subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system + - kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/trustyaiservice_editor_role.yaml b/config/rbac/trustyaiservice_editor_role.yaml index 1c0d4dcb..a238bff7 100644 --- a/config/rbac/trustyaiservice_editor_role.yaml +++ b/config/rbac/trustyaiservice_editor_role.yaml @@ -12,21 +12,21 @@ metadata: rbac.authorization.k8s.io/aggregate-to-admin: "true" name: trustyaiservice-editor-role rules: -- apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io - resources: - - trustyaiservices - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io - resources: - - trustyaiservices/status - verbs: - - get + - apiGroups: + - trustyai.opendatahub.io.trustyai.opendatahub.io + resources: + - trustyaiservices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - trustyai.opendatahub.io.trustyai.opendatahub.io + resources: + - trustyaiservices/status + verbs: + - get diff --git a/config/rbac/trustyaiservice_viewer_role.yaml b/config/rbac/trustyaiservice_viewer_role.yaml index a3a5ebf1..3a03ba46 100644 --- a/config/rbac/trustyaiservice_viewer_role.yaml +++ b/config/rbac/trustyaiservice_viewer_role.yaml @@ -13,17 +13,17 @@ metadata: rbac.authorization.k8s.io/aggregate-to-admin: "true" name: trustyaiservice-viewer-role rules: -- apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io - resources: - - trustyaiservices - verbs: - - get - - list - - watch -- apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io - resources: - - trustyaiservices/status - verbs: - - get + - apiGroups: + - trustyai.opendatahub.io.trustyai.opendatahub.io + resources: + - trustyaiservices + verbs: + - get + - list + - watch + - apiGroups: + - trustyai.opendatahub.io.trustyai.opendatahub.io + resources: + - trustyaiservices/status + verbs: + - get diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml index c7704784..707a5c25 100644 --- a/config/scorecard/bases/config.yaml +++ b/config/scorecard/bases/config.yaml @@ -3,5 +3,5 @@ kind: Configuration metadata: name: config stages: -- parallel: true - tests: [] + - parallel: true + tests: [] diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml index 50cd2d08..ee7181bb 100644 --- a/config/scorecard/kustomization.yaml +++ b/config/scorecard/kustomization.yaml @@ -1,16 +1,16 @@ resources: -- bases/config.yaml + - bases/config.yaml patchesJson6902: -- path: patches/basic.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -- path: patches/olm.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config + - path: patches/basic.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config + - path: patches/olm.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config #+kubebuilder:scaffold:patchesJson6902 diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml index c8455f1b..b27e7397 100644 --- a/config/scorecard/patches/basic.config.yaml +++ b/config/scorecard/patches/basic.config.yaml @@ -2,8 +2,8 @@ path: /stages/0/tests/- value: entrypoint: - - scorecard-test - - basic-check-spec + - scorecard-test + - basic-check-spec image: quay.io/operator-framework/scorecard-test:v1.28.1 labels: suite: basic diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml index 8680989e..6f51a3d7 100644 --- a/config/scorecard/patches/olm.config.yaml +++ b/config/scorecard/patches/olm.config.yaml @@ -2,8 +2,8 @@ path: /stages/0/tests/- value: entrypoint: - - scorecard-test - - olm-bundle-validation + - scorecard-test + - olm-bundle-validation image: quay.io/operator-framework/scorecard-test:v1.28.1 labels: suite: olm @@ -12,8 +12,8 @@ path: /stages/0/tests/- value: entrypoint: - - scorecard-test - - olm-crds-have-validation + - scorecard-test + - olm-crds-have-validation image: quay.io/operator-framework/scorecard-test:v1.28.1 labels: suite: olm @@ -22,8 +22,8 @@ path: /stages/0/tests/- value: entrypoint: - - scorecard-test - - olm-crds-have-resources + - scorecard-test + - olm-crds-have-resources image: quay.io/operator-framework/scorecard-test:v1.28.1 labels: suite: olm @@ -32,8 +32,8 @@ path: /stages/0/tests/- value: entrypoint: - - scorecard-test - - olm-spec-descriptors + - scorecard-test + - olm-spec-descriptors image: quay.io/operator-framework/scorecard-test:v1.28.1 labels: suite: olm @@ -42,8 +42,8 @@ path: /stages/0/tests/- value: entrypoint: - - scorecard-test - - olm-status-descriptors + - scorecard-test + - olm-status-descriptors image: quay.io/operator-framework/scorecard-test:v1.28.1 labels: suite: olm From 3af6912ec7493d625a6fc336413bd40d1360029f Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Tue, 28 Nov 2023 17:47:57 +0000 Subject: [PATCH 3/5] Add OAuth to external endpoints (#140) * Add OAuth to external endpoints * Change ServiceAccount. Remove unnecessary secrets. * Remove scope * Add ClusterRoleBindings creation * Move OAuth proxy image reference to params.env * Fix route test * Refactor deployment tests * Refactor suite_test.go file * Update dependencies in go.mod file * Add time import to trustyaiservice_controller.go * Add container images to deployment spec * Add OAuth-proxy image retrieval from ConfigMap * Refactor deployment_test.go file * Fix OAuth volume generation * Add WaitFor function to suite_test.go * Add test for custom images * Add ConfigMap tests * Add fake client builder for unit tests * Fix ConfigMap deletion in tests * Add test case for default values in ConfigMap * Add ConfigMap test for deploying with wrong keys * Refactor route creation in route_test.go * Refactor status and condition tests * Refactor storage_test.go to improve PVC reconciliation * Formatting of rbac/role.yaml * Formatting of base/kustomization.yaml --- config/base/kustomization.yaml | 7 + config/base/params.env | 3 +- config/rbac/auth-delegator.yml | 11 + config/rbac/role.yaml | 34 +++ controllers/config_maps.go | 46 +++ controllers/config_maps_test.go | 147 +++++++++ controllers/constants.go | 15 + controllers/deployment.go | 46 ++- controllers/deployment_test.go | 351 ++++++++++++++++++---- controllers/oauth.go | 181 +++++++++++ controllers/route.go | 84 +++++- controllers/route_test.go | 13 +- controllers/service_accounts.go | 138 +++++++++ controllers/statuses_test.go | 90 +++--- controllers/storage_test.go | 9 +- controllers/suite_test.go | 139 ++------- controllers/trustyaiservice_controller.go | 62 ++-- go.mod | 4 +- 18 files changed, 1092 insertions(+), 288 deletions(-) create mode 100644 config/rbac/auth-delegator.yml create mode 100644 controllers/config_maps.go create mode 100644 controllers/config_maps_test.go create mode 100644 controllers/oauth.go create mode 100644 controllers/service_accounts.go diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml index 74947cbe..e1305dc6 100644 --- a/config/base/kustomization.yaml +++ b/config/base/kustomization.yaml @@ -37,3 +37,10 @@ vars: apiVersion: v1 fieldref: fieldpath: data.trustyaiOperatorImage + - name: oauthProxyImage + objref: + kind: ConfigMap + name: config + apiVersion: v1 + fieldref: + fieldpath: data.oauthProxyImage diff --git a/config/base/params.env b/config/base/params.env index d05cce30..a5d271b1 100644 --- a/config/base/params.env +++ b/config/base/params.env @@ -1,2 +1,3 @@ trustyaiServiceImage=quay.io/trustyai/trustyai-service:latest -trustyaiOperatorImage=quay.io/trustyai/trustyai-service-operator:latest \ No newline at end of file +trustyaiOperatorImage=quay.io/trustyai/trustyai-service-operator:latest +oauthProxyImage=registry.redhat.io/openshift4/ose-oauth-proxy:latest \ No newline at end of file diff --git a/config/rbac/auth-delegator.yml b/config/rbac/auth-delegator.yml new file mode 100644 index 00000000..de9ac7db --- /dev/null +++ b/config/rbac/auth-delegator.yml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: + - kind: ServiceAccount + name: controller-manager \ No newline at end of file diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 39497986..93551452 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -37,6 +37,29 @@ rules: - patch - update - watch + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -103,6 +126,17 @@ rules: - create - list - watch + - apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - route.openshift.io resources: diff --git a/controllers/config_maps.go b/controllers/config_maps.go new file mode 100644 index 00000000..795a3485 --- /dev/null +++ b/controllers/config_maps.go @@ -0,0 +1,46 @@ +package controllers + +import ( + "context" + "fmt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" +) + +// getImageFromConfigMap gets a custom image value from a ConfigMap in the operator's namespace +func (r *TrustyAIServiceReconciler) getImageFromConfigMap(ctx context.Context, key string, defaultImage string) (string, error) { + if r.Namespace != "" { + // Define the key for the ConfigMap + configMapKey := types.NamespacedName{ + Namespace: r.Namespace, + Name: imageConfigMap, + } + + // Create an empty ConfigMap object + var cm corev1.ConfigMap + + // Try to get the ConfigMap + if err := r.Get(ctx, configMapKey, &cm); err != nil { + if errors.IsNotFound(err) { + // ConfigMap not found, fallback to default values + return defaultImage, nil + } + // Other error occurred when trying to fetch the ConfigMap + return defaultImage, fmt.Errorf("error reading configmap %s", configMapKey) + } + + // ConfigMap is found, extract the image and tag + image, ok := cm.Data[key] + + if !ok { + // One or both of the keys are not present in the ConfigMap, return error + return defaultImage, fmt.Errorf("configmap %s does not contain necessary keys", configMapKey) + } + + // Return the image and tag + return image, nil + } else { + return defaultImage, nil + } +} diff --git a/controllers/config_maps_test.go b/controllers/config_maps_test.go new file mode 100644 index 00000000..6dfee49d --- /dev/null +++ b/controllers/config_maps_test.go @@ -0,0 +1,147 @@ +package controllers + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var _ = Describe("ConfigMap tests", func() { + + BeforeEach(func() { + recorder = record.NewFakeRecorder(10) + k8sClient = fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + reconciler = &TrustyAIServiceReconciler{ + Client: k8sClient, + Scheme: scheme.Scheme, + EventRecorder: recorder, + Namespace: operatorNamespace, + } + ctx = context.Background() + }) + + AfterEach(func() { + // Attempt to delete the ConfigMap + configMap := &corev1.ConfigMap{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: operatorNamespace, + Name: imageConfigMap, + }, configMap) + + // If the ConfigMap exists, delete it + if err == nil { + Expect(k8sClient.Delete(ctx, configMap)).To(Succeed()) + } else if !apierrors.IsNotFound(err) { + Fail(fmt.Sprintf("Unexpected error while getting ConfigMap: %s", err)) + } + }) + + Context("When deploying a ConfigMap to the operator's namespace", func() { + + It("Should get back the correct values", func() { + + serviceImage := "custom-service-image:foo" + oauthImage := "custom-oauth-proxy:bar" + + WaitFor(func() error { + configMap := createConfigMap(operatorNamespace, oauthImage, serviceImage) + return k8sClient.Create(ctx, configMap) + }, "failed to create ConfigMap") + + var actualOAuthImage string + var actualServiceImage string + + WaitFor(func() error { + var err error + actualOAuthImage, err = reconciler.getImageFromConfigMap(ctx, configMapOAuthProxyImageKey, defaultOAuthProxyImage) + return err + }, "failed to get oauth image from ConfigMap") + + WaitFor(func() error { + var err error + actualServiceImage, err = reconciler.getImageFromConfigMap(ctx, configMapServiceImageKey, defaultImage) + return err + }, "failed to get service image from ConfigMap") + + Expect(actualOAuthImage).Should(Equal(oauthImage)) + Expect(actualServiceImage).Should(Equal(serviceImage)) + }) + }) + + Context("When no ConfigMap in the operator's namespace", func() { + + It("Should get back the default values", func() { + + var actualOAuthImage string + var actualServiceImage string + + WaitFor(func() error { + var err error + actualOAuthImage, err = reconciler.getImageFromConfigMap(ctx, configMapOAuthProxyImageKey, defaultOAuthProxyImage) + return err + }, "failed to get oauth image from ConfigMap") + + WaitFor(func() error { + var err error + actualServiceImage, err = reconciler.getImageFromConfigMap(ctx, configMapServiceImageKey, defaultImage) + return err + }, "failed to get service image from ConfigMap") + + Expect(actualOAuthImage).Should(Equal(defaultOAuthProxyImage)) + Expect(actualServiceImage).Should(Equal(defaultImage)) + }) + }) + + Context("When deploying a ConfigMap to the operator's namespace with the wrong keys", func() { + + It("Should get back the default values", func() { + + serviceImage := "custom-service-image:foo" + oauthImage := "custom-oauth-proxy:bar" + + WaitFor(func() error { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: imageConfigMap, + Namespace: operatorNamespace, + }, + Data: map[string]string{ + "foo-oauth-image": oauthImage, + "foo-image": serviceImage, + }, + } + return k8sClient.Create(ctx, configMap) + }, "failed to create ConfigMap") + + var actualOAuthImage string + var actualServiceImage string + + configMapPath := operatorNamespace + "/" + imageConfigMap + + Eventually(func() error { + var err error + actualOAuthImage, err = reconciler.getImageFromConfigMap(ctx, configMapOAuthProxyImageKey, defaultOAuthProxyImage) + return err + }, defaultTimeout, defaultPolling).Should(MatchError(fmt.Sprintf("configmap %s does not contain necessary keys", configMapPath)), "failed to get oauth image from ConfigMap") + + Eventually(func() error { + var err error + actualServiceImage, err = reconciler.getImageFromConfigMap(ctx, configMapServiceImageKey, defaultImage) + return err + }, defaultTimeout, defaultPolling).Should(MatchError(fmt.Sprintf("configmap %s does not contain necessary keys", configMapPath)), "failed to get oauth image from ConfigMap") + + Expect(actualOAuthImage).Should(Equal(defaultOAuthProxyImage)) + Expect(actualServiceImage).Should(Equal(defaultImage)) + }) + }) + +}) diff --git a/controllers/constants.go b/controllers/constants.go index 5d1ea4ab..51762802 100644 --- a/controllers/constants.go +++ b/controllers/constants.go @@ -15,6 +15,21 @@ const ( defaultRequeueDelay = time.Minute ) +// Configuration constants +const ( + imageConfigMap = "trustyai-service-operator-config" + configMapOAuthProxyImageKey = "oauthProxyImage" + configMapServiceImageKey = "trustyaiServiceImage" +) + +// OAuth constants +const ( + OAuthServicePort = 443 + OAuthName = "oauth-proxy" + OAuthServicePortName = "oauth-proxy" + defaultOAuthProxyImage = "registry.redhat.io/openshift4/ose-oauth-proxy:latest" +) + // Status types const ( StatusTypeInferenceServicesPresent = "InferenceServicesPresent" diff --git a/controllers/deployment.go b/controllers/deployment.go index 09919a12..b27cd5d1 100644 --- a/controllers/deployment.go +++ b/controllers/deployment.go @@ -2,6 +2,8 @@ package controllers import ( "context" + "strconv" + trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -10,12 +12,13 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" - "strconv" ) +// createDeploymentObject returns a Deployment for the TrustyAI Service instance func (r *TrustyAIServiceReconciler) createDeploymentObject(ctx context.Context, cr *trustyaiopendatahubiov1alpha1.TrustyAIService, image string) *appsv1.Deployment { labels := getCommonLabels(cr.Name) pvcName := generatePVCName(cr) + serviceAccountName := generateServiceAccountName(cr) replicas := int32(1) if cr.Spec.Replicas == nil { @@ -29,6 +32,15 @@ func (r *TrustyAIServiceReconciler) createDeploymentObject(ctx context.Context, batchSize = *cr.Spec.Metrics.BatchSize } + // Create the OAuth-Proxy container spec + + // Get OAuth-proxy image from ConfigMap + oauthProxyImage, err := r.getImageFromConfigMap(ctx, configMapOAuthProxyImageKey, defaultOAuthProxyImage) + if err != nil { + log.FromContext(ctx).Error(err, "Error getting OAuth image from ConfigMap. Using the default image value of "+defaultOAuthProxyImage) + } + oauthProxyContainer := generateOAuthProxyContainer(cr, oauthProxyImage) + containers := []corev1.Container{ { Name: containerName, @@ -67,8 +79,23 @@ func (r *TrustyAIServiceReconciler) createDeploymentObject(ctx context.Context, }, }, }, + oauthProxyContainer, } + volume := corev1.Volume{ + + Name: volumeMountName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + ReadOnly: false, + }, + }, + } + volumes := generateOAuthVolumes(cr, OAuthConfig{ProxyImage: defaultOAuthProxyImage}) + + volumes = append(volumes, volume) + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: cr.Name, @@ -90,18 +117,9 @@ func (r *TrustyAIServiceReconciler) createDeploymentObject(ctx context.Context, }, }, Spec: corev1.PodSpec{ - Containers: containers, - Volumes: []corev1.Volume{ - { - Name: volumeMountName, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvcName, - ReadOnly: false, - }, - }, - }, - }, + ServiceAccountName: serviceAccountName, + Containers: containers, + Volumes: volumes, }, }, }, @@ -148,7 +166,7 @@ func (r *TrustyAIServiceReconciler) ensureDeployment(ctx context.Context, instan // Get image and tag from ConfigMap // If there's a ConfigMap with custom images, it is only applied when the operator is first deployed // Changing (or creating) the ConfigMap after the operator is deployed will not have any effect - image, err := r.getImageFromConfigMap(ctx) + image, err := r.getImageFromConfigMap(ctx, configMapServiceImageKey, defaultImage) if err != nil { return err } diff --git a/controllers/deployment_test.go b/controllers/deployment_test.go index 3dffbf1b..4f4a6829 100644 --- a/controllers/deployment_test.go +++ b/controllers/deployment_test.go @@ -2,22 +2,36 @@ package controllers import ( "context" + "encoding/json" + "fmt" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" - "time" + "sigs.k8s.io/controller-runtime/pkg/client/fake" //+kubebuilder:scaffold:imports ) +func printKubeObject(obj interface{}) { + bytes, err := json.MarshalIndent(obj, "", " ") + if err != nil { + fmt.Println("Error printing object:", err) + } else { + fmt.Println(string(bytes)) + } +} + var _ = Describe("TrustyAI operator", func() { BeforeEach(func() { recorder = record.NewFakeRecorder(10) + k8sClient = fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() reconciler = &TrustyAIServiceReconciler{ Client: k8sClient, Scheme: scheme.Scheme, @@ -27,22 +41,34 @@ var _ = Describe("TrustyAI operator", func() { ctx = context.Background() }) + AfterEach(func() { + // Attempt to delete the ConfigMap + configMap := &corev1.ConfigMap{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: operatorNamespace, + Name: imageConfigMap, + }, configMap) + + // If the ConfigMap exists, delete it + if err == nil { + Expect(k8sClient.Delete(ctx, configMap)).To(Succeed()) + } else if !apierrors.IsNotFound(err) { + Fail(fmt.Sprintf("Unexpected error while getting ConfigMap: %s", err)) + } + }) + Context("When deploying with default settings without an InferenceService", func() { var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + It("Creates a deployment and a service with the default configuration", func() { - namespace := "trusty-ns-1" - instance = createDefaultCR(namespace) - Eventually(func() error { - return createNamespace(ctx, k8sClient, namespace) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create namespace") - Eventually(func() error { - return createTestPVC(ctx, k8sClient, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create PVC") + namespace := "trusty-ns-a-1" + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + instance = createDefaultCR(namespace) - Eventually(func() error { - return reconciler.ensureDeployment(ctx, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create deployment") + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + Expect(reconciler.ensureDeployment(ctx, instance)).To(Succeed()) deployment := &appsv1.Deployment{} err := k8sClient.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, deployment) @@ -58,70 +84,273 @@ var _ = Describe("TrustyAI operator", func() { Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal("0.1.0")) + Expect(len(deployment.Spec.Template.Spec.Containers)).Should(Equal(2)) Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal("quay.io/trustyai/trustyai-service:latest")) + Expect(deployment.Spec.Template.Spec.Containers[1].Image).Should(Equal("registry.redhat.io/openshift4/ose-oauth-proxy:latest")) - Eventually(func() error { + WaitFor(func() error { service, _ := reconciler.reconcileService(instance) return reconciler.Create(ctx, service) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create service") + }, "failed to create service") service := &corev1.Service{} - Eventually(func() error { + WaitFor(func() error { return k8sClient.Get(ctx, types.NamespacedName{Name: defaultServiceName, Namespace: namespace}, service) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to get Service") + }, "failed to get Service") Expect(service.Annotations["prometheus.io/path"]).Should(Equal("/q/metrics")) Expect(service.Annotations["prometheus.io/scheme"]).Should(Equal("http")) Expect(service.Annotations["prometheus.io/scrape"]).Should(Equal("true")) Expect(service.Namespace).Should(Equal(namespace)) + + WaitFor(func() error { + err := reconciler.reconcileOAuthService(ctx, instance) + return err + }, "failed to create oauth service") + + desiredOAuthService := generateTrustyAIOAuthService(instance) + + oauthService := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) + }, "failed to get OAuth Service") + }) + }) - Context("When deploying with an associated InferenceService", func() { - It("Sets up the InferenceService and links it to the TrustyAIService deployment", func() { - namespace := "trusty-ns-2" - instance = createDefaultCR(namespace) + Context("When deploying with a ConfigMap and without an InferenceService", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService - Eventually(func() error { - return createNamespace(ctx, k8sClient, namespace) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create namespace") - Eventually(func() error { - return createTestPVC(ctx, k8sClient, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create PVC") - Eventually(func() error { - return reconciler.ensureDeployment(ctx, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create deployment") + It("Creates a deployment and a service with the ConfigMap configuration", func() { - // Creating the InferenceService - inferenceService := createInferenceService("my-model", namespace) - Eventually(func() error { - return k8sClient.Create(ctx, inferenceService) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create deployment") + namespace := "trusty-ns-a-1-cm" + serviceImage := "custom-service-image:foo" + oauthImage := "custom-oauth-proxy:bar" + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) - Expect(reconciler.patchKServe(ctx, instance, *inferenceService, namespace, instance.Name, false)).ToNot(HaveOccurred()) + WaitFor(func() error { + configMap := createConfigMap(operatorNamespace, oauthImage, serviceImage) + return k8sClient.Create(ctx, configMap) + }, "failed to create ConfigMap") - deployment := &appsv1.Deployment{} - Eventually(func() error { - // Define defaultServiceName for the deployment created by the operator - namespacedNamed := types.NamespacedName{ - Namespace: namespace, - Name: instance.Name, + instance = createDefaultCR(namespace) + + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance) + }, "failed to reconcile deployment") + + deployment := &appsv1.Deployment{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: namespace}, deployment) + }, "failed to get updated deployment") + Expect(deployment).ToNot(BeNil()) + + Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) + Expect(deployment.Namespace).Should(Equal(namespace)) + Expect(deployment.Name).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal("0.1.0")) + + Expect(len(deployment.Spec.Template.Spec.Containers)).Should(Equal(2)) + Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(serviceImage)) + Expect(deployment.Spec.Template.Spec.Containers[1].Image).Should(Equal(oauthImage)) + + WaitFor(func() error { + service, _ := reconciler.reconcileService(instance) + return reconciler.Create(ctx, service) + }, "failed to create service") + + service := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: defaultServiceName, Namespace: namespace}, service) + }, "failed to get Service") + + Expect(service.Annotations["prometheus.io/path"]).Should(Equal("/q/metrics")) + Expect(service.Annotations["prometheus.io/scheme"]).Should(Equal("http")) + Expect(service.Annotations["prometheus.io/scrape"]).Should(Equal("true")) + Expect(service.Namespace).Should(Equal(namespace)) + + WaitFor(func() error { + err := reconciler.reconcileOAuthService(ctx, instance) + return err + }, "failed to create oauth service") + + desiredOAuthService := generateTrustyAIOAuthService(instance) + + oauthService := &corev1.Service{} + WaitFor(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: desiredOAuthService.Name, Namespace: namespace}, oauthService) + }, "failed to get OAuth Service") + + }) + }) + + Context("When deploying with default settings without an InferenceService", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + + It("should set environment variables correctly", func() { + + namespace := "trusty-ns-a-4" + instance = createDefaultCR(namespace) + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + Expect(reconciler.ensureDeployment(ctx, instance)).To(Succeed()) + + deployment := &appsv1.Deployment{} + namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} + Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) + + foundEnvVar := func(envVars []corev1.EnvVar, name string) *corev1.EnvVar { + for _, env := range envVars { + if env.Name == name { + return &env } - return k8sClient.Get(ctx, namespacedNamed, deployment) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to get Deployment") + } + return nil + } - Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) - Expect(deployment.Namespace).Should(Equal(namespace)) - Expect(deployment.Name).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) - Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) - Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal("0.1.0")) + var trustyaiServiceContainer *corev1.Container + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == "trustyai-service" { + trustyaiServiceContainer = &container + break + } + } + + Expect(trustyaiServiceContainer).NotTo(BeNil(), "trustyai-service container not found") + + // Checking the environment variables of the trustyai-service container + var envVar *corev1.EnvVar + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_BATCH_SIZE") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_BATCH_SIZE not found") + Expect(envVar.Value).To(Equal("5000")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FILENAME") + Expect(envVar).NotTo(BeNil(), "Env var STORAGE_DATA_FILENAME not found") + Expect(envVar.Value).To(Equal("data.csv")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_STORAGE_FORMAT") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_STORAGE_FORMAT not found") + Expect(envVar.Value).To(Equal("PVC")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "STORAGE_DATA_FOLDER") + Expect(envVar).NotTo(BeNil(), "Env var STORAGE_DATA_FOLDER not found") + Expect(envVar.Value).To(Equal("/data")) + + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_DATA_FORMAT") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_DATA_FORMAT not found") + Expect(envVar.Value).To(Equal("CSV")) - }) + envVar = foundEnvVar(trustyaiServiceContainer.Env, "SERVICE_METRICS_SCHEDULE") + Expect(envVar).NotTo(BeNil(), "Env var SERVICE_METRICS_SCHEDULE not found") + Expect(envVar.Value).To(Equal("5s")) }) }) + Context("When deploying with default settings without an InferenceService", func() { + var instance *trustyaiopendatahubiov1alpha1.TrustyAIService + + It("should use the correct service account", func() { + + namespace := "trusty-ns-a-6" + instance = createDefaultCR(namespace) + Expect(createNamespace(ctx, k8sClient, namespace)).To(Succeed()) + Expect(createTestPVC(ctx, k8sClient, instance)).To(Succeed()) + Expect(reconciler.createServiceAccount(ctx, instance)).To(Succeed()) + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance) + }, "failed to create deployment") + + deployment := &appsv1.Deployment{} + namespacedName := types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace} + Expect(k8sClient.Get(ctx, namespacedName, deployment)).Should(Succeed()) + + Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(instance.Name + "-proxy")) + }) + }) + +}) + +var _ = Describe("TrustyAI operator", func() { + + BeforeEach(func() { + recorder = record.NewFakeRecorder(10) + reconciler = &TrustyAIServiceReconciler{ + Client: k8sClient, + Scheme: scheme.Scheme, + EventRecorder: recorder, + Namespace: operatorNamespace, + } + ctx = context.Background() + }) + + Context("When deploying with an associated InferenceService", func() { + + It("Sets up the InferenceService and links it to the TrustyAIService deployment", func() { + + namespace := "trusty-ns-2" + instance := createDefaultCR(namespace) + WaitFor(func() error { + return createNamespace(ctx, k8sClient, namespace) + }, "failed to create namespace") + WaitFor(func() error { + return createTestPVC(ctx, k8sClient, instance) + }, "failed to create PVC") + WaitFor(func() error { + return reconciler.ensureDeployment(ctx, instance) + }, "failed to create deployment") + + // Creating the InferenceService + inferenceService := createInferenceService("my-model", namespace) + WaitFor(func() error { + return k8sClient.Create(ctx, inferenceService) + }, "failed to create deployment") + + Expect(reconciler.patchKServe(ctx, instance, *inferenceService, namespace, instance.Name, false)).ToNot(HaveOccurred()) + + deployment := &appsv1.Deployment{} + WaitFor(func() error { + // Define defaultServiceName for the deployment created by the operator + namespacedNamed := types.NamespacedName{ + Namespace: namespace, + Name: instance.Name, + } + return k8sClient.Get(ctx, namespacedNamed, deployment) + }, "failed to get Deployment") + + Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) + Expect(deployment.Namespace).Should(Equal(namespace)) + Expect(deployment.Name).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/name"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/instance"]).Should(Equal(defaultServiceName)) + Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) + Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal("0.1.0")) + + }) + }) +}) + +var _ = Describe("TrustyAI operator", func() { + + BeforeEach(func() { + recorder = record.NewFakeRecorder(10) + reconciler = &TrustyAIServiceReconciler{ + Client: k8sClient, + Scheme: scheme.Scheme, + EventRecorder: recorder, + Namespace: operatorNamespace, + } + ctx = context.Background() + }) + Context("Across multiple namespaces", func() { var instances []*trustyaiopendatahubiov1alpha1.TrustyAIService @@ -133,28 +362,28 @@ var _ = Describe("TrustyAI operator", func() { for i, namespace := range namespaces { instances[i] = createDefaultCR(namespace) instances[i].Namespace = namespace - Eventually(func() error { + WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create namespace") + }, "failed to create namespace") } for _, instance := range instances { - Eventually(func() error { + WaitFor(func() error { return createTestPVC(ctx, k8sClient, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create PVC") - Eventually(func() error { + }, "failed to create PVC") + WaitFor(func() error { return reconciler.ensureDeployment(ctx, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create deployment") + }, "failed to create deployment") //Expect(k8sClient.Create(ctx, instance)).Should(Succeed()) deployment := &appsv1.Deployment{} - Eventually(func() error { + WaitFor(func() error { // Define defaultServiceName for the deployment created by the operator namespacedNamed := types.NamespacedName{ Namespace: instance.Namespace, Name: defaultServiceName, } return k8sClient.Get(ctx, namespacedNamed, deployment) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to get Deployment") + }, "failed to get Deployment") Expect(*deployment.Spec.Replicas).Should(Equal(int32(1))) Expect(deployment.Namespace).Should(Equal(instance.Namespace)) @@ -165,7 +394,9 @@ var _ = Describe("TrustyAI operator", func() { Expect(deployment.Labels["app.kubernetes.io/part-of"]).Should(Equal(componentName)) Expect(deployment.Labels["app.kubernetes.io/version"]).Should(Equal("0.1.0")) + Expect(len(deployment.Spec.Template.Spec.Containers)).Should(Equal(2)) Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal("quay.io/trustyai/trustyai-service:latest")) + Expect(deployment.Spec.Template.Spec.Containers[1].Image).Should(Equal("registry.redhat.io/openshift4/ose-oauth-proxy:latest")) } }) diff --git a/controllers/oauth.go b/controllers/oauth.go new file mode 100644 index 00000000..fd136ab9 --- /dev/null +++ b/controllers/oauth.go @@ -0,0 +1,181 @@ +package controllers + +import ( + "context" + "fmt" + + trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type OAuthConfig struct { + ProxyImage string +} + +// generateOAuthProxyContainer create the OAuth-proxy container object for a TrustyAI service instance +func generateOAuthProxyContainer(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, oauthProxyImage string) corev1.Container { + proxyContainer := corev1.Container{ + Name: OAuthName, + Image: oauthProxyImage, + ImagePullPolicy: corev1.PullAlways, + Env: []corev1.EnvVar{{ + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }}, + Args: []string{ + "--cookie-secret=SECRET", + "--https-address=:8443", + "--email-domain=*", + fmt.Sprintf("--openshift-service-account=%s", instance.Name+"-proxy"), + "--provider=openshift", + "--tls-cert=/etc/tls/private/tls.crt", + "--tls-key=/etc/tls/private/tls.key", + "--upstream=http://localhost:8080", + "--skip-auth-regex='(^/metrics|^/apis/v1beta1/healthz)'", + fmt.Sprintf("--openshift-sar={\"namespace\":\"%s\",\"resource\":\"pods\",\"verb\":\"get\"}", instance.Namespace), + fmt.Sprintf("--openshift-delegate-urls={\"/\": {\"namespace\": \"%s\", \"resource\": \"pods\", \"verb\": \"get\"}}", instance.Namespace), + }, + Ports: []corev1.ContainerPort{{ + Name: OAuthServicePortName, + ContainerPort: 8443, + Protocol: corev1.ProtocolTCP, + }}, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/oauth/healthz", + Port: intstr.FromString(OAuthServicePortName), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/oauth/healthz", + Port: intstr.FromString(OAuthServicePortName), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("100m"), + "memory": resource.MustParse("64Mi"), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("100m"), + "memory": resource.MustParse("64Mi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: instance.Name + "-tls", + MountPath: "/etc/tls/private", + }, + }, + } + return proxyContainer +} + +// generateOAuthVolumes create the necessary OAuth volume objects +func generateOAuthVolumes(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, oauth OAuthConfig) []corev1.Volume { + + volumes := []corev1.Volume{ + { + Name: instance.Name + "-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: instance.Name + "-tls", + }, + }, + }, + } + return volumes +} + +// generateTrustyAIOAuthService defines the desired OAuth service object +func generateTrustyAIOAuthService(instance *trustyaiopendatahubiov1alpha1.TrustyAIService) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name + "-tls", + Namespace: instance.Namespace, + Labels: map[string]string{ + "trustyai-service-name": instance.Name, + }, + Annotations: map[string]string{ + "service.beta.openshift.io/serving-cert-secret-name": instance.Name + "-tls", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{ + Name: OAuthServicePortName, + Port: OAuthServicePort, + TargetPort: intstr.FromInt(8443), + Protocol: corev1.ProtocolTCP, + }}, + Selector: map[string]string{ + "app": instance.Name, + }, + }, + } +} + +// reconcileOAuthService will manage the OAuth service reconciliation required +// by the service's OAuth proxy +func (r *TrustyAIServiceReconciler) reconcileOAuthService(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService) error { + + // Generate the desired OAuth service object + desiredService := generateTrustyAIOAuthService(instance) + + // Create the OAuth service if it does not already exist + foundService := &corev1.Service{} + err := r.Get(ctx, types.NamespacedName{ + Name: desiredService.GetName(), + Namespace: instance.GetNamespace(), + }, foundService) + if err != nil { + if errors.IsNotFound(err) { + log.FromContext(ctx).Info("Creating OAuth Service") + // Add .metatada.ownerReferences to the OAuth service to be deleted by + // the Kubernetes garbage collector if the service is deleted + err = ctrl.SetControllerReference(instance, desiredService, r.Scheme) + if err != nil { + log.FromContext(ctx).Error(err, "Unable to add OwnerReference to the OAuth Service") + return err + } + // Create the OAuth service in the Openshift cluster + err = r.Create(ctx, desiredService) + if err != nil && !errors.IsAlreadyExists(err) { + log.FromContext(ctx).Error(err, "Unable to create the OAuth Service") + return err + } + } else { + log.FromContext(ctx).Error(err, "Unable to fetch the OAuth Service") + return err + } + } + + return nil +} diff --git a/controllers/route.go b/controllers/route.go index 64871dec..686f3779 100644 --- a/controllers/route.go +++ b/controllers/route.go @@ -9,12 +9,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" "reflect" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" ) -func (r *TrustyAIServiceReconciler) createRouteObject(cr *trustyaiopendatahubiov1alpha1.TrustyAIService) (*routev1.Route, error) { +func (r *TrustyAIServiceReconciler) createRouteObjectNoAuth(cr *trustyaiopendatahubiov1alpha1.TrustyAIService) (*routev1.Route, error) { labels := getCommonLabels(cr.Name) route := &routev1.Route{ @@ -35,7 +37,7 @@ func (r *TrustyAIServiceReconciler) createRouteObject(cr *trustyaiopendatahubiov }, }, TLS: &routev1.TLSConfig{ - Termination: routev1.TLSTerminationEdge, + Termination: routev1.TLSTerminationReencrypt, InsecureEdgeTerminationPolicy: routev1.InsecureEdgeTerminationPolicyRedirect, }, }, @@ -49,7 +51,7 @@ func (r *TrustyAIServiceReconciler) createRouteObject(cr *trustyaiopendatahubiov } func (r *TrustyAIServiceReconciler) reconcileRoute(cr *trustyaiopendatahubiov1alpha1.TrustyAIService, ctx context.Context) error { - createdRoute, err := r.createRouteObject(cr) + createdRoute, err := r.createRouteObjectNoAuth(cr) if err != nil { log.FromContext(ctx).Error(err, "Error creating Route object.") return err @@ -89,6 +91,82 @@ func (r *TrustyAIServiceReconciler) reconcileRoute(cr *trustyaiopendatahubiov1al return nil } +func (r *TrustyAIServiceReconciler) createRouteObject(instance *trustyaiopendatahubiov1alpha1.TrustyAIService) *routev1.Route { + return &routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name, + Namespace: instance.Namespace, + Labels: map[string]string{ + "trustyai-service-name": instance.Name, + }, + }, + Spec: routev1.RouteSpec{ + To: routev1.RouteTargetReference{ + Kind: "Service", + Name: instance.Name + "-tls", + Weight: pointer.Int32Ptr(100), + }, + Port: &routev1.RoutePort{ + TargetPort: intstr.FromString(OAuthServicePortName), + }, + TLS: &routev1.TLSConfig{ + Termination: routev1.TLSTerminationPassthrough, + }, + }, + Status: routev1.RouteStatus{ + Ingress: []routev1.RouteIngress{}, + }, + } +} + +// Reconcile will manage the creation, update and deletion of the route returned +// by the newRoute function +func (r *TrustyAIServiceReconciler) reconcileRouteAuth(instance *trustyaiopendatahubiov1alpha1.TrustyAIService, + ctx context.Context, newRoute func(*trustyaiopendatahubiov1alpha1.TrustyAIService) *routev1.Route) error { + + // Generate the desired route + desiredRoute := newRoute(instance) + + // Create the route if it does not already exist + foundRoute := &routev1.Route{} + //justCreated := false + err := r.Get(ctx, types.NamespacedName{ + Name: desiredRoute.Name, + Namespace: instance.Namespace, + }, foundRoute) + if err != nil { + if errors.IsNotFound(err) { + log.FromContext(ctx).Info("Creating Route") + // Add .metatada.ownerReferences to the route to be deleted by the + // Kubernetes garbage collector if the service is deleted + err = ctrl.SetControllerReference(instance, desiredRoute, r.Scheme) + if err != nil { + log.FromContext(ctx).Error(err, "Unable to add OwnerReference to the Route") + return err + } + // Create the route in the Openshift cluster + err = r.Create(ctx, desiredRoute) + if err != nil && !errors.IsAlreadyExists(err) { + log.FromContext(ctx).Error(err, "Unable to create the Route") + return err + } + //justCreated = true + } else { + log.FromContext(ctx).Error(err, "Unable to fetch the Route") + return err + } + } + + return nil +} + +// ReconcileRoute will manage the creation, update and deletion of the +// TLS route when the service is reconciled +func (r *TrustyAIServiceReconciler) ReconcileRoute( + instance *trustyaiopendatahubiov1alpha1.TrustyAIService, ctx context.Context) error { + return r.reconcileRouteAuth(instance, ctx, r.createRouteObject) +} + func (r *TrustyAIServiceReconciler) checkRouteReady(ctx context.Context, cr *trustyaiopendatahubiov1alpha1.TrustyAIService) (bool, error) { existingRoute := &routev1.Route{} diff --git a/controllers/route_test.go b/controllers/route_test.go index 98d8f7dc..0a17c29f 100644 --- a/controllers/route_test.go +++ b/controllers/route_test.go @@ -9,7 +9,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" - "time" ) var _ = Describe("Route Reconciliation", func() { @@ -30,9 +29,9 @@ var _ = Describe("Route Reconciliation", func() { namespace := "route-test-namespace-1" instance = createDefaultCR(namespace) - Eventually(func() error { + WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create namespace") + }, "failed to create namespace") err := reconciler.reconcileRoute(instance, ctx) Expect(err).ToNot(HaveOccurred()) @@ -50,15 +49,15 @@ var _ = Describe("Route Reconciliation", func() { It("Should not update Route", func() { namespace := "route-test-namespace-2" instance = createDefaultCR(namespace) - Eventually(func() error { + WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create namespace") + }, "failed to create namespace") // Create a Route with the expected spec - existingRoute, _ := reconciler.createRouteObject(instance) + existingRoute := reconciler.createRouteObject(instance) Expect(reconciler.Client.Create(ctx, existingRoute)).To(Succeed()) - err := reconciler.reconcileRoute(instance, ctx) + err := reconciler.reconcileRouteAuth(instance, ctx, reconciler.createRouteObject) Expect(err).ToNot(HaveOccurred()) // Fetch the Route diff --git a/controllers/service_accounts.go b/controllers/service_accounts.go new file mode 100644 index 00000000..156ec226 --- /dev/null +++ b/controllers/service_accounts.go @@ -0,0 +1,138 @@ +package controllers + +import ( + "context" + trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/json" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" +) + +func generateServiceAccountName(instance *trustyaiopendatahubiov1alpha1.TrustyAIService) string { + return instance.Name + "-proxy" +} + +// createServiceAccount creates a service account for this instance's OAuth proxy +func (r *TrustyAIServiceReconciler) createServiceAccount(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService) error { + routeName := instance.Name + serviceAccountName := generateServiceAccountName(instance) + + // Define the OAuth redirect reference + oauthRedirectRef := struct { + Kind string `json:"kind"` + APIVersion string `json:"apiVersion"` + Reference struct { + Kind string `json:"kind"` + Name string `json:"name"` + } `json:"reference"` + }{ + Kind: "OAuthRedirectReference", + APIVersion: "v1", + Reference: struct { + Kind string `json:"kind"` + Name string `json:"name"` + }{ + Kind: "Route", + Name: routeName, + }, + } + + // Marshal the struct into JSON format for the annotation + oauthRedirectRefJSON, err := json.Marshal(oauthRedirectRef) + + if err != nil { + // Handle error + return err + } + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceAccountName, + Namespace: instance.Namespace, + Annotations: map[string]string{ + "serviceaccounts.openshift.io/oauth-redirectreference.primary": string(oauthRedirectRefJSON), + }, + Labels: map[string]string{ + "app": componentName, + "app.kubernetes.io/name": serviceAccountName, + "app.kubernetes.io/instance": instance.Name, + "app.kubernetes.io/part-of": componentName, + "app.kubernetes.io/version": "0.1.0", + }, + }, + } + + // Set instance as the owner and controller + if err := ctrl.SetControllerReference(instance, sa, r.Scheme); err != nil { + return err + } + + // Check if this ServiceAccount already exists + found := &corev1.ServiceAccount{} + err = r.Get(ctx, types.NamespacedName{Name: sa.Name, Namespace: sa.Namespace}, found) + if err != nil && errors.IsNotFound(err) { + log.FromContext(ctx).Info("Creating a new ServiceAccount", "Namespace", sa.Namespace, "Name", sa.Name) + err = r.Create(ctx, sa) + if err != nil { + return err + } + } else if err != nil { + return err + } + + err = r.createClusterRoleBinding(ctx, instance, serviceAccountName) + if err != nil { + return err + } + + return nil +} + +// createClusterRoleBinding creates a binding between the service account and token review cluster role +func (r *TrustyAIServiceReconciler) createClusterRoleBinding(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, serviceAccountName string) error { + clusterRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name + "-proxy-rolebinding", + Namespace: instance.Namespace, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: serviceAccountName, + Namespace: instance.Namespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "trustyai-service-operator-proxy-role", + APIGroup: rbacv1.GroupName, + }, + } + + // Set instance as the owner of the ClusterRoleBinding + if err := controllerutil.SetControllerReference(instance, clusterRoleBinding, r.Scheme); err != nil { + return err + } + + // Check if this ClusterRoleBinding already exists + found := &rbacv1.ClusterRoleBinding{} + err := r.Get(ctx, types.NamespacedName{Name: clusterRoleBinding.Name}, found) + if err != nil && errors.IsNotFound(err) { + log.FromContext(ctx).Info("Creating a new ClusterRoleBinding", "Name", clusterRoleBinding.Name) + err = r.Create(ctx, clusterRoleBinding) + if err != nil { + return err + } + } else if err != nil { + return err + } + + return nil +} diff --git a/controllers/statuses_test.go b/controllers/statuses_test.go index 46f778ce..1521920d 100644 --- a/controllers/statuses_test.go +++ b/controllers/statuses_test.go @@ -11,7 +11,6 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "time" ) func checkCondition(conditions []trustyaiopendatahubiov1alpha1.Condition, conditionType string, expectedStatus corev1.ConditionStatus, allowMissing bool) (*trustyaiopendatahubiov1alpha1.Condition, bool, error) { @@ -46,9 +45,9 @@ var _ = Describe("Status and condition tests", func() { namespace := "statuses-test-namespace-1" instance = createDefaultCR(namespace) - Eventually(func() error { + WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create namespace") + }, "failed to create namespace") // Call the reconcileStatuses function _, _ = reconciler.reconcileStatuses(ctx, instance) @@ -73,45 +72,44 @@ var _ = Describe("Status and condition tests", func() { It("Should be available", func() { namespace := "statuses-test-namespace-2" instance = createDefaultCR(namespace) - Eventually(func() error { + WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create namespace") - Eventually(func() error { + }, "failed to create namespace") + WaitFor(func() error { return reconciler.reconcileRoute(instance, ctx) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create route") - Eventually(func() error { + }, "failed to create route") + WaitFor(func() error { return makeRouteReady(ctx, k8sClient, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to make route ready") - Eventually(func() error { + }, "failed to make route ready") + WaitFor(func() error { return reconciler.ensurePVC(ctx, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create PVC") - Eventually(func() error { + }, "failed to create PVC") + WaitFor(func() error { return makePVCReady(ctx, k8sClient, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to bind PVC") - Eventually(func() error { + }, "failed to bind PVC") + WaitFor(func() error { return reconciler.ensureDeployment(ctx, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create deployment") - Eventually(func() error { + }, "failed to create deployment") + WaitFor(func() error { return makeDeploymentReady(ctx, k8sClient, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to make deployment ready") - - Eventually(func() error { + }, "failed to make deployment ready") + WaitFor(func() error { return k8sClient.Create(ctx, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create TrustyAIService") + }, "failed to create TrustyAIService") // Call the reconcileStatuses function - Eventually(func() error { + WaitFor(func() error { _, err := reconciler.reconcileStatuses(ctx, instance) return err - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to update statuses") + }, "failed to update statuses") // Fetch the updated instance - Eventually(func() error { + WaitFor(func() error { return k8sClient.Get(ctx, types.NamespacedName{ Name: instance.Name, Namespace: instance.Namespace, }, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to get updated instance") + }, "failed to get updated instance") readyCondition, statusMatch, err := checkCondition(instance.Status.Conditions, "Ready", corev1.ConditionTrue, true) Expect(err).NotTo(HaveOccurred(), "Error checking Ready condition") @@ -146,52 +144,52 @@ var _ = Describe("Status and condition tests", func() { It("Should be available", func() { namespace := "statuses-test-namespace-2" instance = createDefaultCR(namespace) - Eventually(func() error { + WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create namespace") - Eventually(func() error { + }, "failed to create namespace") + WaitFor(func() error { return reconciler.reconcileRoute(instance, ctx) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create route") - Eventually(func() error { + }, "failed to create route") + WaitFor(func() error { return makeRouteReady(ctx, k8sClient, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to make route ready") - Eventually(func() error { + }, "failed to make route ready") + WaitFor(func() error { return reconciler.ensurePVC(ctx, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create PVC") - Eventually(func() error { + }, "failed to create PVC") + WaitFor(func() error { return makePVCReady(ctx, k8sClient, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to bind PVC") - Eventually(func() error { + }, "failed to bind PVC") + WaitFor(func() error { return reconciler.ensureDeployment(ctx, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create deployment") - Eventually(func() error { + }, "failed to create deployment") + WaitFor(func() error { return makeDeploymentReady(ctx, k8sClient, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to make deployment ready") + }, "failed to make deployment ready") inferenceService := createInferenceService("my-model", namespace) - Eventually(func() error { + WaitFor(func() error { return k8sClient.Create(ctx, inferenceService) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create InferenceService") + }, "failed to create InferenceService") Expect(reconciler.patchKServe(ctx, instance, *inferenceService, namespace, instance.Name, false)).ToNot(HaveOccurred()) - Eventually(func() error { + WaitFor(func() error { return k8sClient.Create(ctx, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create TrustyAIService") + }, "failed to create TrustyAIService") // Call the reconcileStatuses function - Eventually(func() error { + WaitFor(func() error { _, err := reconciler.reconcileStatuses(ctx, instance) return err - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to update statuses") + }, "failed to update statuses") // Fetch the updated instance - Eventually(func() error { + WaitFor(func() error { return k8sClient.Get(ctx, types.NamespacedName{ Name: instance.Name, Namespace: instance.Namespace, }, instance) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to get updated instance") + }, "failed to get updated instance") readyCondition, statusMatch, err := checkCondition(instance.Status.Conditions, "Ready", corev1.ConditionTrue, true) Expect(err).NotTo(HaveOccurred(), "Error checking Ready condition") diff --git a/controllers/storage_test.go b/controllers/storage_test.go index f026f7a5..f11e6394 100644 --- a/controllers/storage_test.go +++ b/controllers/storage_test.go @@ -12,7 +12,6 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "time" ) var _ = Describe("PVC Reconciliation", func() { @@ -33,9 +32,9 @@ var _ = Describe("PVC Reconciliation", func() { It("should create a new PVC and emit an event", func() { namespace := "pvc-test-namespace-1" instance = createDefaultCR(namespace) - Eventually(func() error { + WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create namespace") + }, "failed to create namespace") err := reconciler.ensurePVC(ctx, instance) Expect(err).ToNot(HaveOccurred()) @@ -56,9 +55,9 @@ var _ = Describe("PVC Reconciliation", func() { It("should not attempt to create the PVC", func() { namespace := "pvc-test-namespace-2" instance = createDefaultCR(namespace) - Eventually(func() error { + WaitFor(func() error { return createNamespace(ctx, k8sClient, namespace) - }, time.Second*10, time.Millisecond*250).Should(Succeed(), "failed to create namespace") + }, "failed to create namespace") // Simulate existing PVC existingPVC := &corev1.PersistentVolumeClaim{ diff --git a/controllers/suite_test.go b/controllers/suite_test.go index e49632da..feb89dae 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -19,6 +19,10 @@ package controllers import ( "context" "fmt" + "path/filepath" + "testing" + "time" + "github.com/google/uuid" kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" @@ -37,14 +41,11 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "k8s.io/utils/pointer" - "path/filepath" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "testing" - "time" //+kubebuilder:scaffold:imports ) @@ -66,12 +67,24 @@ const ( operatorNamespace = "system" ) +const ( + defaultTimeout = time.Second * 10 + defaultPolling = time.Millisecond * 250 +) + func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Controller Suite") } +// WaitFor is a function that takes a function which returns an error, and an error message. +// It will repeatedly call the provided function until it succeeds or the timeout is reached. +func WaitFor(operation func() error, errorMsg string) { + Eventually(operation, defaultTimeout, defaultPolling).Should(Succeed(), errorMsg) +} + +// createDefaultCR creates a TrustyAIService instance with default values func createDefaultCR(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.TrustyAIService { service := trustyaiopendatahubiov1alpha1.TrustyAIService{ ObjectMeta: metav1.ObjectMeta{ @@ -97,6 +110,7 @@ func createDefaultCR(namespaceCurrent string) *trustyaiopendatahubiov1alpha1.Tru return &service } +// createNamespace creates a new namespace func createNamespace(ctx context.Context, k8sClient client.Client, namespace string) error { ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -117,6 +131,21 @@ func createNamespace(ctx context.Context, k8sClient client.Client, namespace str return nil } +// createConfigMap creates a configuration in the specified namespace +func createConfigMap(namespace string, oauthImage string, trustyaiServiceImage string) *corev1.ConfigMap { + // Define the ConfigMap with the necessary data + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: imageConfigMap, + Namespace: namespace, + }, + Data: map[string]string{ + configMapOAuthProxyImageKey: oauthImage, + configMapServiceImageKey: trustyaiServiceImage, + }, + } +} + func createMockPV(ctx context.Context, k8sClient client.Client, pvName string, size string) error { pv := &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ @@ -186,27 +215,6 @@ func createTestPVC(ctx context.Context, k8sClient client.Client, instance *trust return nil } -func removeFinalizerAndDeleteInstance(ctx context.Context, k8sClient client.Client, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, finalizerName string) { - // Get the latest state of the TrustyAIService instance - _ = k8sClient.Get(ctx, client.ObjectKey{Name: instance.Name, Namespace: instance.Namespace}, instance) - - // Remove the finalizer from the TrustyAIService instance - finalizerIndex := -1 - for i, f := range instance.Finalizers { - if f == finalizerName { - finalizerIndex = i - break - } - } - if finalizerIndex >= 0 { - instance.Finalizers = append(instance.Finalizers[:finalizerIndex], instance.Finalizers[finalizerIndex+1:]...) - _ = k8sClient.Update(ctx, instance) - } - - // Delete the TrustyAIService instance - _ = k8sClient.Delete(ctx, instance) -} - // createInferenceService Function to create the InferenceService func createInferenceService(name string, namespace string) *kservev1beta1.InferenceService { return &kservev1beta1.InferenceService{ @@ -226,89 +234,6 @@ func createInferenceService(name string, namespace string) *kservev1beta1.Infere } } -// createDeploymentWithInferenceService creates a Deployment with multiple containers -func createDeploymentWithInferenceService(ctx context.Context, k8sClient client.Client, name string, namespace string, inferenceServiceName string) error { - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: map[string]string{ - "app": name, - "app.kubernetes.io/part-of": inferenceServiceName, // Label to associate with the InferenceService - }, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: pointer.Int32Ptr(1), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": name, - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": name, - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "main-container", - Image: "main-container-image", // Mock container - }, - { - Name: "helper-container", - Image: "helper-container-image", // Mock container - }, - }, - }, - }, - }, - } - - if err := k8sClient.Create(ctx, deployment); err != nil { - return err - } - - return nil -} - -func checkTrustyAIServiceCondition(client client.Client, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, expectedType string, expectedStatus corev1.ConditionStatus, expectedReason string) error { - err := client.Get(context.TODO(), types.NamespacedName{ - Namespace: instance.Namespace, - Name: instance.Name, - }, instance) - - if err != nil { - return err - } - - for _, condition := range instance.Status.Conditions { - if condition.Type == expectedType && condition.Status == expectedStatus && condition.Reason == expectedReason { - return nil // Condition matches expectations - } - } - - return fmt.Errorf("Condition did not match expectations. Expected Type: %s, Status: %s, Reason: %s", expectedType, expectedStatus, expectedReason) -} - -func checkTrustyAIServiceReadyStatus(client client.Client, instance *trustyaiopendatahubiov1alpha1.TrustyAIService, expectedStatus corev1.ConditionStatus) error { - err := client.Get(context.TODO(), types.NamespacedName{ - Namespace: instance.Namespace, - Name: instance.Name, - }, instance) - - if err != nil { - return err - } - - if instance.Status.Ready == expectedStatus { - return nil // Ready status matches expectations - } - - return fmt.Errorf("Ready status did not match expectations. Expected: %s, Actual: %s", expectedStatus, instance.Status.Ready) -} - func makePVCReady(ctx context.Context, k8sClient client.Client, instance *trustyaiopendatahubiov1alpha1.TrustyAIService) error { pvc := &corev1.PersistentVolumeClaim{} pvcName := types.NamespacedName{ diff --git a/controllers/trustyaiservice_controller.go b/controllers/trustyaiservice_controller.go index a3d5a7c0..a6157ead 100644 --- a/controllers/trustyaiservice_controller.go +++ b/controllers/trustyaiservice_controller.go @@ -19,17 +19,16 @@ package controllers import ( "context" goerrors "errors" - "fmt" + "time" + kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" @@ -37,7 +36,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/source" - "time" ) var ErrPVCNotReady = goerrors.New("PVC is not ready") @@ -68,6 +66,9 @@ type TrustyAIServiceReconciler struct { //+kubebuilder:rbac:groups=serving.kserve.io,resources=inferenceservices,verbs=list;watch;get;update;patch //+kubebuilder:rbac:groups=serving.kserve.io,resources=inferenceservices/finalizers,verbs=list;watch;get;update;patch;delete //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch;update +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;delete +//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;list;watch;create;update;delete // getCommonLabels returns the service's common labels func getCommonLabels(serviceName string) map[string]string { @@ -126,6 +127,16 @@ func (r *TrustyAIServiceReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } + err = r.createServiceAccount(ctx, instance) + if err != nil { + return RequeueWithError(err) + } + + err = r.reconcileOAuthService(ctx, instance) + if err != nil { + return RequeueWithError(err) + } + // CR found, add or update the URL // Call the function to patch environment variables for Deployments that match the label shouldContinue, err := r.handleInferenceServices(ctx, instance, req.Namespace, modelMeshLabelKey, modelMeshLabelValue, payloadProcessorName, req.Name, false) @@ -171,7 +182,7 @@ func (r *TrustyAIServiceReconciler) Reconcile(ctx context.Context, req ctrl.Requ return RequeueWithError(err) } if err := r.Create(ctx, service); err != nil { - if apierrors.IsAlreadyExists(err) { + if errors.IsAlreadyExists(err) { // Service already exists, no problem } else { // handle any other error @@ -192,7 +203,9 @@ func (r *TrustyAIServiceReconciler) Reconcile(ctx context.Context, req ctrl.Requ } // Create route - err = r.reconcileRoute(instance, ctx) + // TODO: Change argument order + err = r.ReconcileRoute(instance, ctx) + //err = r.reconcileRoute(instance, ctx) if err != nil { // Could not create Route object, update status and return. _, updateErr := r.updateStatus(ctx, instance, UpdateRouteNotAvailable) @@ -270,40 +283,3 @@ func (r *TrustyAIServiceReconciler) SetupWithManager(mgr ctrl.Manager) error { Watches(&source.Kind{Type: &kservev1alpha1.ServingRuntime{}}, &handler.EnqueueRequestForObject{}). Complete(r) } - -// getTrustyAIImageAndTagFromConfigMap gets a custom TrustyAI image and tag from a ConfigMap in the operator's namespace -func (r *TrustyAIServiceReconciler) getImageFromConfigMap(ctx context.Context) (string, error) { - if r.Namespace != "" { - // Define the key for the ConfigMap - configMapKey := types.NamespacedName{ - Namespace: r.Namespace, - Name: "trustyai-service-operator-config", - } - - // Create an empty ConfigMap object - var cm corev1.ConfigMap - - // Try to get the ConfigMap - if err := r.Get(ctx, configMapKey, &cm); err != nil { - if errors.IsNotFound(err) { - // ConfigMap not found, fallback to default values - return defaultImage, nil - } - // Other error occurred when trying to fetch the ConfigMap - return defaultImage, fmt.Errorf("error reading configmap %s", configMapKey) - } - - // ConfigMap is found, extract the image and tag - image, ok := cm.Data["trustyaiServiceImage"] - - if !ok { - // One or both of the keys are not present in the ConfigMap, return error - return defaultImage, fmt.Errorf("configmap %s does not contain necessary keys", configMapKey) - } - - // Return the image and tag - return image, nil - } else { - return defaultImage, nil - } -} diff --git a/go.mod b/go.mod index 15fd2572..e341df1f 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/s2a-go v0.1.4 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.0 github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 // indirect @@ -92,7 +92,7 @@ require ( k8s.io/component-base v0.26.4 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230515203736-54b630e78af5 // indirect - k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect + k8s.io/utils v0.0.0-20230505201702-9f6742963106 knative.dev/networking v0.0.0-20230511122402-33636d99d870 // indirect knative.dev/pkg v0.0.0-20230502134655-db8a35330281 // indirect knative.dev/serving v0.37.1 // indirect From ce98f1631ae4d742b11dda29c0ad542535f5082c Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Wed, 13 Dec 2023 15:24:43 +0000 Subject: [PATCH 4/5] Remove obsolete patches (#172) --- config/base/kustomization.yaml | 3 --- config/base/manager_auth_proxy_patch.yaml | 15 --------------- config/base/manager_config_patch.yaml | 11 ----------- 3 files changed, 29 deletions(-) delete mode 100644 config/base/manager_auth_proxy_patch.yaml delete mode 100644 config/base/manager_config_patch.yaml diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml index e1305dc6..ab0b9738 100644 --- a/config/base/kustomization.yaml +++ b/config/base/kustomization.yaml @@ -9,9 +9,6 @@ resources: commonLabels: app.kubernetes.io/part-of: trustyai -patchesStrategicMerge: - - manager_auth_proxy_patch.yaml - configMapGenerator: - env: params.env name: config diff --git a/config/base/manager_auth_proxy_patch.yaml b/config/base/manager_auth_proxy_patch.yaml deleted file mode 100644 index 2935b633..00000000 --- a/config/base/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=0.0.0.0:8080" - - "--leader-elect" diff --git a/config/base/manager_config_patch.yaml b/config/base/manager_config_patch.yaml deleted file mode 100644 index ee54324f..00000000 --- a/config/base/manager_config_patch.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager From dc5d489eb8df7e61d968574f6407da8f7c09cc53 Mon Sep 17 00:00:00 2001 From: Rui Vieira Date: Thu, 11 Jan 2024 11:27:58 +0000 Subject: [PATCH 5/5] Correct operator's instance label (#176) --- config/manager/manager.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 058fa9bc..dfd88a06 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -6,7 +6,7 @@ metadata: labels: control-plane: controller-manager app.kubernetes.io/name: deployment - app.kubernetes.io/instance: controller-manager + app.kubernetes.io/instance: trustyai-service-operator-controller-manager app.kubernetes.io/component: manager app.kubernetes.io/created-by: trustyai-service-operator app.kubernetes.io/part-of: trustyai-service-operator