RHCLOUD-36175 - Kind cluster setup for inventory (#226)

* Kind cluster setup for inventory * add github workflow & remove service account from kind setup * replace quay images with local inventory image * fixup kessel-deploy.yaml
project-kessel · Nov 14, 2024 · 063368e · 063368e
1 parent fd4d905
commit 063368e
Show file tree

Hide file tree

Showing 9 changed files with 354 additions and 2 deletions.
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
@@ -0,0 +1,29 @@
+name: E2E Testing with Kind
+
+on:
+  workflow_call:
+  pull_request:
+    branches: ['main']
+
+jobs:
+  e2e-test:
+    name: E2E Test Inventory API
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Inventory Up - Kind Cluster
+        run: make inventory-up-kind
+
+      - name: Monitor Pods in Kind
+        run: |
+          timeout 90s kubectl get pods -w || exit 0
+
+      - name: View Test Pod Logs
+        run: |
+          TEST_POD=$(kubectl get pods --selector=job-name=e2e-inventory-http-tests -o jsonpath='{.items[0].metadata.name}')
+          kubectl logs $TEST_POD
+
+      - name: Inventory Down - Kind Cluster
+        run: make inventory-down-kind
+
diff --git a/Makefile b/Makefile
@@ -131,6 +131,10 @@ inventory-up-sso:
 inventory-up-kafka:
 	./scripts/start-inventory-kafka.sh
 
+.PHONY: inventory-up-kind
+inventory-up-kind:
+	./scripts/start-inventory-kind.sh
+
 .PHONY: get-token
 get-token:
 	./scripts/get-token.sh
@@ -147,6 +151,10 @@ inventory-down-sso:
 inventory-down-kafka:
 	./scripts/stop-inventory-kafka.sh
 
+.PHONY: inventory-down-kind
+inventory-down-kind:
+	./scripts/stop-inventory-kind.sh
+
 .PHONY: run
 # run api locally
 run: build

diff --git a/deploy/kind/e2e/e2e-batch.yaml b/deploy/kind/e2e/e2e-batch.yaml
@@ -0,0 +1,31 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: e2e-inventory-http-tests
+spec:
+  template:
+    spec:
+      initContainers:
+        - name: wait-for-inventory
+          image: curlimages/curl:latest
+          command:
+            - /bin/sh
+            - "-c"
+            - |
+              echo "Waiting for kessel-inventory-service to be ready..."
+              while ! curl -s http://kessel-inventory-service/api/inventory/v1/readyz; do
+                echo "Inventory service not ready yet. Sleeping for 5 seconds..."
+                sleep 5
+              done
+              echo "Inventory service is ready!"
+      containers:
+        - name: e2e-http-tests
+          image: localhost/inventory-e2e-tests:e2e-test
+          env:
+            - name: INV_HTTP_URL
+              value: "kessel-inventory-service:80"
+            - name: KAFKA_BOOTSTRAP_SERVERS
+              value: "localhost:9092"
+          command: ["/usr/local/bin/e2e-inventory-tests", "-test.v"]
+      restartPolicy: Never
+  backoffLimit: 4
diff --git a/deploy/kind/inventory/invdatabase.yaml b/deploy/kind/inventory/invdatabase.yaml
@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: invdatabase
+  labels:
+    app: invdatabase
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: invdatabase
+  template:
+    metadata:
+      labels:
+        app: invdatabase
+    spec:
+      containers:
+        - name: postgres
+          image: postgres:latest
+          env:
+            - name: POSTGRES_DB
+              value: spicedb
+            - name: POSTGRES_USER
+              value: postgres
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: inventory-api-config
+                  key: db_password
+          ports:
+            - containerPort: 5432
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: invdatabase
+  labels:
+    app: invdatabase
+spec:
+  ports:
+    - port: 5433
+      targetPort: 5432
+  selector:
+    app: invdatabase
+  type: ClusterIP
diff --git a/deploy/kind/inventory/kessel-inventory.yaml b/deploy/kind/inventory/kessel-inventory.yaml
@@ -0,0 +1,163 @@
+# Deployment
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kessel-inventory
+  labels:
+    app: kessel-inventory
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kessel-inventory
+  template:
+    metadata:
+      labels:
+        app: kessel-inventory
+    spec:
+      initContainers:
+        - name: migration
+          image: localhost/inventory-api:e2e-test
+          command:
+            - /bin/sh
+            - "-c"
+            - |
+              echo "Waiting for PostgreSQL to be ready..."
+              sleep 20 
+              
+              inventory-api migrate
+          env:
+            - name: CLOWDER_ENABLED
+              value: "true"
+            - name: INVENTORY_API_CONFIG
+              value: "/inventory/inventory-api-config.yaml"
+            - name: PGDATA
+              value: /temp/data
+            - name: POSTGRES_HOST
+              value: "invdatabase"
+            - name: POSTGRES_PORT
+              value: "5433"
+            - name: POSTGRES_USER
+              value: "postgres"
+            - name: POSTGRES_DB
+              value: "spicedb"
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: inventory-api-config
+                  key: db_password
+          volumeMounts:
+            - name: config-volume
+              mountPath: "/inventory"
+      containers:
+        - name: api
+          image: localhost/inventory-api:e2e-test
+          command: ["inventory-api"]
+          args: ["serve"]
+          env:
+            - name: CLOWDER_ENABLED
+              value: "true"
+            - name: INVENTORY_API_CONFIG
+              value: "/inventory/inventory-api-config.yaml"
+            - name: PGDATA
+              value: /temp/data
+            - name: POSTGRES_HOST
+              value: "invdatabase"
+            - name: POSTGRES_PORT
+              value: "5433"
+            - name: POSTGRES_USER
+              value: "postgres"
+            - name: POSTGRES_DB
+              value: "spicedb"
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: inventory-api-config
+                  key: db_password
+          livenessProbe:
+            httpGet:
+              path: /api/inventory/v1/livez
+              port: 8081
+          readinessProbe:
+            httpGet:
+              path: /api/inventory/v1/readyz
+              port: 8081
+          volumeMounts:
+            - name: psks-volume
+              mountPath: "/psks.yaml"
+              subPath: psks.yaml
+            - name: config-volume
+              mountPath: "/inventory"
+      volumes:
+        - name: config-volume
+          secret:
+            secretName: inventory-api-config
+        - name: psks-volume
+          configMap:
+            name: inventory-api-psks
+---
+# Services
+apiVersion: v1
+kind: Service
+metadata:
+  name: kessel-inventory-service
+  labels:
+    app: kessel-inventory
+spec:
+  selector:
+    app: kessel-inventory
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8081
+---
+# PodDisruptionBudget
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: kessel-inventory-api-pdb
+  labels:
+    app: kessel-inventory
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: kessel-inventory
+---
+# Secret
+apiVersion: v1
+kind: Secret
+metadata:
+  name: inventory-api-config
+type: Opaque
+stringData:
+  inventory-api-config.yaml: |
+    server:
+      http:
+        address: 0.0.0.0:8081
+      grpc:
+        address: 0.0.0.0:9081
+    authn:
+      psk:
+        pre-shared-key-file: /psks.yaml
+    authz:
+      impl: allow-all
+    eventing:
+      eventer: stdout
+      kafka:
+    storage:
+      database: postgres
+      sqlite3:
+        dsn: inventory.db
+      postgres:
+        host: "invdatabase"
+        port: "5433"
+        user: "postgres"
+        password: "yPsw5e6ab4bvAGe5H"
+        dbname: "spicedb"
+    log:
+      level: "debug"
+      livez: true
+      readyz: true
+  db_password: "yPsw5e6ab4bvAGe5H"
+
diff --git a/deploy/kind/inventory/strimzi.yaml b/deploy/kind/inventory/strimzi.yaml
@@ -0,0 +1,35 @@
+# Taken from: https://github.com/strimzi/strimzi-kafka-operator/blob/main/examples/kafka/kafka-ephemeral.yaml
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: my-cluster
+spec:
+  kafka:
+    version: 3.8.0
+    replicas: 1
+    listeners:
+      - name: plain
+        port: 9092
+        type: internal
+        tls: false
+      - name: tls
+        port: 9093
+        type: internal
+        tls: true
+    config:
+      offsets.topic.replication.factor: 3
+      transaction.state.log.replication.factor: 3
+      transaction.state.log.min.isr: 2
+      default.replication.factor: 3
+      min.insync.replicas: 2
+      inter.broker.protocol.version: "3.8"
+    storage:
+      type: ephemeral
+  zookeeper:
+    replicas: 1
+    storage:
+      type: ephemeral
+  entityOperator:
+    topicOperator: {}
+    userOperator: {}
+
diff --git a/scripts/start-inventory-kind.sh b/scripts/start-inventory-kind.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+
+source ./scripts/check_docker_podman.sh
+
+kind create cluster --name inventory-cluster
+
+# build/tag image
+${DOCKER} build -t localhost/inventory-api:latest -f Dockerfile .
+${DOCKER} build -t localhost/inventory-e2e-tests:latest -f Dockerfile-e2e .
+
+${DOCKER} tag localhost/inventory-api:latest localhost/inventory-api:e2e-test
+${DOCKER} tag localhost/inventory-e2e-tests:latest localhost/inventory-e2e-tests:e2e-test
+
+rm -rf inventory-api.tar
+rm -rf inventory-e2e-tests.tar
+
+${DOCKER} save -o inventory-api.tar localhost/inventory-api:e2e-test
+${DOCKER} save -o inventory-e2e-tests.tar localhost/inventory-e2e-tests:e2e-test
+
+kind load image-archive inventory-api.tar --name inventory-cluster
+kind load image-archive inventory-e2e-tests.tar --name inventory-cluster
+
+kubectl create configmap inventory-api-psks --from-file=config/psks.yaml
+
+kubectl apply -f https://strimzi.io/install/latest\?namespace\=default
+kubectl apply -f deploy/kind/inventory/kessel-inventory.yaml
+kubectl apply -f deploy/kind/inventory/invdatabase.yaml
+kubectl apply -f deploy/kind/e2e/e2e-batch.yaml
+kubectl apply -f deploy/kind/inventory/strimzi.yaml
diff --git a/scripts/stop-inventory-kind.sh b/scripts/stop-inventory-kind.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+set -oe errexit
+
+if ! command -v kind &> /dev/null
+then
+    echo "kind could not be found"
+    exit
+fi
+
+echo "Deleting kind inventory-cluster"
+kind delete clusters inventory-cluster
diff --git a/test/e2e/inventory_http_test.go b/test/e2e/inventory_http_test.go
@@ -113,15 +113,15 @@ func TestInventoryAPIHTTP_Readyz(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NotNil(t, resp)
 
-	expectedStatus := "Storage type sqlite"
+	expectedStatus := "Storage type postgres"
 	expectedCode := uint32(200)
 
 	assert.Equal(t, expectedStatus, resp.Status)
 	assert.Equal(t, expectedCode, resp.Code)
 }
 
 func TestInventoryAPIHTTP_Metrics(t *testing.T) {
-	resp, err := nethttp.Get("http://localhost:8081/metrics")
+	resp, err := nethttp.Get("http://" + inventoryapi_http_url + "/metrics")
 	if err != nil {
 		t.Fatal("Failed to send request: ", err)
 	}