From 13e0bbe999482be193dbca64420308991c4e05e6 Mon Sep 17 00:00:00 2001 From: Chaitanya Baraskar Date: Fri, 14 Jan 2022 16:31:43 +0530 Subject: [PATCH] Added integration test for MSSQL app (#1180) * Added integration test for mssql blueprint * Added clusterrolebinding for service account and cleanup * cleanup * added custom clusterrole specific to required operation and addressing review comments * fixing review comments and added mssql app in integration test * cleanup * typo * removing test image * import order * addressing review comments * removing utils file * lint fix * addressing review comments * moving connection string to constant * review fixes --- build/integration-test.sh | 2 +- examples/stable/mssql/mssql-blueprint.yaml | 1 - pkg/app/mssql.go | 347 ++++++++++++++++++ pkg/blueprint/blueprints/mssql-blueprint.yaml | 1 + pkg/testing/integration_register.go | 14 + pkg/testing/integration_test.go | 104 +++++- 6 files changed, 457 insertions(+), 12 deletions(-) create mode 100644 pkg/app/mssql.go create mode 120000 pkg/blueprint/blueprints/mssql-blueprint.yaml diff --git a/build/integration-test.sh b/build/integration-test.sh index 5f6b2be508..8eeda56663 100755 --- a/build/integration-test.sh +++ b/build/integration-test.sh @@ -25,7 +25,7 @@ TEST_TIMEOUT="30m" # Set default options TEST_OPTIONS="-tags=integration -timeout ${TEST_TIMEOUT} -check.suitep ${DOP}" # Regex to match apps to run in short mode -SHORT_APPS="^PostgreSQL$|^PITRPostgreSQL|^MySQL$|Elasticsearch|^MongoDB$|Maria" +SHORT_APPS="^PostgreSQL$|^PITRPostgreSQL|^MySQL$|Elasticsearch|^MongoDB$|Maria|^MSSQL$" # OCAPPS has all the apps that are to be tested against openshift cluster OC_APPS3_11="MysqlDBDepConfig$|MongoDBDepConfig$|PostgreSQLDepConfig$" OC_APPS4_4="MysqlDBDepConfig4_4|MongoDBDepConfig4_4|PostgreSQLDepConfig4_4" diff --git a/examples/stable/mssql/mssql-blueprint.yaml b/examples/stable/mssql/mssql-blueprint.yaml index 6f32a1c67a..4668eab41e 100644 --- a/examples/stable/mssql/mssql-blueprint.yaml +++ b/examples/stable/mssql/mssql-blueprint.yaml @@ -2,7 +2,6 @@ apiVersion: cr.kanister.io/v1alpha1 kind: Blueprint metadata: name: mssql-blueprint - namespace: kanister actions: backup: outputArtifacts: diff --git a/pkg/app/mssql.go b/pkg/app/mssql.go new file mode 100644 index 0000000000..7486d043fe --- /dev/null +++ b/pkg/app/mssql.go @@ -0,0 +1,347 @@ +package app + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes" + + crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1" + "github.com/kanisterio/kanister/pkg/field" + "github.com/kanisterio/kanister/pkg/kube" + "github.com/kanisterio/kanister/pkg/log" +) + +const ( + mssqlWaitTimeout = 5 * time.Minute + dbUserName = "sa" + dbPass = "MyC0m9l&xP@ssw0rd" + connString = "/opt/mssql-tools/bin/sqlcmd -S localhost -U %s -P \"%s\" -Q " +) + +type MssqlDB struct { + cli kubernetes.Interface + namespace string + name string + deployment *appsv1.Deployment + service *v1.Service + pvc *v1.PersistentVolumeClaim + secret *v1.Secret +} + +func NewMssqlDB(name string) App { + return &MssqlDB{ + name: name, + } +} + +func (m *MssqlDB) ConfigMaps() map[string]crv1alpha1.ObjectReference { + return nil +} + +func (m *MssqlDB) Secrets() map[string]crv1alpha1.ObjectReference { + return map[string]crv1alpha1.ObjectReference{ + "mssql": { + Kind: "Secret", + Name: m.name, + Namespace: m.namespace, + }, + } +} + +func (m *MssqlDB) Init(ctx context.Context) error { + cfg, err := kube.LoadConfig() + if err != nil { + return err + } + + m.cli, err = kubernetes.NewForConfig(cfg) + return err +} + +func (m *MssqlDB) Install(ctx context.Context, namespace string) error { + m.namespace = namespace + secret, err := m.cli.CoreV1().Secrets(namespace).Create(ctx, m.getSecretObj(), metav1.CreateOptions{}) + if err != nil { + return err + } + log.Print("Secret created successfully", field.M{"app": m.name, "secret": secret.Name}) + m.secret = secret + + pvcObj, err := m.getPVCObj() + if err != nil { + return err + } + pvc, err := m.cli.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvcObj, metav1.CreateOptions{}) + if err != nil { + return err + } + log.Print("PVC created successfully", field.M{"app": m.name, "pvc": pvc.Name}) + m.pvc = pvc + + deploymentObj, err := m.getDeploymentObj() + if err != nil { + return err + } + deployment, err := m.cli.AppsV1().Deployments(namespace).Create(ctx, deploymentObj, metav1.CreateOptions{}) + if err != nil { + return err + } + log.Print("Deployment created successfully", field.M{"app": m.name, "deployment": deployment.Name}) + m.deployment = deployment + + serviceObj, err := m.getServiceObj() + if err != nil { + return err + } + service, err := m.cli.CoreV1().Services(namespace).Create(ctx, serviceObj, metav1.CreateOptions{}) + if err != nil { + return err + } + log.Print("Service created successfully", field.M{"app": m.name, "service": service.Name}) + m.service = service + + return nil +} + +func (m *MssqlDB) IsReady(ctx context.Context) (bool, error) { + log.Print("Waiting for the mssql application to be ready.", field.M{"app": m.name}) + ctx, cancel := context.WithTimeout(ctx, mssqlWaitTimeout) + defer cancel() + + err := kube.WaitOnDeploymentReady(ctx, m.cli, m.namespace, m.deployment.Name) + if err != nil { + return false, err + } + log.Print("Application instance is ready.", field.M{"app": m.name}) + return true, nil +} + +func (m *MssqlDB) Object() crv1alpha1.ObjectReference { + return crv1alpha1.ObjectReference{ + Kind: "deployment", + Name: "mssql-deployment", + Namespace: m.namespace, + } +} + +func (m *MssqlDB) Uninstall(ctx context.Context) error { + err := m.cli.AppsV1().Deployments(m.namespace).Delete(ctx, m.deployment.Name, metav1.DeleteOptions{}) + if err != nil { + return err + } + log.Print("Deployment deleted successfully", field.M{"app": m.name}) + + err = m.cli.CoreV1().PersistentVolumeClaims(m.namespace).Delete(ctx, m.pvc.Name, metav1.DeleteOptions{}) + if err != nil { + return err + } + log.Print("PVC deleted successfully", field.M{"app": m.name}) + + err = m.cli.CoreV1().Services(m.namespace).Delete(ctx, m.service.Name, metav1.DeleteOptions{}) + if err != nil { + return err + } + log.Print("Service deleted successfully", field.M{"app": m.name}) + + err = m.cli.CoreV1().Secrets(m.namespace).Delete(ctx, m.secret.Name, metav1.DeleteOptions{}) + if err != nil { + return err + } + log.Print("Secret deleted successfully", field.M{"app": m.name}) + return nil +} + +func (m *MssqlDB) Ping(ctx context.Context) error { + log.Print("Pinging mssql database", field.M{"app": m.name}) + count := fmt.Sprintf(connString+ + "\"SELECT name FROM sys.databases WHERE name NOT IN ('master','model','msdb','tempdb')\" -b -s \",\" -h -1", dbUserName, dbPass) + + loginMssql := []string{"sh", "-c", count} + _, stderr, err := m.execCommand(ctx, loginMssql) + if err != nil { + return errors.Wrapf(err, "Error while Pinging the database: %s", stderr) + } + log.Print("Ping to the application was success.", field.M{"app": m.name}) + return nil +} + +func (m *MssqlDB) Insert(ctx context.Context) error { + log.Print("Adding entry to database", field.M{"app": m.name}) + insert := fmt.Sprintf(connString+ + "\"USE test; INSERT INTO Inventory VALUES (1, 'banana', 150)\"", dbUserName, dbPass) + + insertQuery := []string{"sh", "-c", insert} + _, stderr, err := m.execCommand(ctx, insertQuery) + if err != nil { + return errors.Wrapf(err, "Error while inserting data into table: %s", stderr) + } + return nil +} + +func (m *MssqlDB) Count(ctx context.Context) (int, error) { + log.Print("Counting entries from database", field.M{"app": m.name}) + insert := fmt.Sprintf(connString+ + "\"SET NOCOUNT ON; USE test; SELECT COUNT(*) FROM Inventory\" -h -1", dbUserName, dbPass) + + insertQuery := []string{"sh", "-c", insert} + stdout, stderr, err := m.execCommand(ctx, insertQuery) + if err != nil { + return 0, errors.Wrapf(err, "Error while inserting data into table: %s", stderr) + } + rowsReturned, err := strconv.Atoi(strings.TrimSpace(strings.Split(stdout, "\n")[1])) + if err != nil { + return 0, errors.Wrapf(err, "Error while converting response of count query: %s", stderr) + } + return rowsReturned, nil +} + +func (m *MssqlDB) Reset(ctx context.Context) error { + log.Print("Reseting database", field.M{"app": m.name}) + delete := fmt.Sprintf(connString+"\"DROP DATABASE test\"", dbUserName, dbPass) + deleteQuery := []string{"sh", "-c", delete} + _, stderr, err := m.execCommand(ctx, deleteQuery) + if err != nil { + return errors.Wrapf(err, "Error while inserting data into table: %s", stderr) + } + return nil +} + +func (m *MssqlDB) Initialize(ctx context.Context) error { + log.Print("Initializing database", field.M{"app": m.name}) + createDB := fmt.Sprintf(connString+"\"CREATE DATABASE test\"", dbUserName, dbPass) + + createTable := fmt.Sprintf(connString+ + "\"USE test; CREATE TABLE Inventory (id INT, name NVARCHAR(50), quantity INT)\"", dbUserName, dbPass) + + execQuery := []string{"sh", "-c", createDB} + _, stderr, err := m.execCommand(ctx, execQuery) + if err != nil { + return errors.Wrapf(err, "Error while creating the database: %s", stderr) + } + + execQuery = []string{"sh", "-c", createTable} + _, stderr, err = m.execCommand(ctx, execQuery) + if err != nil { + return errors.Wrapf(err, "Error while creating table: %s", stderr) + } + return nil +} + +func (m *MssqlDB) GetClusterScopedResources(ctx context.Context) []crv1alpha1.ObjectReference { + return nil +} + +func (m MssqlDB) execCommand(ctx context.Context, command []string) (string, string, error) { + podName, containerName, err := kube.GetPodContainerFromDeployment(ctx, m.cli, m.namespace, m.deployment.Name) + if err != nil || podName == "" { + return "", "", errors.Wrapf(err, "Error getting pod and container name for app %s.", m.name) + } + return kube.Exec(m.cli, m.namespace, podName, containerName, command, nil) +} + +func (m *MssqlDB) getDeploymentObj() (*appsv1.Deployment, error) { + deploymentManifest := + `apiVersion: apps/v1 +kind: Deployment +metadata: + name: mssql-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: mssql + template: + metadata: + labels: + app: mssql + spec: + terminationGracePeriodSeconds: 30 + hostname: mssqlinst + securityContext: + fsGroup: 10001 + containers: + - name: mssql + image: mcr.microsoft.com/mssql/server:2019-latest + ports: + - containerPort: 1433 + env: + - name: MSSQL_PID + value: "Developer" + - name: ACCEPT_EULA + value: "Y" + - name: SA_PASSWORD + valueFrom: + secretKeyRef: + name: mssql + key: SA_PASSWORD + volumeMounts: + - name: mssqldb + mountPath: /var/opt/mssql + volumes: + - name: mssqldb + persistentVolumeClaim: + claimName: mssql-data` + + var deployment *appsv1.Deployment + err := yaml.Unmarshal([]byte(deploymentManifest), &deployment) + return deployment, err +} + +func (m *MssqlDB) getPVCObj() (*v1.PersistentVolumeClaim, error) { + pvcmaniFest := + `kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mssql-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi` + + var pvc *v1.PersistentVolumeClaim + err := yaml.Unmarshal([]byte(pvcmaniFest), &pvc) + return pvc, err +} + +func (m *MssqlDB) getServiceObj() (*v1.Service, error) { + serviceManifest := + `apiVersion: v1 +kind: Service +metadata: + name: mssql-deployment +spec: + selector: + app: mssql + ports: + - protocol: TCP + port: 1433 + targetPort: 1433 + type: ClusterIP` + + var service *v1.Service + err := yaml.Unmarshal([]byte(serviceManifest), &service) + return service, err +} + +func (m MssqlDB) getSecretObj() *v1.Secret { + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: m.name, + }, + Data: map[string][]byte{ + "SA_PASSWORD": []byte(dbPass), + }, + Type: "Opaque", + } +} diff --git a/pkg/blueprint/blueprints/mssql-blueprint.yaml b/pkg/blueprint/blueprints/mssql-blueprint.yaml new file mode 120000 index 0000000000..851196f47e --- /dev/null +++ b/pkg/blueprint/blueprints/mssql-blueprint.yaml @@ -0,0 +1 @@ +../../../examples/stable/mssql/mssql-blueprint.yaml \ No newline at end of file diff --git a/pkg/testing/integration_register.go b/pkg/testing/integration_register.go index 07f3052335..ee4771c0f4 100644 --- a/pkg/testing/integration_register.go +++ b/pkg/testing/integration_register.go @@ -217,6 +217,20 @@ var _ = Suite(&RDSPostgreSQLSnap{ }, }) +type MSSQL struct { + IntegrationSuite +} + +var _ = Suite(&MSSQL{ + IntegrationSuite{ + name: "mssql", + namespace: "mssql-test", + app: app.NewMssqlDB("mssql"), + bp: app.NewBlueprint("mssql", "", true), + profile: newSecretProfile(), + }, +}) + // OpenShift apps for version 3.11 // Mysql Instance that is deployed through DeploymentConfig on OpenShift cluster type MysqlDBDepConfig struct { diff --git a/pkg/testing/integration_test.go b/pkg/testing/integration_test.go index a6cf410038..3228116ed0 100644 --- a/pkg/testing/integration_test.go +++ b/pkg/testing/integration_test.go @@ -1,4 +1,6 @@ +//go:build integration // +build integration + // Copyright 2019 The Kanister Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +18,7 @@ package testing import ( - "context" + context "context" "os" test "testing" "time" @@ -24,6 +26,7 @@ import ( "github.com/pkg/errors" . "gopkg.in/check.v1" v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes" @@ -51,10 +54,13 @@ func Test(t *test.T) { // Global variables shared across Suite instances type kanisterKontroller struct { - namespace string - context context.Context - cancel context.CancelFunc - kubeCli *kubernetes.Clientset + namespace string + context context.Context + cancel context.CancelFunc + kubeCli *kubernetes.Clientset + serviceAccount *v1.ServiceAccount + clusterRole *rbacv1.ClusterRole + clusterRoleBinding *rbacv1.ClusterRoleBinding } var kontroller kanisterKontroller @@ -69,10 +75,22 @@ func integrationSetup(t *test.T) { } cli, err := kubernetes.NewForConfig(cfg) if err != nil { - t.Fatalf("Integration test setup failure: Error createing kubeCli; err=%v", err) + t.Fatalf("Integration test setup failure: Error creating kubeCli; err=%v", err) } if err = createNamespace(cli, ns); err != nil { - t.Fatalf("Integration test setup failure: Error createing namespace; err=%v", err) + t.Fatalf("Integration test setup failure: Error creating namespace; err=%v", err) + } + sa, err := cli.CoreV1().ServiceAccounts(ns).Create(ctx, getServiceAccount(ns, controllerSA), metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Integration test setup failure: Error creating service account; err=%v", err) + } + clusterRole, err := cli.RbacV1().ClusterRoles().Create(ctx, getClusterRole(ns), metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Integration test setup failure: Error creating clusterrole; err=%v", err) + } + crb, err := cli.RbacV1().ClusterRoleBindings().Create(ctx, getClusterRoleBinding(sa, clusterRole), metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Integration test setup failure: Error creating clusterRoleBinding; err=%v", err) } // Set Controller namespace and service account os.Setenv(kube.PodNSEnvVar, ns) @@ -85,25 +103,39 @@ func integrationSetup(t *test.T) { if err = ctlr.StartWatch(ctx, ns); err != nil { t.Fatalf("Integration test setup failure: Error starting controller; err=%v", err) } + kontroller.namespace = ns kontroller.context = ctx kontroller.cancel = cancel kontroller.kubeCli = cli + kontroller.serviceAccount = sa + kontroller.clusterRole = clusterRole + kontroller.clusterRoleBinding = crb } func integrationCleanup(t *test.T) { + ctx, cancel := context.WithTimeout(context.Background(), contextWaitTimeout) + defer cancel() + if kontroller.cancel != nil { kontroller.cancel() } if kontroller.namespace != "" { - kontroller.kubeCli.CoreV1().Namespaces().Delete(context.TODO(), kontroller.namespace, metav1.DeleteOptions{}) + kontroller.kubeCli.CoreV1().Namespaces().Delete(ctx, kontroller.namespace, metav1.DeleteOptions{}) + } + if kontroller.clusterRoleBinding != nil && kontroller.clusterRoleBinding.Name != "" { + kontroller.kubeCli.RbacV1().ClusterRoleBindings().Delete(ctx, kontroller.clusterRoleBinding.Name, metav1.DeleteOptions{}) + } + if kontroller.clusterRole != nil && kontroller.clusterRole.Name != "" { + kontroller.kubeCli.RbacV1().ClusterRoles().Delete(ctx, kontroller.clusterRole.Name, metav1.DeleteOptions{}) } } const ( // appWaitTimeout decides the time we are going to wait for app to be ready - appWaitTimeout = 3 * time.Minute - controllerSA = "default" + appWaitTimeout = 3 * time.Minute + controllerSA = "kanister-sa" + contextWaitTimeout = 10 * time.Minute ) type secretProfile struct { @@ -437,3 +469,55 @@ func pingAppAndWait(ctx context.Context, a app.DatabaseApp) error { }) return err } + +func getServiceAccount(namespace, name string) *v1.ServiceAccount { + return &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } +} + +func getClusterRole(namespace string) *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterRole", + APIVersion: "rbac.authorization.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: namespace + "-pod-reader", + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create"}, + APIGroups: []string{""}, + Resources: []string{"pods", "pods/exec"}, + }, + }, + } +} + +func getClusterRoleBinding(sa *v1.ServiceAccount, role *rbacv1.ClusterRole) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterRoleBinding", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: sa.Namespace + "-global-pod-reader", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: sa.Name, + Namespace: sa.Namespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: role.Name, + }, + } +}