diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index db9d763a..0a523fb9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -1,124 +1,157 @@ name: CI on: - - pull_request + pull_request: + push: + branches: + - master env: ACTIONS_RUNNER_DEBUG: true KUBEPLUS_TEST_OUTPUT: yes + KUBEPLUS_CI: true jobs: job1: runs-on: ubuntu-20.04 - name: Deploy to minikube + name: Test kubeplus deployment on kind steps: - - uses: actions/checkout@v2 - - name: Start minikube - uses: medyagh/setup-minikube@master - - name: Try the cluster ! - run: kubectl get pods -A - - name: Deploy KubePlus and application to minikube - run: | - export SHELL=/bin/bash - eval $(minikube -p minikube docker-env) - echo "Glibc version" - ldd --version + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Create k8s Kind Cluster + uses: helm/kind-action@v1 + + - name: Verify Cluster + run: | + kubectl get pods -A + echo "CLUSTER_NAME=chart-testing" >> $GITHUB_ENV + + - name: Install Helm, Python3 LXML and Golang + run: | sudo apt-get install python3-lxml - echo "Installing helm..." + echo "Installing Helm..." wget https://get.helm.sh/helm-v3.12.1-linux-amd64.tar.gz gunzip helm-v3.12.1-linux-amd64.tar.gz tar -xvf helm-v3.12.1-linux-amd64.tar sudo mv linux-amd64/helm /usr/local/bin/. - echo "Installing golang..." - rm -rf /usr/local/go + echo "Installing Golang..." + rm -rf /usr/local/go wget https://go.dev/dl/go1.22.4.linux-amd64.tar.gz sudo tar -C /usr/local -xzf go1.22.4.linux-amd64.tar.gz export PATH=$PATH:/usr/local/go/bin go version + + - name: Prepare KubePlus Environment + run: | + echo "Setting up KubePlus environment..." echo "Current directory:`pwd`" - echo "Folders:`ls`" - kubeplus_folder="$(basename `pwd`)" - echo "KubePlus folder name:$kubeplus_folder" mkdir -p $HOME/go/src/github.com/cloud-ark cd .. - runner_dir=`pwd` echo "Current directory:`pwd`" - mv $kubeplus_folder $HOME/go/src/github.com/cloud-ark/kubeplus + kubeplus_folder="$(basename `pwd`)" + echo "KubePlus folder name:$kubeplus_folder" + cp -R $kubeplus_folder $HOME/go/src/github.com/cloud-ark/kubeplus cd $HOME/go/src/github.com/cloud-ark/kubeplus + export KUBEPLUS_NS=default + echo "KUBEPLUS_NS=default" >> $GITHUB_ENV export KUBEPLUS_HOME=`pwd` + echo "KUBEPLUS_HOME=$KUBEPLUS_HOME" >> $GITHUB_ENV export PATH=$KUBEPLUS_HOME/plugins:$PATH - echo "PATH:$PATH" - echo "KUBEPLUS_HOME:$KUBEPLUS_HOME" - kubectl kubeplus commands - export KUBEPLUS_NS=default - python3 -m venv venv - source venv/bin/activate - pip3 install -r requirements.txt - apiserver=`kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}'` - echo "API_SERVER_URL:$apiserver" - python3 provider-kubeconfig.py -s $apiserver create $KUBEPLUS_NS - deactivate + echo "PATH=$PATH" >> $GITHUB_ENV - echo "Building mutating-webhook..." + - name: Build Mutating Webhook + run: | + echo "KUBEPLUS_HOME:$KUBEPLUS_HOME" cd $KUBEPLUS_HOME/mutating-webhook - export GO111MODULE=on; go get github.com/googleapis/gnostic@v0.4.0 + export GO111MODULE=on + go get github.com/googleapis/gnostic@v0.4.0 ./build-artifact.sh latest - echo "Building helmer..." + - name: Build Helmer + run: | cd $KUBEPLUS_HOME/platform-operator/helm-pod/ go mod vendor ./build-artifact.sh latest - echo "Building platform-operator..." + - name: Build Platform Operator + run: | cd $KUBEPLUS_HOME/platform-operator ./build-artifact.sh latest - echo "Building kubeconfiggenerator..." + - name: Build KubeConfig Generator + run: | cd $KUBEPLUS_HOME/deploy ./build-artifact-kubeconfiggenerator.sh latest - #echo "Building webhook_init_container..." - #./build-artifact.sh latest - #echo "Building resource cleaner..." - #./build-artifact-clean.sh latest - - #cd $KUBEPLUS_HOME/consumerui - #echo "Building consumer ui..." - #./build-artifact.sh latest - cd $KUBEPLUS_HOME - ls - docker images + - name: Load Docker Images to Kind cluster + run: | + for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep '^gcr.io/cloudark-kubeplus'); do + kind load docker-image "$image" --name $CLUSTER_NAME + done + docker exec $CLUSTER_NAME-control-plane crictl images - echo "Installing KubePlus..." - helm install kubeplus ./deploy/kubeplus-chart --kubeconfig=kubeplus-saas-provider.json --set MUTATING_WEBHOOK=gcr.io/cloudark-kubeplus/pac-mutating-admission-webhook:latest --set PLATFORM_OPERATOR=gcr.io/cloudark-kubeplus/platform-operator:latest --set HELMER=gcr.io/cloudark-kubeplus/helm-pod:latest --set CRD_REGISTRATION_HELPER=gcr.io/cloudark-kubeplus/kubeconfiggenerator:latest -n $KUBEPLUS_NS + - name: Deploy KubePlus and Kubeplus plugin + run: | + ./install.sh --kubeplus-plugin --kubeplus $KUBEPLUS_NS - kubectl get pods -A + - name: Verify KubePlus Installation + run: | + echo "Verifying KubePlus installation..." + kubectl get pods -n $KUBEPLUS_NS | grep kubeplus - until kubectl get pods -A | grep kubeplus | grep -i Running; do echo "Waiting for KubePlus to start.."; sleep 1; kubeplus_pod=`kubectl get pods | grep kubeplus | awk '{print $1}'`; kubectl get pods $kubeplus_pod; done + - name: Retrieve KubePlus Pod Logs + run: | kubeplus_pod=`kubectl get pods | grep kubeplus | awk '{print $1}'` - echo "helmer logs..." + echo "Helmer logs..." kubectl logs $kubeplus_pod -c helmer - echo "platform-operator logs..." + echo "Platform Operator logs..." kubectl logs $kubeplus_pod -c platform-operator - echo "crd-hook logs..." + echo "CRD Hook logs..." kubectl logs $kubeplus_pod -c crd-hook - kubectl upload chart ./examples/multitenancy/application-hosting/wordpress/wordpress-chart-0.0.3.tgz kubeplus-saas-provider.json - echo "Sleeping for 10 seconds before continuing..." - sleep 10 + + - name: Upload Example Chart + run: kubectl upload chart ./examples/multitenancy/application-hosting/wordpress/wordpress-chart-0.0.3.tgz kubeplus-saas-provider.json + + - name: Sleep Before Continuing + run: sleep 10 + + - name: Deploy WordPress Service Composition + run: | kubectl create -f ./examples/multitenancy/application-hosting/wordpress/wordpress-service-composition-localchart.yaml --kubeconfig=kubeplus-saas-provider.json - until kubectl get crds | grep wordpressservices.platformapi.kubeplus; do echo "Waiting for CRD to be registered.."; sleep 1; done + + - name: Wait for CRD Registration + run: | + until kubectl get crds | grep wordpressservices.platformapi.kubeplus; do + echo "Waiting for CRD to be registered..." + sleep 1 + done + + - name: Deploy Tenant Example + run: | kubectl create -f ./examples/multitenancy/application-hosting/wordpress/tenant1.yaml --kubeconfig=kubeplus-saas-provider.json kubectl get resourcecompositions kubectl describe resourcecomposition wordpress-service-composition - until kubectl get pods -n wp-tenant1 | grep Running; do echo "Waiting for Application Pods to start.."; sleep 1; done + + - name: Wait for Application Pods to Start + run: | + until kubectl get pods -n wp-tenant1 | grep Running; do + echo "Waiting for Application Pods to start..." + sleep 1 + done + + - name: Interact with Deployed Application + run: | kubectl appresources WordpressService wp-tenant1 –k kubeplus-saas-provider.json kubectl metrics WordpressService wp-tenant1 $KUBEPLUS_NS -k kubeplus-saas-provider.json + + - name: Cleanup Deployed Resources + run: | kubectl delete wordpressservice wp-tenant1 --kubeconfig=kubeplus-saas-provider.json kubectl delete resourcecomposition wordpress-service-composition --kubeconfig=kubeplus-saas-provider.json - echo "Running tests..starting in 5 seconds" - sleep 5 + + - name: Run Unit Tests + run: | cd tests python3 -m venv venv source venv/bin/activate pip3 install -r requirements.txt python3 -m unittest -v tests deactivate - cd ../.. - mv kubeplus $runner_dir diff --git a/README.md b/README.md index 386e7091..e566ae3a 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,15 @@ KubePlus takes an application Helm chart and wraps it under a Kubernetes API (CR

+## Key Features + ### Isolation -KubePlus takes an application Helm chart and wraps it in a Kubernetes API (CRD). This API is used to provision application instances on a cluster. KubePlus isolates each application instance in a separate namespace. It adds a safety perimeter around such namespaces using Kubernetes network policies and non-shared persistent volumes ensuring that each application instance is appropriately isolated from other instances. Additionally, it provides controls for application providers to deploy different tenant application instances on different worker nodes for node isolation. +KubePlus takes an application Helm chart and wraps it in a Kubernetes API (CRD). This API is used to provision application instances on a cluster. KubePlus isolates each application instance in a separate namespace. It adds a safety perimeter around such namespaces using Kubernetes network policies and non-shared persistent volumes ensuring that each application instance is appropriately isolated from other instances. Additionally, it provides controls for application providers to deploy different tenant application instances on different worker nodes for node isolation. ### Security -The KubePlus Operator does not need any admin-level permissions on a cluster for application providers. This allows application providers to offer their managed services on any K8s clusters including those owned by their customers. KubePlus comes with a small utility that allows you to create provider specific kubeconfig on a cluster in order to enable application deployments and management. Providers have an ability to create a consumer specific further limited kubeconfig to allow for self-service provisioning of application instances as well. +The KubePlus Operator does not need any admin-level permissions on a cluster for application providers. This allows application providers to offer their managed services on any K8s clusters including those owned by their customers. KubePlus comes with a small utility that allows you to create provider specific kubeconfig on a cluster in order to enable application deployments and management. Providers have an ability to create a consumer specific further limited kubeconfig to allow for self-service provisioning of application instances as well. ### Resource utilization @@ -32,23 +34,18 @@ KubePlus provides controls to set per-namespace resource quotas. It also monitor ### Upgrades A running application instance can be updated by making changes to the spec properties of the CRD instance and applying it. -KubePlus will update that application instance (i.e. helm upgrade of the corresponding helm release). +KubePlus will update that application instance (i.e. helm upgrade of the corresponding helm release). A new version of an application can be deployed by updating the application Helm chart under the existing Kubernetes CRD or registering the new chart under a new Kubernetes CRD. If the existing Kubernetes CRD object is updated, KubePlus will update all the running application instances (helm releases) to the new version of the application Helm chart. ### Customization -The spec properties of the Kubernetes CRD wrapping the application Helm chart are the fields defined in the chart’s values.yaml file. Application deployments can be customized by specifying different values for these spec properties. - - -## Demo - -https://github.com/cloud-ark/kubeplus/assets/732525/efb255ff-fc73-446b-a583-4b89dbf61638 +The spec properties of the Kubernetes CRD wrapping the application Helm chart are the fields defined in the chart’s values.yaml file. Application deployments can be customized by specifying different values for these spec properties. -## Getting Started with an example +## Installation -Let’s look at an example of creating a multi-instance WordPress Service using KubePlus. The WordPress service provider goes through the following steps towards this on their cluster: +### Manual steps -1. Create cluster or use an existing cluster. For testing purposes you can create a [minikube](https://minikube.sigs.k8s.io/docs/) or [kind](https://kind.sigs.k8s.io/) cluster: +* Create cluster or use an existing cluster. For testing purposes you can create a [minikube](https://minikube.sigs.k8s.io/docs/) or [kind](https://kind.sigs.k8s.io/) cluster: `minikube start` @@ -56,9 +53,15 @@ Let’s look at an example of creating a multi-instance WordPress Service using `kind create cluster` -2. Unzip KubePlus plugins and set up the PATH +* Set the Namespace in which to deploy KubePlus + ```sh + export KUBEPLUS_NS=default ``` + +* Unzip KubePlus plugins and set up the PATH + + ```sh wget https://github.com/cloud-ark/kubeplus/raw/master/kubeplus-kubectl-plugins.tar.gz tar -zxvf kubeplus-kubectl-plugins.tar.gz export KUBEPLUS_HOME=`pwd` @@ -66,13 +69,9 @@ Let’s look at an example of creating a multi-instance WordPress Service using kubectl kubeplus commands ``` -3. Set the Namespace in which to deploy KubePlus - - `export KUBEPLUS_NS=default` +* Create provider kubeconfig using provider-kubeconfig.py -4. Create provider kubeconfig using provider-kubeconfig.py - - ``` + ```sh wget https://raw.githubusercontent.com/cloud-ark/kubeplus/master/requirements.txt wget https://raw.githubusercontent.com/cloud-ark/kubeplus/master/provider-kubeconfig.py python3 -m venv venv @@ -83,109 +82,28 @@ Let’s look at an example of creating a multi-instance WordPress Service using deactivate ``` -5. Install KubePlus Operator using the generated provider kubeconfig +* Install KubePlus Operator using the generated provider kubeconfig - ``` + ```sh helm install kubeplus "https://github.com/cloud-ark/operatorcharts/blob/master/kubeplus-chart-4.0.0.tgz?raw=true" --kubeconfig=kubeplus-saas-provider.json -n $KUBEPLUS_NS until kubectl get pods -A | grep kubeplus | grep Running; do echo "Waiting for KubePlus to start.."; sleep 1; done ``` -6. Create Kubernetes CRD representing WordPress Helm chart. - - *The WordPress Helm chart can be specified as a [public url](./examples/multitenancy/application-hosting/wordpress/wordpress-service-composition.yaml) or can be [available locally](./examples/multitenancy/application-hosting/wordpress/wordpress-service-composition-localchart.yaml).* - - ``` - kubectl create -f https://raw.githubusercontent.com/cloud-ark/kubeplus/master/examples/multitenancy/application-hosting/wordpress/wordpress-service-composition.yaml --kubeconfig=kubeplus-saas-provider.json - kubectl get resourcecompositions - kubectl describe resourcecomposition wordpress-service-composition - ``` - - If the status of the `wordpress-service-composition` indicates that the new CRD has been created successfully, verify it: - - ``` - kubectl get crds - ``` - - You should see `wordpressservices.platformapi.kubeplus` CRD registered. - -7. Create WordpressService instance `wp-tenant1` - - ``` - kubectl create -f https://raw.githubusercontent.com/cloud-ark/kubeplus/master/examples/multitenancy/application-hosting/wordpress/tenant1.yaml --kubeconfig=kubeplus-saas-provider.json - ``` - -8. Create WordpressService instance `wp-tenant2` - - ``` - kubectl create -f https://raw.githubusercontent.com/cloud-ark/kubeplus/master/examples/multitenancy/application-hosting/wordpress/tenant2.yaml --kubeconfig=kubeplus-saas-provider.json - ``` - -9. Check created WordpressService instances - - ``` - kubectl get wordpressservices - - NAME AGE - wp-tenant1 86s - wp-tenant2 26s - ``` - -10. Check the details of created instance: - - ``` - kubectl describe wordpressservices wp-tenant1 - ``` +### Script for linux -11. Check created application resources. Notice that the `WordpressService` instance resources are deployed in a Namespace `wp-tenant1`, which was created by KubePlus. +* To install KubePlus and its kubectl plugin, use the following commands: - ``` - kubectl appresources WordpressService wp-tenant1 –k kubeplus-saas-provider.json - - NAMESPACE KIND NAME - default WordpressService wp-tenant1 - wp-tenant1 PersistentVolumeClaim mysql-pv-claim - wp-tenant1 PersistentVolumeClaim wp-for-tenant1 - wp-tenant1 Service wordpress-mysql - wp-tenant1 Service wp-for-tenant1 - wp-tenant1 Deployment mysql - wp-tenant1 Deployment wp-for-tenant1 - wp-tenant1 Pod mysql-76d6d9bdfd-2wl2p - wp-tenant1 Pod wp-for-tenant1-87c4c954-s2cct - wp-tenant1 NetworkPolicy allow-external-traffic - wp-tenant1 NetworkPolicy restrict-cross-ns-traffic - wp-tenant1 ResourceQuota wordpressservice-wp-tenant1 + ```sh + wget https://raw.githubusercontent.com/cloud-ark/kubeplus/master/install.sh + chmod +x install.sh + ./install.sh --kubeplus --kubeplus-plugin ``` -12. Check application resource consumption - - ``` - kubectl metrics WordpressService wp-tenant1 $KUBEPLUS_NS -k kubeplus-saas-provider.json - - ---------------------------------------------------------- - Kubernetes Resources created: - Number of Sub-resources: - - Number of Pods: 2 - Number of Containers: 2 - Number of Nodes: 1 - Number of Not Running Pods: 0 - Underlying Physical Resoures consumed: - Total CPU(cores): 0.773497m - Total MEMORY(bytes): 516.30859375Mi - Total Storage(bytes): 40Gi - Total Network bytes received: 0 - Total Network bytes transferred: 0 - ---------------------------------------------------------- - ``` +## Demo -13. Cleanup +https://github.com/cloud-ark/kubeplus/assets/732525/efb255ff-fc73-446b-a583-4b89dbf61638 - ``` - kubectl delete wordpressservice wp-tenant1 --kubeconfig=kubeplus-saas-provider.json - kubectl delete wordpressservice wp-tenant2 --kubeconfig=kubeplus-saas-provider.json - kubectl delete resourcecomposition wordpress-service-composition --kubeconfig=kubeplus-saas-provider.json - helm delete kubeplus -n $KUBEPLUS_NS - python3 provider-kubeconfig.py delete $KUBEPLUS_NS - ``` +To get started with an example, follow: [kubeplus/examples/getting-started.md](kubeplus/examples/getting-started.md)