diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index db9d763a..8bb015f5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -4,116 +4,173 @@ on: env: ACTIONS_RUNNER_DEBUG: true KUBEPLUS_TEST_OUTPUT: yes + KUBEPLUS_CI: true jobs: job1: runs-on: ubuntu-20.04 name: Deploy to minikube steps: - - uses: actions/checkout@v2 - - name: Start minikube + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Start Minikube uses: medyagh/setup-minikube@master - - name: Try the cluster ! + + - name: Verify Cluster run: kubectl get pods -A - - name: Deploy KubePlus and application to minikube + + - name: Setup Minikube Docker Environment run: | export SHELL=/bin/bash eval $(minikube -p minikube docker-env) - echo "Glibc version" - ldd --version - sudo apt-get install python3-lxml - echo "Installing helm..." + + - name: Display Glibc Version + run: ldd --version + + - name: Install Python3 LXML + run: sudo apt-get install python3-lxml + + - name: Install Helm + run: | + echo "Installing Helm..." wget https://get.helm.sh/helm-v3.12.1-linux-amd64.tar.gz gunzip helm-v3.12.1-linux-amd64.tar.gz tar -xvf helm-v3.12.1-linux-amd64.tar sudo mv linux-amd64/helm /usr/local/bin/. - echo "Installing golang..." - rm -rf /usr/local/go + + - name: Install Golang + run: | + echo "Installing Golang..." + rm -rf /usr/local/go wget https://go.dev/dl/go1.22.4.linux-amd64.tar.gz sudo tar -C /usr/local -xzf go1.22.4.linux-amd64.tar.gz export PATH=$PATH:/usr/local/go/bin go version + + - name: Prepare KubePlus Environment + run: | + echo "Setting up KubePlus environment..." echo "Current directory:`pwd`" echo "Folders:`ls`" + mkdir -p $HOME/go/src/github.com/cloud-ark + cd .. + runner_dir=`pwd` + echo "Current directory:`pwd`" kubeplus_folder="$(basename `pwd`)" echo "KubePlus folder name:$kubeplus_folder" mkdir -p $HOME/go/src/github.com/cloud-ark cd .. runner_dir=`pwd` - echo "Current directory:`pwd`" mv $kubeplus_folder $HOME/go/src/github.com/cloud-ark/kubeplus cd $HOME/go/src/github.com/cloud-ark/kubeplus + runner_dir=`pwd` + export KUBEPLUS_NS=default export KUBEPLUS_HOME=`pwd` export PATH=$KUBEPLUS_HOME/plugins:$PATH echo "PATH:$PATH" echo "KUBEPLUS_HOME:$KUBEPLUS_HOME" - kubectl kubeplus commands - export KUBEPLUS_NS=default - python3 -m venv venv - source venv/bin/activate - pip3 install -r requirements.txt - apiserver=`kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}'` - echo "API_SERVER_URL:$apiserver" - python3 provider-kubeconfig.py -s $apiserver create $KUBEPLUS_NS - deactivate - echo "Building mutating-webhook..." + - name: Build Mutating Webhook + run: | cd $KUBEPLUS_HOME/mutating-webhook - export GO111MODULE=on; go get github.com/googleapis/gnostic@v0.4.0 + export GO111MODULE=on + go get github.com/googleapis/gnostic@v0.4.0 ./build-artifact.sh latest - echo "Building helmer..." + - name: Build Helmer + run: | cd $KUBEPLUS_HOME/platform-operator/helm-pod/ go mod vendor ./build-artifact.sh latest - echo "Building platform-operator..." + - name: Build Platform Operator + run: | cd $KUBEPLUS_HOME/platform-operator ./build-artifact.sh latest - echo "Building kubeconfiggenerator..." + - name: Build KubeConfig Generator + run: | cd $KUBEPLUS_HOME/deploy ./build-artifact-kubeconfiggenerator.sh latest - #echo "Building webhook_init_container..." - #./build-artifact.sh latest - #echo "Building resource cleaner..." - #./build-artifact-clean.sh latest - - #cd $KUBEPLUS_HOME/consumerui - #echo "Building consumer ui..." - #./build-artifact.sh latest + - name: List Docker Images + run: | cd $KUBEPLUS_HOME - ls docker images - echo "Installing KubePlus..." - helm install kubeplus ./deploy/kubeplus-chart --kubeconfig=kubeplus-saas-provider.json --set MUTATING_WEBHOOK=gcr.io/cloudark-kubeplus/pac-mutating-admission-webhook:latest --set PLATFORM_OPERATOR=gcr.io/cloudark-kubeplus/platform-operator:latest --set HELMER=gcr.io/cloudark-kubeplus/helm-pod:latest --set CRD_REGISTRATION_HELPER=gcr.io/cloudark-kubeplus/kubeconfiggenerator:latest -n $KUBEPLUS_NS + - name: Deploy KubePlus, Prometheus, and OpenCost to minikube + run: | + echo "Deploying KubePlus, Prometheus, and OpenCost..." + wget https://raw.githubusercontent.com/opencost/opencost/develop/configs/default.json + ./install.sh --prometheus --opencost default.json --kubeplus-plugin --kubeplus $KUBEPLUS_NS - kubectl get pods -A + - name: Verify Prometheus Installation + run: | + echo "Verifying Prometheus installation..." + kubectl get pods -n prometheus-system | grep prometheus + + - name: Verify OpenCost Installation + run: | + echo "Verifying OpenCost installation..." + kubectl get pods -n opencost | grep opencost - until kubectl get pods -A | grep kubeplus | grep -i Running; do echo "Waiting for KubePlus to start.."; sleep 1; kubeplus_pod=`kubectl get pods | grep kubeplus | awk '{print $1}'`; kubectl get pods $kubeplus_pod; done + - name: Verify KubePlus Installation + run: | + echo "Verifying KubePlus installation..." + kubectl get pods -n $KUBEPLUS_NS | grep kubeplus + + - name: Retrieve KubePlus Pod Logs + run: | kubeplus_pod=`kubectl get pods | grep kubeplus | awk '{print $1}'` - echo "helmer logs..." + echo "Helmer logs..." kubectl logs $kubeplus_pod -c helmer - echo "platform-operator logs..." + echo "Platform Operator logs..." kubectl logs $kubeplus_pod -c platform-operator - echo "crd-hook logs..." + echo "CRD Hook logs..." kubectl logs $kubeplus_pod -c crd-hook - kubectl upload chart ./examples/multitenancy/application-hosting/wordpress/wordpress-chart-0.0.3.tgz kubeplus-saas-provider.json - echo "Sleeping for 10 seconds before continuing..." - sleep 10 + + - name: Upload Example Chart + run: kubectl upload chart ./examples/multitenancy/application-hosting/wordpress/wordpress-chart-0.0.3.tgz kubeplus-saas-provider.json + + - name: Sleep Before Continuing + run: sleep 10 + + - name: Deploy WordPress Service Composition + run: | kubectl create -f ./examples/multitenancy/application-hosting/wordpress/wordpress-service-composition-localchart.yaml --kubeconfig=kubeplus-saas-provider.json - until kubectl get crds | grep wordpressservices.platformapi.kubeplus; do echo "Waiting for CRD to be registered.."; sleep 1; done + + - name: Wait for CRD Registration + run: | + until kubectl get crds | grep wordpressservices.platformapi.kubeplus; do + echo "Waiting for CRD to be registered..." + sleep 1 + done + + - name: Deploy Tenant Example + run: | kubectl create -f ./examples/multitenancy/application-hosting/wordpress/tenant1.yaml --kubeconfig=kubeplus-saas-provider.json kubectl get resourcecompositions kubectl describe resourcecomposition wordpress-service-composition - until kubectl get pods -n wp-tenant1 | grep Running; do echo "Waiting for Application Pods to start.."; sleep 1; done + + - name: Wait for Application Pods to Start + run: | + until kubectl get pods -n wp-tenant1 | grep Running; do + echo "Waiting for Application Pods to start..." + sleep 1 + done + + - name: Interact with Deployed Application + run: | kubectl appresources WordpressService wp-tenant1 –k kubeplus-saas-provider.json kubectl metrics WordpressService wp-tenant1 $KUBEPLUS_NS -k kubeplus-saas-provider.json + + - name: Cleanup Deployed Resources + run: | kubectl delete wordpressservice wp-tenant1 --kubeconfig=kubeplus-saas-provider.json kubectl delete resourcecomposition wordpress-service-composition --kubeconfig=kubeplus-saas-provider.json - echo "Running tests..starting in 5 seconds" - sleep 5 + + - name: Run Unit Tests + run: | cd tests python3 -m venv venv source venv/bin/activate diff --git a/README.md b/README.md index 386e7091..62585f4e 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ KubePlus takes an application Helm chart and wraps it under a Kubernetes API (CR

+## Key Features + ### Isolation KubePlus takes an application Helm chart and wraps it in a Kubernetes API (CRD). This API is used to provision application instances on a cluster. KubePlus isolates each application instance in a separate namespace. It adds a safety perimeter around such namespaces using Kubernetes network policies and non-shared persistent volumes ensuring that each application instance is appropriately isolated from other instances. Additionally, it provides controls for application providers to deploy different tenant application instances on different worker nodes for node isolation. @@ -37,155 +39,23 @@ A new version of an application can be deployed by updating the application Helm ### Customization -The spec properties of the Kubernetes CRD wrapping the application Helm chart are the fields defined in the chart’s values.yaml file. Application deployments can be customized by specifying different values for these spec properties. - - -## Demo - -https://github.com/cloud-ark/kubeplus/assets/732525/efb255ff-fc73-446b-a583-4b89dbf61638 - -## Getting Started with an example - -Let’s look at an example of creating a multi-instance WordPress Service using KubePlus. The WordPress service provider goes through the following steps towards this on their cluster: - -1. Create cluster or use an existing cluster. For testing purposes you can create a [minikube](https://minikube.sigs.k8s.io/docs/) or [kind](https://kind.sigs.k8s.io/) cluster: - - `minikube start` - - or - - `kind create cluster` - -2. Unzip KubePlus plugins and set up the PATH - - ``` - wget https://github.com/cloud-ark/kubeplus/raw/master/kubeplus-kubectl-plugins.tar.gz - tar -zxvf kubeplus-kubectl-plugins.tar.gz - export KUBEPLUS_HOME=`pwd` - export PATH=$KUBEPLUS_HOME/plugins:$PATH - kubectl kubeplus commands - ``` - -3. Set the Namespace in which to deploy KubePlus - - `export KUBEPLUS_NS=default` - -4. Create provider kubeconfig using provider-kubeconfig.py - - ``` - wget https://raw.githubusercontent.com/cloud-ark/kubeplus/master/requirements.txt - wget https://raw.githubusercontent.com/cloud-ark/kubeplus/master/provider-kubeconfig.py - python3 -m venv venv - source venv/bin/activate - pip3 install -r requirements.txt - apiserver=`kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}'` - python3 provider-kubeconfig.py -s $apiserver create $KUBEPLUS_NS - deactivate - ``` - -5. Install KubePlus Operator using the generated provider kubeconfig - - ``` - helm install kubeplus "https://github.com/cloud-ark/operatorcharts/blob/master/kubeplus-chart-4.0.0.tgz?raw=true" --kubeconfig=kubeplus-saas-provider.json -n $KUBEPLUS_NS - until kubectl get pods -A | grep kubeplus | grep Running; do echo "Waiting for KubePlus to start.."; sleep 1; done - ``` - -6. Create Kubernetes CRD representing WordPress Helm chart. - - *The WordPress Helm chart can be specified as a [public url](./examples/multitenancy/application-hosting/wordpress/wordpress-service-composition.yaml) or can be [available locally](./examples/multitenancy/application-hosting/wordpress/wordpress-service-composition-localchart.yaml).* +The spec properties of the Kubernetes CRD wrapping the application Helm chart are the fields defined in the chart’s values.yaml file. Application deployments can be customized by specifying different values for these spec properties. - ``` - kubectl create -f https://raw.githubusercontent.com/cloud-ark/kubeplus/master/examples/multitenancy/application-hosting/wordpress/wordpress-service-composition.yaml --kubeconfig=kubeplus-saas-provider.json - kubectl get resourcecompositions - kubectl describe resourcecomposition wordpress-service-composition - ``` - - If the status of the `wordpress-service-composition` indicates that the new CRD has been created successfully, verify it: - - ``` - kubectl get crds - ``` - - You should see `wordpressservices.platformapi.kubeplus` CRD registered. +## Quick Installation -7. Create WordpressService instance `wp-tenant1` +To install KubePlus and its kubectl plugin, use the following commands: - ``` - kubectl create -f https://raw.githubusercontent.com/cloud-ark/kubeplus/master/examples/multitenancy/application-hosting/wordpress/tenant1.yaml --kubeconfig=kubeplus-saas-provider.json - ``` - -8. Create WordpressService instance `wp-tenant2` - - ``` - kubectl create -f https://raw.githubusercontent.com/cloud-ark/kubeplus/master/examples/multitenancy/application-hosting/wordpress/tenant2.yaml --kubeconfig=kubeplus-saas-provider.json - ``` - -9. Check created WordpressService instances - - ``` - kubectl get wordpressservices - - NAME AGE - wp-tenant1 86s - wp-tenant2 26s - ``` - -10. Check the details of created instance: - - ``` - kubectl describe wordpressservices wp-tenant1 - ``` - -11. Check created application resources. Notice that the `WordpressService` instance resources are deployed in a Namespace `wp-tenant1`, which was created by KubePlus. - - ``` - kubectl appresources WordpressService wp-tenant1 –k kubeplus-saas-provider.json - - NAMESPACE KIND NAME - default WordpressService wp-tenant1 - wp-tenant1 PersistentVolumeClaim mysql-pv-claim - wp-tenant1 PersistentVolumeClaim wp-for-tenant1 - wp-tenant1 Service wordpress-mysql - wp-tenant1 Service wp-for-tenant1 - wp-tenant1 Deployment mysql - wp-tenant1 Deployment wp-for-tenant1 - wp-tenant1 Pod mysql-76d6d9bdfd-2wl2p - wp-tenant1 Pod wp-for-tenant1-87c4c954-s2cct - wp-tenant1 NetworkPolicy allow-external-traffic - wp-tenant1 NetworkPolicy restrict-cross-ns-traffic - wp-tenant1 ResourceQuota wordpressservice-wp-tenant1 - ``` +```sh +wget https://raw.githubusercontent.com/cloud-ark/kubeplus/master/install.sh +chmod +x install.sh +./install.sh --kubeplus --kubeplus-plugin +``` -12. Check application resource consumption - - ``` - kubectl metrics WordpressService wp-tenant1 $KUBEPLUS_NS -k kubeplus-saas-provider.json - - ---------------------------------------------------------- - Kubernetes Resources created: - Number of Sub-resources: - - Number of Pods: 2 - Number of Containers: 2 - Number of Nodes: 1 - Number of Not Running Pods: 0 - Underlying Physical Resoures consumed: - Total CPU(cores): 0.773497m - Total MEMORY(bytes): 516.30859375Mi - Total Storage(bytes): 40Gi - Total Network bytes received: 0 - Total Network bytes transferred: 0 - ---------------------------------------------------------- - ``` +## Demo -13. Cleanup +https://github.com/cloud-ark/kubeplus/assets/732525/efb255ff-fc73-446b-a583-4b89dbf61638 - ``` - kubectl delete wordpressservice wp-tenant1 --kubeconfig=kubeplus-saas-provider.json - kubectl delete wordpressservice wp-tenant2 --kubeconfig=kubeplus-saas-provider.json - kubectl delete resourcecomposition wordpress-service-composition --kubeconfig=kubeplus-saas-provider.json - helm delete kubeplus -n $KUBEPLUS_NS - python3 provider-kubeconfig.py delete $KUBEPLUS_NS - ``` +To get started with an example, follow: [kubeplus/examples/getting-started.md](kubeplus/examples/getting-started.md)