From 13aea07d0d78d27be696c709cfb2df1651e1d9e0 Mon Sep 17 00:00:00 2001 From: Yue Yang Date: Thu, 11 Jan 2024 19:16:25 +0800 Subject: [PATCH] refactor: running on k8s (#57) --- docs/deploy/_index.md | 5 +- docs/deploy/deploy-helm.md | 106 --------------- docs/deploy/deploy-k8s.md | 226 ------------------------------- docs/deploy/running-on-k8s.md | 175 ++++++++++++++++++++++++ docs/release-notes.md | 2 +- docs/v0.16.0/deploy/_index.md | 6 +- docs/v0.17.0/deploy/_index.md | 6 +- docs/zh/deploy/_index.md | 6 +- docs/zh/v0.16.0/deploy/_index.md | 6 +- docs/zh/v0.17.0/deploy/_index.md | 6 +- 10 files changed, 205 insertions(+), 339 deletions(-) delete mode 100644 docs/deploy/deploy-helm.md delete mode 100644 docs/deploy/deploy-k8s.md create mode 100644 docs/deploy/running-on-k8s.md diff --git a/docs/deploy/_index.md b/docs/deploy/_index.md index fbb5b7f..08255ac 100644 --- a/docs/deploy/_index.md +++ b/docs/deploy/_index.md @@ -1,5 +1,8 @@ --- -order: ["deploy-helm", "deploy-k8s", "deploy-docker", "quick-deploy-ssh"] +order: + - deploy-docker.md + - running-on-k8s.md + - quick-deploy-ssh.md collapsed: false --- diff --git a/docs/deploy/deploy-helm.md b/docs/deploy/deploy-helm.md deleted file mode 100644 index 37413bd..0000000 --- a/docs/deploy/deploy-helm.md +++ /dev/null @@ -1,106 +0,0 @@ -# Running on Kubernetes by Helm - -This document describes how to run HStreamDB kubernetes using the helm chart -that we provide. The document assumes basic previous kubernetes knowledge. By -the end of this section, you'll have a fully running HStreamDB cluster on -kubernetes that's ready to receive reads/writes, process datas, etc. - -## Building your Kubernetes Cluster - -The first step is to have a running kubernetes cluster. You can use a managed -cluster (provided by your cloud provider), a self-hosted cluster or a local -kubernetes cluster using a tool like minikube. Make sure that kubectl points to -whatever cluster you're planning to use. - -Also, you need a storageClass, you can create by `kubectl`or by your cloud -provider web page if it has. minikube provides a storage class called `standard` -by default, which is used by the helm chart by default. - -## Starting HStreamDB - -### Clone code and get helm dependencies - -```sh -git clone https://github.com/hstreamdb/hstream.git -cd hstream/deploy/chart/hstream/ -helm dependency build . -``` - -### Deploy HStreamDB by Helm - -```sh -helm install my-hstream . -``` - -Helm chart also provides the `value.yaml` file where you can modify your -configuration, for example when you want to use other storage classes to deploy -the cluster, you can modify `logdevice.persistence.storageClass` and -`zookeeper.persistence.storageClass` in `value.yaml`, and use -`helm install my-hstream -f values.yaml .` to deploy. - -### Check Cluster Status - -The `helm install` command will deploy the zookeeper cluster, logdevice cluster -and hstream cluster, this can take some time, you can check the status of the -cluster with `kubectl get pods`, there will be some `Error` and -`CrashLoopBackOff` status during the cluster deployment, these will disappear -after some time, eventually you will see something like the following. - -``` -NAME READY STATUS RESTARTS AGE -my-hstream-0 1/1 Running 3 (16h ago) 16h -my-hstream-1 1/1 Running 2 (16h ago) 16h -my-hstream-2 1/1 Running 0 16h -my-hstream-logdevice-0 1/1 Running 3 (16h ago) 16h -my-hstream-logdevice-1 1/1 Running 3 (16h ago) 16h -my-hstream-logdevice-2 1/1 Running 0 16h -my-hstream-logdevice-3 1/1 Running 0 16h -my-hstream-logdevice-admin-server-6867fd9494-bk5mf 1/1 Running 3 (16h ago) 16h -my-hstream-zookeeper-0 1/1 Running 0 16h -my-hstream-zookeeper-1 1/1 Running 0 16h -my-hstream-zookeeper-2 1/1 Running 0 16h -``` - -You can check the status of the HStreamDB cluster with the `hadmin server` -command. - -```sh -kubectl exec -it hstream-1 -- bash -c "hadmin server status" -``` -``` -+---------+---------+------------------+ -| node_id | state | address | -+---------+---------+------------------+ -| 100 | Running | 172.17.0.4:6570 | -| 101 | Running | 172.17.0.10:6570 | -| 102 | Running | 172.17.0.12:6570 | -+---------+---------+------------------+ -``` - -## Manage HStore Cluster - -Now you can run `hadmin store` to manage the hstore cluster. -To check the state of the cluster, you can then run: - -```sh -kubectl exec -it my-hstream-0 -- bash -c "hadmin store --host my-hstream-logdevice-admin-server status" -``` -``` -+----+------------------------+----------+-------+--------------+----------+ -| ID | NAME | PACKAGE | STATE | UPTIME | LOCATION | -+----+------------------------+----------+-------+--------------+----------+ -| 0 | my-hstream-logdevice-0 | 99.99.99 | ALIVE | 16 hours ago | | -| 1 | my-hstream-logdevice-1 | 99.99.99 | DEAD | 16 hours ago | | -| 2 | my-hstream-logdevice-2 | 99.99.99 | DEAD | 16 hours ago | | -| 3 | my-hstream-logdevice-3 | 99.99.99 | DEAD | 16 hours ago | | -+----+------------------------+----------+-------+--------------+----------+ -+---------+-------------+---------------+------------+---------------+ -| SEQ. | DATA HEALTH | STORAGE STATE | SHARD OP. | HEALTH STATUS | -+---------+-------------+---------------+------------+---------------+ -| ENABLED | HEALTHY(1) | READ_WRITE(1) | ENABLED(1) | HEALTHY | -| ENABLED | HEALTHY(1) | READ_WRITE(1) | ENABLED(1) | HEALTHY | -| ENABLED | HEALTHY(1) | READ_WRITE(1) | ENABLED(1) | HEALTHY | -| ENABLED | HEALTHY(1) | READ_WRITE(1) | ENABLED(1) | HEALTHY | -+---------+-------------+---------------+------------+---------------+ -Took 16.727s -``` diff --git a/docs/deploy/deploy-k8s.md b/docs/deploy/deploy-k8s.md deleted file mode 100644 index ddd0c94..0000000 --- a/docs/deploy/deploy-k8s.md +++ /dev/null @@ -1,226 +0,0 @@ -# Running on Kubernetes - -This document describes how to run HStreamDB kubernetes using the specs that we -provide. The document assumes basic previous kubernetes knowledge. By the end of -this section, you'll have a fully running HStreamDB cluster on kubernetes that's -ready to receive reads/writes, process datas, etc. - -## Building your Kubernetes Cluster - -The first step is to have a running kubernetes cluster. You can use a managed -cluster (provided by your cloud provider), a self-hosted cluster or a local -kubernetes cluster using a tool like minikube. Make sure that kubectl points to -whatever cluster you're planning to use. - -Also, you need a storageClass named `hstream-store`, you can create by `kubectl` -or by your cloud provider web page if it has. - -::: tip - -For minikube user, you can use the default storage class called `standard`. - -::: - -## Install Zookeeper - -HStreamDB depends on Zookeeper for storing queries information and some internal -storage configuration. So we will need to provision a zookeeper ensemble that -HStreamDB will be able to access. For this demo, we will use -[helm](https://helm.sh/) (A package manager for kubernetes) to install -zookeeper. After installing helm run: - -```sh -helm repo add bitnami https://charts.bitnami.com/bitnami -helm repo update - -helm install zookeeper bitnami/zookeeper \ - --set image.tag=3.6 \ - --set replicaCount=3 \ - --set persistence.storageClass=hstream-store \ - --set persistence.size=20Gi -``` - -``` -NAME: zookeeper -LAST DEPLOYED: Tue Jul 6 10:51:37 2021 -NAMESPACE: test -STATUS: deployed -REVISION: 1 -TEST SUITE: None -NOTES: -** Please be patient while the chart is being deployed ** - -ZooKeeper can be accessed via port 2181 on the following DNS name from within your cluster: - - zookeeper.svc.cluster.local - -To connect to your ZooKeeper server run the following commands: - - export POD_NAME=$(kubectl get pods -l "app.kubernetes.io/name=zookeeper,app.kubernetes.io/instance=zookeeper,app.kubernetes.io/component=zookeeper" -o jsonpath="{.items[0].metadata.name}") - kubectl exec -it $POD_NAME -- zkCli.sh - -To connect to your ZooKeeper server from outside the cluster execute the following commands: - - kubectl port-forward svc/zookeeper 2181:2181 & - zkCli.sh 127.0.0.1:2181 -WARNING: Rolling tag detected (bitnami/zookeeper:3.6), please note that it is strongly recommended to avoid using rolling tags in a production environment. -+info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ -``` - -This will by default install a 3 nodes zookeeper ensemble. Wait until all the -three pods are marked as ready: - -```sh -kubectl get pods -``` - -``` -NAME READY STATUS RESTARTS AGE -zookeeper-0 1/1 Running 0 22h -zookeeper-1 1/1 Running 0 4d22h -zookeeper-2 1/1 Running 0 16m -``` - -## Configuring and Starting HStreamDB - -Once all the zookeeper pods are ready, we're ready to start installing the -HStreamDB cluster. - -### Fetching The K8s Specs - -```sh -git clone git@github.com:hstreamdb/hstream.git -cd hstream/deploy/k8s -``` - -### Update Configuration - -If you used a different way to install zookeeper, make sure to update the -zookeeper connection string in storage config file `config.json` and server -service file `hstream-server.yaml`. - -It should look something like this: - -```sh -cat config.json | grep -A 2 zookeeper -``` -``` - "zookeeper": { - "zookeeper_uri": "ip://zookeeper-0.zookeeper-headless:2181,zookeeper-1.zookeeper-headless:2181,zookeeper-2.zookeeper-headless:2181", - "timeout": "30s" - } -``` - -```sh -cat hstream-server.yaml | grep -A 1 metastore-uri -``` - -``` -- "--metastore-uri" -- "zk://zookeeper-0.zookeeper-headless:2181,zookeeper-1.zookeeper-headless:2181,zookeeper-2.zookeeper-headless:2181" -``` - -::: tip - -The zookeeper connection string in storage config file and the service file can -be different. But for normal scenario, they are the same. - -::: - -By default, this spec installs a 3 nodes HStream server cluster and 4 nodes -storage cluster. If you want a bigger cluster, modify the `hstream-server.yaml` -and `logdevice-statefulset.yaml` file, and increase the number of replicas to -the number of nodes you want in the cluster. Also by default, we attach a 40GB -persistent storage to the nodes, if you want more you can change that under the -volumeClaimTemplates section. - -### Starting the Cluster - -```sh -kubectl apply -k . -``` - -When you run `kubectl get pods`, you should see something like this: - -``` -NAME READY STATUS RESTARTS AGE -hstream-server-0 1/1 Running 0 6d18h -hstream-server-1 1/1 Running 0 6d18h -hstream-server-2 1/1 Running 0 6d18h -logdevice-0 1/1 Running 0 6d18h -logdevice-1 1/1 Running 0 6d18h -logdevice-2 1/1 Running 0 6d18h -logdevice-3 1/1 Running 0 6d18h -logdevice-admin-server-deployment-5c5fb9f8fb-27jlk 1/1 Running 0 6d18h -zookeeper-0 1/1 Running 0 6d22h -zookeeper-1 1/1 Running 0 10d -zookeeper-2 1/1 Running 0 6d -``` - -### Bootstrapping cluster - -Once all the logdevice pods are running and ready, you'll need to bootstrap the -cluster to enable all the nodes. To do that, run: - -```sh-vue -kubectl run hstream-admin -it --rm --restart=Never --image=hstreamdb/hstream:{{ $version() }} -- \ - hadmin store --host logdevice-admin-server-service \ - nodes-config bootstrap --metadata-replicate-across 'node:3' -``` - -This will start a hstream-admin pod, that connects to the store admin server and -invokes the `nodes-config bootstrap` hadmin store command and sets the metadata -replication property of the cluster to be replicated across three different -nodes. On success, you should see something like: - -```txt -Successfully bootstrapped the cluster -pod "hstream-admin" deleted -``` - -Now, you can boostrap hstream server, by running the following command: - -```sh-vue -kubectl run hstream-admin -it --rm --restart=Never --image=hstreamdb/hstream:{{ $version() }} -- \ - hadmin server --host hstream-server-0.hstream-server init -``` - -On success, you should see something like: - -```txt -Cluster is ready! -pod "hstream-admin" deleted -``` - -Note that depending on how fast the storage cluster completes bootstrap, running -`hadmin init` may fail. So you may need to run the command more than once. - -## Managing the Storage Cluster - -```sh-vue -kubectl run hstream-admin -it --rm --restart=Never --image=hstreamdb/hstream:{{ $version() }} -- bash -``` - -Now you can run `hadmin store` to manage the cluster: - -```sh -hadmin store --help -``` - -To check the state of the cluster, you can then run: - -```sh -hadmin store --host logdevice-admin-server-service status -``` - -```txt -+----+-------------+-------+---------------+ -| ID | NAME | STATE | HEALTH STATUS | -+----+-------------+-------+---------------+ -| 0 | logdevice-0 | ALIVE | HEALTHY | -| 1 | logdevice-1 | ALIVE | HEALTHY | -| 2 | logdevice-2 | ALIVE | HEALTHY | -| 3 | logdevice-3 | ALIVE | HEALTHY | -+----+-------------+-------+---------------+ -Took 2.567s -``` diff --git a/docs/deploy/running-on-k8s.md b/docs/deploy/running-on-k8s.md new file mode 100644 index 0000000..1c517ed --- /dev/null +++ b/docs/deploy/running-on-k8s.md @@ -0,0 +1,175 @@ +# Running on Kubernetes + +This document describes how to run a HStreamDB cluster on Kubernetes. We will assume +that you have a basic understanding of Kubernetes. At the end of this document, you +will have a HStreamDB cluster running on Kubernetes, and in doing so, familiarize yourself +with how HStreamDB works on Kubernetes. + +::: tip + +For production environment, please refer to [Deploy with Helm](#deploy-with-helm) section. + +::: + +## Prerequisites + +Before starting, you need to have a Kubernetes cluster running. If you don't have one, +we recommend using [minikube](https://minikube.sigs.k8s.io/docs/start/), [microk8s](https://microk8s.io/), etc. +They are easy to install and use in a local environment. For production environment, +Please refer to the official documentation of Kubernetes or other cloud providers. + +## HStream Operator + +### Introduction + +HStream Operator is a Kubernetes operator designed to manage and maintain the HStreamDB cluster within a Kubernetes environment. The HStreamDB cluster comprises of several components including: + +- HMeta +- AdminServer +- [HServer](../reference/architecture/hserver.md) +- [HStore](../reference/architecture/hstore.md) + +It simplifies the deployment, scaling, and operations of HStreamDB clusters on Kubernetes, making it easier for users to manage their HStream components effectively. We use and get benefits from [kubebuilder](https://book.kubebuilder.io/) to simplify the development of the operator. + +### Installation + +We recommend using the [Helm](https://helm.sh/) package manager to install the HStreamDB operator on your Kubernetes cluster. + +> Currently, we haven't released the chart because of this operator is still at an early stage. So you +> need to clone this repo and install the chart from the local directory. + +```sh +git clone https://github.com/hstreamdb/hstream-operator.git && cd hstream-operator +helm install hstream-operator deploy/charts/hstream-operator -n hstream-operator-system --create-namespace +``` + +Every releases will be published to [GitHub Releases](https://github.com/hstreamdb/hstream-operator/releases), you +can also install the operator with the following command: + +```sh +kubectl create -f https://github.com/hstreamdb/hstream-operator/releases/download/0.0.9/hstream-operator.yaml +``` + +> You can also use server-side apply to install the operator by running `kubectl apply --server-side`, please refer to +> [comparison-with-client-side-apply](https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply) for more details. + +Replace `0.0.9` with the version you want to install. + +### Check the status + +You can check the status of the operator by running: + +```sh +kubectl get pods -l "control-plane=hstream-operator-manager" -n hstream-operator-system +``` + +Expected output: + +```sh +NAME READY STATUS RESTARTS AGE +hstream-operator-controller-manager-f989476d4-qllfs 1/1 Running 1 (16h ago) 16h +``` + +After the operator is running, you can create a HStreamDB cluster by applying a `HStreamDB` resource. We will +introduce the `HStreamDB` resource in the next section. + +## Deploy a HStreamDB Cluster + +By applying a `HStreamDB` resource, the operator we create in the previous section will create a HStreamDB +cluster for us. We provide a sample `HStreamDB` resource in , you can apply it by running: + +```sh +kubectl apply -f https://raw.githubusercontent.com/hstreamdb/hstream-operator/main/config/samples/hstreamdb.yaml +``` + +After a few seconds, you can check the status of the cluster by running: + +```sh +kubectl get po -n hstreamdb +``` + +```sh +NAME READY STATUS RESTARTS AGE +hstreamdb-sample-hmeta-2 1/1 Running 0 7m45s +hstreamdb-sample-hmeta-0 1/1 Running 0 7m45s +hstreamdb-sample-hmeta-1 1/1 Running 0 7m45s +hstreamdb-sample-admin-server-6c547b85c7-7h9gw 1/1 Running 0 7m34s +hstreamdb-sample-hstore-0 1/1 Running 0 7m34s +hstreamdb-sample-hstore-1 1/1 Running 0 7m34s +hstreamdb-sample-hstore-2 1/1 Running 0 7m34s +hstreamdb-sample-hserver-0 1/1 Running 0 7m18s +hstreamdb-sample-hserver-2 1/1 Running 0 7m18s +hstreamdb-sample-hserver-1 1/1 Running 0 7m18s +``` + +If all the pods are running, congratulations, you have successfully deployed a HStreamDB cluster on Kubernetes. + +## Deploy with Helm + +For production environment, we recommend using Helm to deploy HStreamDB cluster. We provide a Helm chart for +deploying HStreamDB cluster, you can find the chart repo at . + +### Add Chart + +```sh +helm repo add hstreamdb https://hstreamdb.github.io/helm-charts +``` + +For updating the repo, you can run: + +```sh +helm repo update +``` + +This will update all the repo in your local machine, include hstreamdb repo. + +For searching different versions of the chart, you can run: + +```sh +helm search repo hstreamdb/hstreamdb +``` + +### Install Chart + +::: code-group + +```sh [minikube] +helm install hstreamdb hstreamdb/hstreamdb --namespace hstreamdb --create-namespace +``` + +```sh [microk8s] +helm install hstreamdb hstreamdb/hstreamdb --namespace hstreamdb --create-namespace --set storageClassName=microk8s-hostpath +``` + +::: + +After installing the chart, you can check the status of the cluster by running: + +```sh +kubectl get po -n hstreamdb +``` + +Now you have successfully deployed a HStreamDB cluster on Kubernetes with Helm. + +## Helm Chart Configuration + +Refer to for more details. +It contains all the configuration options for the chart. + +## Kafka Mode + +HStreamDB can be deployed in Kafka mode, which means that HStreamDB will use Kafka protocol to communicate with +the clients. To deploy HStreamDB in Kafka mode, you need to set `clusterConfig.kafkaMode` to `true`: + +```sh +helm install hstreamdb hstreamdb/hstreamdb --namespace hstreamdb --create-namespace --set clusterConfig.kafkaMode=true +``` + +## Enable Console + +HStreamDB provides a web console for users to manage their HStreamDB cluster. To enable the console, you need to +set `console.enabled` to `true`: + +```sh +helm install hstreamdb hstreamdb/hstreamdb --namespace hstreamdb --create-namespace --set console.enabled=true +``` diff --git a/docs/release-notes.md b/docs/release-notes.md index 41203d7..ab125aa 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -490,7 +490,7 @@ displayed on the Grafana board. #### Deployment on K8s with Helm We provide a helm chart to support deploying HStreamDB on k8s using Helm. You -can refer to [the documentation](./deploy/deploy-helm.md) for +can refer to [the documentation](./deploy/running-on-k8s.md) for details. ### Java Client diff --git a/docs/v0.16.0/deploy/_index.md b/docs/v0.16.0/deploy/_index.md index fbb5b7f..2acc965 100644 --- a/docs/v0.16.0/deploy/_index.md +++ b/docs/v0.16.0/deploy/_index.md @@ -1,5 +1,9 @@ --- -order: ["deploy-helm", "deploy-k8s", "deploy-docker", "quick-deploy-ssh"] +order: + - deploy-helm.md + - deploy-k8s.md + - deploy-docker.md + - quick-deploy-ssh.md collapsed: false --- diff --git a/docs/v0.17.0/deploy/_index.md b/docs/v0.17.0/deploy/_index.md index fbb5b7f..2acc965 100644 --- a/docs/v0.17.0/deploy/_index.md +++ b/docs/v0.17.0/deploy/_index.md @@ -1,5 +1,9 @@ --- -order: ["deploy-helm", "deploy-k8s", "deploy-docker", "quick-deploy-ssh"] +order: + - deploy-helm.md + - deploy-k8s.md + - deploy-docker.md + - quick-deploy-ssh.md collapsed: false --- diff --git a/docs/zh/deploy/_index.md b/docs/zh/deploy/_index.md index a909e7a..0cefe94 100644 --- a/docs/zh/deploy/_index.md +++ b/docs/zh/deploy/_index.md @@ -1,5 +1,9 @@ --- -order: ["deploy-helm", "deploy-k8s", "deploy-docker", "quick-deploy-ssh"] +order: + - deploy-helm.md + - deploy-k8s.md + - deploy-docker.md + - quick-deploy-ssh.md collapsed: false --- diff --git a/docs/zh/v0.16.0/deploy/_index.md b/docs/zh/v0.16.0/deploy/_index.md index a909e7a..0cefe94 100644 --- a/docs/zh/v0.16.0/deploy/_index.md +++ b/docs/zh/v0.16.0/deploy/_index.md @@ -1,5 +1,9 @@ --- -order: ["deploy-helm", "deploy-k8s", "deploy-docker", "quick-deploy-ssh"] +order: + - deploy-helm.md + - deploy-k8s.md + - deploy-docker.md + - quick-deploy-ssh.md collapsed: false --- diff --git a/docs/zh/v0.17.0/deploy/_index.md b/docs/zh/v0.17.0/deploy/_index.md index a909e7a..0cefe94 100644 --- a/docs/zh/v0.17.0/deploy/_index.md +++ b/docs/zh/v0.17.0/deploy/_index.md @@ -1,5 +1,9 @@ --- -order: ["deploy-helm", "deploy-k8s", "deploy-docker", "quick-deploy-ssh"] +order: + - deploy-helm.md + - deploy-k8s.md + - deploy-docker.md + - quick-deploy-ssh.md collapsed: false ---