diff --git a/README.md b/README.md index de526c45..ac994d1e 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,20 @@ +#### English | [简体中文](https://github.com/opencurve/curve-operator/tree/master/docs/readme_cn.md) + # Curve-Operator ## What's curve-operator -The curve-operator project helps deploy a Curve cluster orchestrated by Kubernetes.(Currently only supports CurveBS cluster deployment) +curve-operator manages [Curve](https://github.com/opencurve/curve) cluster on [Kuberentes](https://kubernetes.io/docs/home/). It make Curve a truly cloud-native distributed storage system. -![Curve BS deploy architecture](./docs/images/curvebs-deploy-arch.png) +![Curve BS deploy architecture](./docs/images/curve-deploy-arch.jpg) -## Prerequisite +## Setup -* Kubernetes 1.19,1.20 -* Three-nodes cluster(stand-alone deployment is not supported at persent) +### 0.Prerequisite -## Install and Deploy +* Kubernetes 1.19,1.20 -### 1. Install Operator +### 1.Install Operator The first step is to install the Curve Operator. @@ -23,126 +24,35 @@ $ cd curve-operator $ kubectl apply -f config/deploy/ ``` -verify the curve-operator is in the `Running` state in `curve` namespace. +verify the curve-operator on the `Running` state in `curve` namespace. ```shell $ kubectl get pod -n curve - NAME READY STATUS RESTARTS AGE curve-operator-69bc69c75d-jfsjg 1/1 Running 0 7s ``` -### 2. Deploy Curve cluster +### 2. Deploy Curve Cluster -#### Modify the cluster Declarative yaml file +Operator deploys Curve cluster based on declarative API. You can get and modify customized the cluster yaml file in [config/sample.](https://github.com/opencurve/curve-operator/tree/master/config/samples) -```shell -$ vim config/samples/cluster.yaml -``` +[CurveBS Stand-alone deployment](https://github.com/opencurve/curve-operator/blob/master/config/samples/bscluster-onehost.yaml) -```yaml -apiVersion: operator.curve.io/v1 -kind: CurveCluster -metadata: - name: curvebs-cluster-cloud - # The namespace to deploy CurveBS cluster. - # Curve operator is deployed in this namespace,Do not modify if not necessary - namespace: curve -spec: - # The container image used to launch the Curve daemon pods(etcd, mds, chunkserver, snapshotclone). - # v1.2 is Pacific and v1.3 is not tested. - curveVersion: - image: opencurvedocker/curvebs:v1.2.6 - # Container image pull policy, - # By default the pull policy of all containers in that pod will be set to IfNotPresent if it is not explicitly specified and no modification necessary. - imagePullPolicy: IfNotPresent - # The K8s cluster nodes name in cluster that prepare to deploy Curve daemon pods(etcd, mds, snapshotclone). - # For stand-alone deploy, set one node here and see bscluster-onehost.yaml - # - node1 -> etcd-a, mds-a, snapshotclone-a - # - node2 -> etcd-b, mds-b, snapshotclone-b - # - node3 -> etcd-c, mds-c, snapshotclone-c - nodes: - - curve-operator-node1 - - curve-operator-node2 - - curve-operator-node3 - # hostDataDir where data files and log files will be persisted on host machine. Must be specified. - # If you reinstall the cluster, make surce that you delete this directory from each host. - hostDataDir: /curvebs - etcd: - # Port for listening to partner communication. - # Etcd member accept incoming requests from its peers on a specific scheme://IP:port combination and the IP is host ip because we use hostnetwork:true. - peerPort: 23800 - # clientPort for listening server port. - clientPort: 23790 - mds: - port: 6700 - dummyPort: 7700 - storage: - # useSelectedNodes is to control whether to use individual nodes and their configured devices can be specified as well. - # This field is not implemented at present and is must set false here. - # You can refer following selectoedNodes setting commented if the function is completed later. - # But the func is not implemented yet. - useSelectedNodes: false - # The hosts specified to deployment chunkserver as storage resource. - # And you can configure the same nodes above configure that deploy etcd, mds and snapshotclone service. - nodes: - - curve-operator-node1 - - curve-operator-node2 - - curve-operator-node3 - port: 8200 - copySets: 100 - # Make sure the devices configured are available on hosts above. - devices: - - name: /dev/vdc - mountPath: /data/chunkserver0 - percentage: 80 - # Not implement yet - #selectedNodes: - #- node: curve-operator-node1 - # - devices: - # name: /dev/vdd - # mountPath: /data/chunkserver1 - # percentage: 90 - # - devices: - # name: /dev/vdf - # mountPath: /data/chunkserver2 - # percentage: 80 - #- node: curve-operator-node2 - # - devices: - # name: - # mountPath: - # percentage: - snapShotClone: - # set false if there is no S3 service available temporarily or don't need to use the snapshot clone service - # Make sure s3 service exist if enable is set true - enable: false - port: 5555 - dummyPort: 8800 - proxyPort: 8900 - s3Config: - # Access Key for the S3 service. Uploading snapshots - ak: <> - # Access Key for the S3 service. Uploading snapshots - sk: <> - # S3 service address - nosAddress: <> - # S3 service bucket name to store snapshots - bucketName: <> -``` +[CurveBS three replicas deployment](https://github.com/opencurve/curve-operator/blob/master/config/samples/cluster.yaml) + +[CurveFS Stand-alone deployment](https://github.com/opencurve/curve-operator/blob/master/config/samples/fscluster-onehost.yaml) -#### Deploy cluster +[CurveFS three replicase deployment](https://github.com/opencurve/curve-operator/blob/master/config/samples/fscluster.yaml) -The specific meaning of each configuration item is already in the comments above `cluster.yaml`, which can be modified according to the instructions. Then you can deploy it in cluster. +Here we take the deployment of a three-node `CurveBS` cluster as an example. The yaml file is [cluster.yaml](https://github.com/opencurve/curve-operator/blob/master/config/samples/cluster.yaml) and you can learn more about how to modify this configuration file through the comments in it. -Create the cluster: +Type the command to create the cluster： ```shell $ kubectl apply -f config/samples/cluster.yaml - -// or you can deploy stand-alone cluster using `bscluster-onhost.yaml` ``` -using `kubectl` to list pods in the curve namespace. You should be able to see the following pods once they are all running. The chunkserver numbers of will depend on the number of nodes in the cluster and the the number of devices configured. +list all pods in the `curve` namespace: ```shell $ kubectl -n curve get pod @@ -167,7 +77,9 @@ read-config-k272k 0/1 Completed > Tips: The chunkserver pods may not start immediately, because the disk needs to be formatted in the background(`prepare-chunkfile` jobs), so it may take a while to see the chunkserver pod. The waiting time is determined according to the number and percentage of configured disks, and it may be a long time. -To verify that the cluster is in healthy state, enter one curve-chunkserver pod and `curve_ops_tools status` command to check. +### 3. Check cluster health + +To verify that the cluster is in healthy state, enter one `curve-chunkserver` pod and type `curve_ops_tools status` command to check. ```shell $ kubectl exec -it -- bash @@ -186,35 +98,31 @@ nebd-server: version-1.2.5+2c4861ca: 1 ... ``` -The cluster is deploymented completed and successfully if you see `cluster is healthy` prompt. +The cluster deployment completed and successfully if you see `cluster is healthy` prompt. -### 3. Use Curve CSI +## Curve CSI -After installing the operator and curve cluster, you can create a PVC that to use curvebs as pod storage. +Create a PVC that to use curvebs as pod storage. -Refer to the documentation [deploy curve-csi](https://ask.opencurve.io/t/topic/89) to deploy curve-csi and use curve as backend storage of pod. +you can deploy and get more details from [curve-csi](https://github.com/opencurve/curve-csi) project that dock `curvebs` cluster or [curvefs-csi](https://github.com/opencurve/curvefs-csi) project that dock `curvefs` cluster. -More details can see `curve-csi` project at [curve-csi github](https://github.com/opencurve/curve-csi). +## Remove -## Uninstall curve cluster +Remove curve cluster deployed already and clean up data on host. -You can uninstall curve cluster deployed and clean up data on host. - -### 1. Delete the `Curvecluster` CR: +### 1.Delete the cluster cr ```shell $ kubectl -n curve delete curvecluster my-cluster ``` -Verify that the cluster CR has been deleted before continuing to the next step. +Verify the cluster CR has been deleted before continuing to the next step. ```shell $ kubectl -n curve get curvecluster ``` -### 2. Delete the Operator and related Resources - -This will begin the process of the curve-operator and all other resources being cleaned up. +### 2.Delete the Operator and related Resources ```shell $ kubectl delete -f config/deploy/ @@ -222,7 +130,7 @@ $ kubectl delete -f config/deploy/ ### 3. Delete data and log on host -The final cleanup setp requires deleting files on each host in the cluster. All files under the `hostDataDir` property specified in the cluster CRD will need to be deleted. Otherwise, inconsistent state will remain when a new cluster is started. +The final cleanup step requires deleting files on each host in the cluster. All files under the `hostDataDir` property specified in the cluster CRD will need to be deleted. Otherwise, inconsistent state will remain when a new cluster is started. Connect to each machine and delete `/curvebs`, or the path specified by the `dataDirHostPath` and `logDirHostPath`. @@ -230,4 +138,18 @@ Connect to each machine and delete `/curvebs`, or the path specified by the `dat $ rm -rf /curvebs ``` -In the future this setp will not neccssary that we can delete it by running job on cluster if `cleanUpConfirm` is set. +## Contributing + +We welcome help in any form, including but not limited to improving documentation, asking questions, fixing bugs, and adding features. + +## Meeting + +We have an online community meeting every two weeks which talk about what `Curve` is doing and planning to do. You can view meeting minutes and agenda here [Double Week Meetings](https://github.com/opencurve/curve-meetup-slides/tree/main/2023/Double%20Week%20Meetings). + +## License + +You are required to comply with the [CNCF](https://www.cncf.io/) Code of Conduct while participating in this project. + +## Contact + +If you encounter any problems during use, please submit an [Issue](https://github.com/opencurve/curve-operator/issues) for feedback. You can also scan [the WeChat QR code](https://github.com/opencurve/curve-operator/tree/master/docs/images/curve-wechat.jpeg) to join the technical exchange group. diff --git a/docs/images/curve-deploy-arch.jpg b/docs/images/curve-deploy-arch.jpg new file mode 100644 index 00000000..db3f6243 Binary files /dev/null and b/docs/images/curve-deploy-arch.jpg differ diff --git a/docs/images/curve-wechat.jpeg b/docs/images/curve-wechat.jpeg new file mode 100644 index 00000000..0c7514c3 Binary files /dev/null and b/docs/images/curve-wechat.jpeg differ diff --git a/docs/images/curvebs-deploy-arch.png b/docs/images/curvebs-deploy-arch.png deleted file mode 100644 index 75792f0b..00000000 Binary files a/docs/images/curvebs-deploy-arch.png and /dev/null differ diff --git a/docs/readme_cn.md b/docs/readme_cn.md new file mode 100644 index 00000000..912ccc4d --- /dev/null +++ b/docs/readme_cn.md @@ -0,0 +1,155 @@ +#### 简体中文 | [English](https://github.com/opencurve/curve-operator/blob/master/README.md) + +# Curve-Operator + +## 简介 + +Curve-Operator用于管理 [Kubernetes](https://kubernetes.io/docs/home/) 系统上的 [Curve](https://github.com/opencurve/curve) 集群。更进一步地支持Curve实现云原生的分布式存储系统。 + +![Curve BS deploy architecture](./images/curve-deploy-arch.jpg) + +## 部署 + +### 0.先决条件 + +* 安装Kubernetes1.19/1.20版本。 + +### 1.安装Operator + +首先需要安装Curve Operaotr，然后再部署 Curve 集群。 + +```shell +$ git clone https://github.com/opencurve/curve-operator.git +$ cd curve-operator +$ kubectl apply -f config/deploy/ +``` + +安装成功后，默认会创建`curve` namespace，确认curve-operator处于Running状态。 + +```shell +$ kubectl get pod -n curve +NAME READY STATUS RESTARTS AGE +curve-operator-69bc69c75d-jfsjg 1/1 Running 0 7s +``` + +### 2.部署Curve集群 + +Operator部署集群基于声明式的API，你可以在这个目录 [config/sample](https://github.com/opencurve/curve-operator/tree/master/config/samples) 下找到所有的声明式yaml例子。并且可以自定义修改从而能够符合你当前的部署环境。 + +[CurveBS 单机部署](https://github.com/opencurve/curve-operator/blob/master/config/samples/bscluster-onehost.yaml) + +[CurveBS 三副本部署](https://github.com/opencurve/curve-operator/blob/master/config/samples/cluster.yaml) + +[CurveFS 单机部署](https://github.com/opencurve/curve-operator/blob/master/config/samples/fscluster-onehost.yaml) + +[CurveFS 三副本部署](https://github.com/opencurve/curve-operator/blob/master/config/samples/fscluster.yaml) + +这里我们以部署三副本的 `CurveBS` 集群为例进行说明。这里的声明文件是[cluster.yaml](https://github.com/opencurve/curve-operator/blob/master/config/samples/cluster.yaml)。你可以根据yaml文件中的注释详细的了解每一个配置项的作用，从而自定义修改。 + +输入如下命令创建集群: + +```shell +$ kubectl apply -f config/samples/cluster.yaml +``` + +查看当前 `curve` namespac 下的所有的pod： + +```shell +$ kubectl -n curve get pod + +NAME READY STATUS RESTARTS AGE +curve-chunkserver-curve-operator-node1-vdc-556fc99467-5nx9q 1/1 Running 0 5m45s +curve-chunkserver-curve-operator-node2-vdc-7cf89768f9-hmcrs 1/1 Running 0 5m45s +curve-chunkserver-curve-operator-node3-vdc-f77dd85dc-z5bws 1/1 Running 0 5m45s +curve-etcd-a-d5bbfb755-lzgrm 1/1 Running 0 41m +curve-etcd-b-66c5b54f75-6nnnt 1/1 Running 0 41m +curve-etcd-c-86b7964f87-cj8zk 1/1 Running 0 41m +curve-mds-a-7b5989bddd-ln2sm 1/1 Running 0 40m +curve-mds-b-56d8f58645-gv6pd 1/1 Running 0 40m +curve-mds-c-997c7fd-vt5hw 1/1 Running 0 40m +gen-logical-pool-rzhlz 0/1 Completed 0 5m15s +gen-physical-pool-chnw8 0/1 Completed 0 5m45s +prepare-chunkfile-curve-operator-node1-vdc-znb66 0/1 Completed 0 40m +prepare-chunkfile-curve-operator-node2-vdc-6gf2z 0/1 Completed 0 40m +prepare-chunkfile-curve-operator-node3-vdc-2bkxm 0/1 Completed 0 40m +read-config-k272k 0/1 Completed 0 41m +``` + +> 说明：在执行完apply命令之后，通过get命令你可以不能立刻就能看到`chunkserver` pods。因为磁盘需要进行格式化，这个格式化的过程是通过`prepare-chunkfile` jobs去完成的。所以可能会等待一段时间之后才会看到所有的`chunkserver` pods。 +> +> 具体的等待时间需要根据你的磁盘的大小以及你定义的格式化的百分比，这个时间可能会很长。你可以通过日志查看格式化的进程。 + +### 3.检查集群的健康状态 + +为了检查部署的集群是否是健康的，需要进入任何一个`curve-chunkserver` pod，然后使用`curve_ops_tool status` 去查看集群的健康状态。 + +```shell +$ kubectl exec -it -- bash +$ curve_ops_tool status + +Cluster status: +cluster is healthy +total copysets: 100, unhealthy copysets: 0, unhealthy_ratio: 0% +physical pool number: 1, logical pool number: 1 +Space info: +physical: total = 1178GB, used = 6GB(0.56%), left = 1171GB(99.44%) +logical: total = 392GB, used = 41GB(10.44%, can be recycled = 0GB(0.00%)), left = 351GB(89.56%), created file size = 60GB(15.28%) + +Client status: +nebd-server: version-1.2.5+2c4861ca: 1 +... +``` + +## Curve CSI + +在Kubernetes系统中，需要创建PVC从而使用 `Curve` 作为pod的后端存储。 + +你可以部署对接 Curvebs 集群的 [curve-csi](https://github.com/opencurve/curve-csi) 或者是对接 Curvefs 集群的 [curvefs-csi](https://github.com/opencurve/curvefs-csi)。如何部署以及具体的实现细节可以参考对应的项目文档。 + +## 删除 + +为了删除已经部署的Curve集群并且清楚其中的数据，可能需要经历如下步骤。 + +### 1.删除集群CR + +```shell +$ kubectl -n curve delete curvecluster my-cluster +``` + +为了验证cluster CR已经被删除，你可以通过如下命令查看： + +```shell +$ kubectl -n curve get curvecluster +``` + +### 2.删除Operator以及相关的资源 + +```shell +$ kubectl delete -f config/deploy/ +``` + +### 3.删除持久化在宿主机的数据和日志（慎重） + +为了彻底的清除集群，需要把集群中的数据和日志全部清除。这个目录是在`cluster.yaml`文件中定义的`hostDataDir`配置项。注意，在部署新集群之前一定要将这个配置目录下的内容删除。 + +对于多副本部分的集群，数据分布在各个集群节点上，所以需要登录各个节点进行删除。例如，如果配置的目录是`/curvebs`的话，则需要删除这个目录下的所有数据和日志： + +```shell +rm -rf /curvebs +``` + +## 贡献代码 + +我们欢迎任何形式的帮助，包括但不限定于完善文档、提出问题、修复 Bug 和增加特性。 + +## 行为准则 + +您在参与本项目的过程中须遵守 [CNCF 行为准则](https://github.com/cncf/foundation/blob/master/code-of-conduct.md)。 + +## 社区例会 + +我们会在每双周四 17:00 CST 进行例会并对Curve社区当前的状况和未来的发展进行讨论，您可以在 [这里](https://github.com/opencurve/curve-meetup-slides/tree/main/2023/Double%20Week%20Meetings) 查看会议纪要以及日程。 + +## 联系我们 + +如果您在使用过程中遇到了任何问题，欢迎提交 [Issue](https://github.com/opencurve/curve-operator/issues) 进行反馈。您也可以扫描 [微信二维码](https://github.com/opencurve/curve-operator/tree/master/docs/images/curve-wechat.jpeg) 联系小助手加入技术交流群。 \ No newline at end of file diff --git a/pkg/controllers/client.go b/pkg/controllers/client.go index dd640f8a..94f080df 100644 --- a/pkg/controllers/client.go +++ b/pkg/controllers/client.go @@ -5,8 +5,6 @@ import ( "context" "fmt" - "github.com/opencurve/curve-operator/pkg/daemon" - "github.com/opencurve/curve-operator/pkg/k8sutil" "github.com/pkg/errors" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -14,6 +12,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/remotecommand" + + "github.com/opencurve/curve-operator/pkg/daemon" + "github.com/opencurve/curve-operator/pkg/k8sutil" ) const ( @@ -86,6 +87,7 @@ func createSyncDeployment(c *daemon.Cluster) error { return err } } + // update condition type and phase etc. return nil } diff --git a/pkg/controllers/cluster.go b/pkg/controllers/cluster.go index a6b6d26f..1cc125c0 100644 --- a/pkg/controllers/cluster.go +++ b/pkg/controllers/cluster.go @@ -1,6 +1,7 @@ package controllers import ( + "context" "time" "github.com/coreos/pkg/capnslog" @@ -8,7 +9,7 @@ import ( "github.com/opencurve/curve-operator/pkg/chunkserver" "github.com/opencurve/curve-operator/pkg/daemon" "github.com/opencurve/curve-operator/pkg/etcd" - + "github.com/opencurve/curve-operator/pkg/k8sutil" "github.com/opencurve/curve-operator/pkg/logrotate" "github.com/opencurve/curve-operator/pkg/mds" "github.com/opencurve/curve-operator/pkg/metaserver" @@ -122,6 +123,12 @@ func reconcileCurveDaemons(c *daemon.Cluster) error { return err } + // clean up the cluster install environment + err = cleanClusterInstallEnv(c) + if err != nil { + return err + } + return nil } @@ -154,5 +161,16 @@ func reconcileCurveFSDaemons(c *daemon.Cluster) error { return err } + // clean up the cluster install environment + err = cleanClusterInstallEnv(c) + if err != nil { + return err + } + return nil } + +// cleanClusterInstallEnv clean up the cluster install environment +func cleanClusterInstallEnv(c *daemon.Cluster) error { + return k8sutil.DeleteSyncConfigDeployment(context.TODO(), &c.Context, SyncConfigDeployment, c.Namespace) +} diff --git a/pkg/k8sutil/deployment.go b/pkg/k8sutil/deployment.go index c98181a9..97204726 100644 --- a/pkg/k8sutil/deployment.go +++ b/pkg/k8sutil/deployment.go @@ -7,6 +7,7 @@ import ( appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" "github.com/opencurve/curve-operator/pkg/clusterd" "github.com/opencurve/curve-operator/pkg/k8sutil/patch" @@ -99,3 +100,22 @@ func WaitForDeploymentToStart(ctx context.Context, clusterdContext *clusterd.Con } return fmt.Errorf("gave up waiting for deployment %q to update", deployment.Name) } + +// DeleteSyncConfigDeployment delete the SyncConfigDeployment after the cluster is deployed. +func DeleteSyncConfigDeployment(ctx context.Context, clusterdContext *clusterd.Context, syncConfigDeployment, namespace string) error { + err := retry.OnError(retry.DefaultRetry, func(err error) bool { + // retrying for any error that occurs + return true + }, func() error { + return clusterdContext.Clientset.AppsV1().Deployments(namespace).Delete(syncConfigDeployment, &metav1.DeleteOptions{}) + }) + + if err != nil { + return fmt.Errorf("failed to delete deployment %q after the curve cluster has been deployed. %v", + syncConfigDeployment, err) + } + + logger.Infof("the curve cluster has been deployed and the deployment %q has been deleted", syncConfigDeployment) + + return nil +}