From af4af7f427ed2887b42c94dbde2d07bbe5687714 Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Mon, 11 Sep 2023 15:23:49 +0800 Subject: [PATCH] update doc Signed-off-by: zhihuiwan <15779896112@163.com> --- doc/quick_start.md | 484 +++++++++++------------------------------- doc/quick_start.zh.md | 84 ++++---- doc/system_conf.md | 145 ++++++------- doc/system_conf.zh.md | 36 ++-- 4 files changed, 257 insertions(+), 492 deletions(-) diff --git a/doc/quick_start.md b/doc/quick_start.md index 970e9288b..8dd73a2e0 100644 --- a/doc/quick_start.md +++ b/doc/quick_start.md @@ -1,63 +1,70 @@ -## 快速入门 - -### 1. 环境部署 -以下三种模式可根据需求自行选择一种 -#### 1.1 Pypi包安装 -说明：此方式的运行模式为单机模式 -##### 1.1.1 安装 -- [conda](https://docs.conda.io/projects/miniconda/en/latest/)环境准备及安装 -- 创建虚拟环境 +# Quick Start + +## 1. Environment Setup +You can choose one of the following three deployment modes based on your requirements: + +### 1.1 Pypi Package Installation +Note: This mode operates in a single-machine mode. + +#### 1.1.1 Installation +- Prepare and install [conda](https://docs.conda.io/projects/miniconda/en/latest/) environment. +- Create a virtual environment: ```shell -# fate的运行环境为python>=3.8 +# FATE requires Python >= 3.8 conda create -n fate_env python=3.8 conda activate fate_env ``` -- 安装fate flow及相关依赖 +- Install FATE Flow and related dependencies: ```shell pip install fate_client[fate,fate_flow]==2.0.0.b0 ``` -##### 1.1.2 服务初始化 +#### 1.1.2 Service Initialization ```shell fate_flow init --ip 127.0.0.1 --port 9380 --home $HOME_DIR ``` -- ip: 服务运行ip -- port：服务运行时的http端口 -- home: 数据存储目录。主要包括：数据/模型/日志/作业配置/sqlite.db等内容 +- `ip`: The IP address where the service runs. +- `port`: The HTTP port the service runs on. +- `home`: The data storage directory, including data, models, logs, job configurations, and SQLite databases. -##### 1.1.3 服务启停 +#### 1.1.3 Service Start/Stop ```shell fate_flow status/start/stop/restart ``` -#### 1.2 单机版部署 -参考[单机版部署](https://github.com/FederatedAI/FATE/blob/v2.0.0-beta/deploy/standalone-deploy/README.md) +### 1.2 Standalone Deployment +Refer to [Standalone Deployment](https://github.com/FederatedAI/FATE/blob/v2.0.0-beta/deploy/standalone-deploy/README.md). + +### 1.3 Cluster Deployment +Refer to [All-in-one Deployment](https://github.com/FederatedAI/FATE/blob/v2.0.0-beta/deploy/cluster-deploy/allinone/fate-allinone_deployment_guide.md). + +## 2. User Guide +FATE provides client tools including SDK, CLI, and Pipeline. If you don't have FATE Client deployed in your environment, you can download it using `pip install fate_client`. The following operations are based on CLI. + +### 2.1 Data Upload +In version 2.0-beta, data uploading is a two-step process: -#### 1.3 集群部署 -参考[allinone部署](https://github.com/FederatedAI/FATE/blob/v2.0.0-beta/deploy/cluster-deploy/allinone/fate-allinone_deployment_guide.md) +- **upload**: Uploads data to FATE-supported storage services. +- **transformer**: Transforms data into a DataFrame. -### 2. 使用指南 -fate提供的客户端包括SDK、CLI和Pipeline，若你的环境中没有部署FATE Client,可以使用`pip install fate_client`下载，以下的使用操作均基于cli编写。 +#### 2.1.1 upload +##### 2.1.1.1 Configuration and Data +- Upload configuration can be found at [examples-upload](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-beta/examples/upload), and the data is located at [upload-data](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-beta/examples/data). +- You can also use your own data and modify the "meta" information in the upload configuration. -#### 2.1 数据上传 -在2.0-beta版本中，数据上传分为两步： -- upload: 将数据上传到FATE支持存储服务中 -- transformer: 将数据转化成dataframe -##### 2.1.1 upload -##### 2.1.1.1 配置及数据 - - 上传配置位于[examples-upload](../examples/upload)，上传数据位于[upload-data](../examples/data) - - 你也可以使用自己的数据，并修改upload配置中的"meta"信息。 -##### 2.1.1.2 上传guest方数据 +##### 2.1.1.2 Upload Guest Data ```shell flow data upload -c examples/upload/upload_guest.json ``` -- 需要记录返回的name和namespace，作为transformer的参数。 -##### 2.1.1.3 上传host方数据 +- Record the returned "name" and "namespace" for use in the transformer phase. + +##### 2.1.1.3 Upload Host Data ```shell flow data upload -c examples/upload/upload_host.json ``` -- 需要记录返回的name和namespace，作为transformer的参数。 -##### 2.1.1.4 上传结果 +- Record the returned "name" and "namespace" for use in the transformer phase. + +##### 2.1.1.4 Upload Result ```json { "code": 0, @@ -69,68 +76,34 @@ flow data upload -c examples/upload/upload_host.json "message": "success" } ``` -其中"namespace"和"name"是这份数据在fate中的标识，以便下面后续transformer阶段使用时可直接引用。 +Where "namespace" and "name" identify the data in FATE for future reference in the transformer phase. -##### 2.1.1.5 数据查询 -因为upload为异步操作，需要确认是否上传成功才可进行后续操作。 +##### 2.1.1.5 Data Query +Since upload is an asynchronous operation, you need to confirm if it was successful before proceeding to the next step. ```shell flow table query --namespace upload --name 36491bc8-3fef-11ee-be05-16b977118319 ``` -上传成功信息如下： -```json -{ - "code": 0, - "data": { - "count": 569, - "data_type": "table", - "engine": "standalone", - "meta": { - "delimiter": ",", - "dtype": "'float32", - "header": "extend_sid,id,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15,x16,x17,x18,x19", - "input_format": "dense", - "label_type": "int", - "match_id_name": "id", - "match_id_range": 0, - "sample_id_name": "extend_sid", - "tag_value_delimiter": ":", - "tag_with_value": false, - "weight_type": "float32" - }, - "name": "36491bc8-3fef-11ee-be05-16b977118319", - "namespace": "upload", - "path": "xxx", - "source": { - "component": "upload", - "output_artifact_key": "data", - "output_index": null, - "party_task_id": "", - "task_id": "", - "task_name": "upload" - } - }, - "message": "success" -} +If the returned code is 0, the upload was successful. -``` -若返回的code为0即为上传成功。 - -##### 2.1.2 transformer -##### 2.1.2.1 配置 - - transformer配置位于[examples-transformer](../examples/transformer) -##### 2.1.2.2 transformer guest -- 配置路径位于： examples/transformer/transformer_guest.json -- 修改配置中"data_warehouse"的"namespace"和"name"：上面upload guest阶段的输出 +#### 2.1.2 Transformer +##### 2.1.2.1 Configuration +- Transformer configuration can be found at [examples-transformer](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-beta/examples/transformer). + +##### 2.1.2.2 Transform Guest Data +- Configuration path: examples/transformer/transformer_guest.json +- Modify the "namespace" and "name" in the "data_warehouse" section to match the output from the guest data upload. ```shell flow data transformer -c examples/transformer/transformer_guest.json ``` -##### 2.1.2.3 transformer host -- 配置路径位于： examples/transformer/transformer_host.json -- 修改配置中"data_warehouse"的"namespace"和"name"：上面upload host阶段的输出 + +##### 2.1.2.3 Transform Host Data +- Configuration path: examples/transformer/transformer_host.json +- Modify the "namespace" and "name" in the "data_warehouse" section to match the output from the host data upload. ```shell flow data transformer -c examples/transformer/transformer_host.json ``` -##### 2.1.2.4 transformer结果 + +##### 2.1.2.4 Transformer Result ```json { "code": 0, @@ -142,31 +115,31 @@ flow data transformer -c examples/transformer/transformer_host.json "message": "success" } ``` -其中"namespace"和"name"是这份数据在fate中的标识，后续建模作业中使用。 - -##### 2.1.2.5 查看数据是否上传成功 +Where "namespace" and "name" identify the data in FATE for future modeling jobs. -因为transformer也是异步操作，需要确认是否上传成功才可进行后续操作。 +##### 2.1.2.5 Check if Data Upload Was Successful +Since the transformer is also an asynchronous operation, you need to confirm if it was successful before proceeding. ```shell -flow table query --namespace experiment --name breast_hetero_guest +flow table query --namespace experiment --name breast_hetero_guest ``` ```shell -flow table query --namespace experiment --name breast_hetero_host +flow table query --namespace experiment --name breast_hetero_host ``` -若返回的code为0即为上传成功。 - -#### 2.2 开始FATE作业 -##### 2.2.1 提交作业 -当你的数据准备好后，可以开始提交作业给FATE Flow： -- 训练job配置example位于[lr-train](../examples/lr/train_lr.yaml); -- 预测job配置example位于[lr-predict](../examples/lr/predict_lr.yaml);预测任务需要修改"dag.conf.model_warehouse"成训练作业的输出模型。 -- 训练和预测job配置中站点id为"9998"和"9999"。如果你的部署环境为集群版，需要替换成真实的站点id；单机版可使用默认配置。 -- 如果想要使用自己的数据，可以更改配置中guest和host的data_warehouse的namespace和name -- 提交作业的命令为: +If the returned code is 0, the upload was successful. + +### 2.2 Starting FATE Jobs +#### 2.2.1 Submitting a Job +Once your data is prepared, you can start submitting jobs to FATE Flow: + +- The configuration for training jobs can be found in [lr-train](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-beta/examples/lr/train_lr.yaml). +- The configuration for prediction jobs can be found in [lr-predict](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-beta/examples/lr/predict_lr.yaml). To use it, modify the "dag.conf.model_warehouse" to point to the output model of your training job. +- In the training and prediction job configurations, the site IDs are set to "9998" and "9999." If your deployment environment is the cluster version, you need to replace them with the actual site IDs. For the standalone version, you can use the default configuration. +- If you want to use your own data, you can change the "namespace" and "name" of "data_warehouse" for both the guest and host in the configuration. +- To submit a job, use the following command: ```shell flow job submit -c examples/lr/train_lr.yaml ``` -- 提交成功返回结果: +- A successful submission will return the following result: ```json { "code": 0, @@ -177,37 +150,37 @@ flow job submit -c examples/lr/train_lr.yaml "job_id": "202308211911505128750", "message": "success" } - ``` -这里的"data"内容即为该作业的输出模型。 +The "data" section here contains the output model of the job. -##### 2.2.2 查询作业 -在作业的运行过程时，你可以通过查询命令获取作业的运行状态 +#### 2.2.2 Querying a Job +While a job is running, you can check its status using the query command: ```shell flow job query -j $job_id ``` -##### 2.2.3 停止作业 -在作业的运行过程时，你可以通过停止作业命令来终止当前作业 +#### 2.2.3 Stopping a Job +During job execution, you can stop the current job using the stop command: ```shell flow job stop -j $job_id ``` -##### 2.2.4 重跑作业 -在作业的运行过程时，如果运行失败，你可以通过重跑命令来重跑当前作业 +#### 2.2.4 Rerunning a Job +If a job fails during execution, you can rerun it using the rerun command: ```shell flow job rerun -j $job_id ``` -#### 2.3 获取作业输出结果 -作业的输出包括数据、模型和指标 -##### 2.3.1 输出指标 -查询输出指标命令： +### 2.3 Obtaining Job Outputs +Job outputs include data, models, and metrics. + +#### 2.3.1 Output Metrics +To query output metrics, use the following command: ```shell flow output query-metric -j $job_id -r $role -p $party_id -tn $task_name ``` -如使用上面的训练dag提交任务，可以使用`flow output query-metric -j 202308211911505128750 -r arbiter -p 9998 -tn lr_0`查询。 -查询结果如下: +For example, if you used the training DAG from above, you can use `flow output query-metric -j 202308211911505128750 -r arbiter -p 9998 -tn lr_0` to query metrics. +The query result will look like this: ```json { "code": 0, @@ -263,265 +236,62 @@ flow output query-metric -j $job_id -r $role -p $party_id -tn $task_name ], "message": "success" } - ``` - -##### 2.3.2 输出模型 -###### 2.3.2.1 查询模型 +#### 2.3.2 Output Models +##### 2.3.2.1 Querying Models +To query output models, use the following command: ```shell flow output query-model -j $job_id -r $role -p $party_id -tn $task_name ``` -如使用上面的训练dag提交任务，可以使用`flow output query-model -j 202308211911505128750 -r host -p 9998 -tn lr_0`查询。 -查询结果如下： +For example, if you used the training DAG from above, you can use `flow output query-model -j 202308211911505128750 -r host -p 9998 -tn lr_0` to query models. +The query result will be similar to this: + ```json { "code": 0, - "data": { - "output_model": { - "data": { - "estimator": { - "end_epoch": 10, - "is_converged": false, - "lr_scheduler": { - "lr_params": { - "start_factor": 0.7, - "total_iters": 100 - }, - "lr_scheduler": { - "_get_lr_called_within_step": false, - "_last_lr": [ - 0.07269999999999996 - ], - "_step_count": 10, - "base_lrs": [ - 0.1 - ], - "end_factor": 1.0, - "last_epoch": 9, - "start_factor": 0.7, - "total_iters": 100, - "verbose": false - }, - "method": "linear" - }, - "optimizer": { - "alpha": 0.001, - "l1_penalty": false, - "l2_penalty": true, - "method": "sgd", - "model_parameter": [ - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ], - [ - 0.0 - ] - ], - "model_parameter_dtype": "float32", - "optim_param": { - "lr": 0.1 - }, - "optimizer": { - "param_groups": [ - { - "dampening": 0, - "differentiable": false, - "foreach": null, - "initial_lr": 0.1, - "lr": 0.07269999999999996, - "maximize": false, - "momentum": 0, - "nesterov": false, - "params": [ - 0 - ], - "weight_decay": 0 - } - ], - "state": {} - } - }, - "param": { - "coef_": [ - [ - -0.10828543454408646 - ], - [ - -0.07341302931308746 - ], - [ - -0.10850320011377335 - ], - [ - -0.10066638141870499 - ], - [ - -0.04595951363444328 - ], - [ - -0.07001449167728424 - ], - [ - -0.08949052542448044 - ], - [ - -0.10958756506443024 - ], - [ - -0.04012322425842285 - ], - [ - 0.02270071767270565 - ], - [ - -0.07198350876569748 - ], - [ - 0.00548586156219244 - ], - [ - -0.06599288433790207 - ], - [ - -0.06410090625286102 - ], - [ - 0.016374297440052032 - ], - [ - -0.01607361063361168 - ], - [ - -0.011447405442595482 - ], - [ - -0.04352564364671707 - ], - [ - 0.013161249458789825 - ], - [ - 0.013506329618394375 - ] - ], - "dtype": "float32", - "intercept_": null - } - } + "data": [ + { + "model": { + "file": "202308211911505128750_host_9998_lr_0", + "namespace": "202308211911505128750_host_9998_lr_0" }, - "meta": { - "batch_size": null, - "epochs": 10, - "init_param": { - "fill_val": 0.0, - "fit_intercept": false, - "method": "zeros", - "random_state": null - }, - "label_count": false, - "learning_rate_param": { - "method": "linear", - "scheduler_params": { - "start_factor": 0.7, - "total_iters": 100 - } - }, - "optimizer_param": { - "alpha": 0.001, - "method": "sgd", - "optimizer_params": { - "lr": 0.1 - }, - "penalty": "l2" - }, - "ovr": false - } + "name": "HeteroLRHost_9998_0", + "namespace": "202308211911505128750_host_9998_lr_0", + "role": "host", + "party_id": "9998", + "work_mode": 1 } - }, + ], "message": "success" } - ``` -###### 2.3.2.2 下载模型 +##### 2.3.2.2 Downloading Models +To download models, use the following command: ```shell flow output download-model -j $job_id -r $role -p $party_id -tn $task_name -o $download_dir ``` -如使用上面的训练dag提交任务，可以使用`flow output download-model -j 202308211911505128750 -r host -p 9998 -tn lr_0 -o ./`下载。 -下载结果如下： +For example, if you used the training DAG from above, you can use `flow output download-model -j 202308211911505128750 -r host -p 9998 -tn lr_0 -o ./` to download the model. +The download result will be similar to this: + ```json { "code": 0, "directory": "./output_model_202308211911505128750_host_9998_lr_0", "message": "download success, please check the path: ./output_model_202308211911505128750_host_9998_lr_0" } - - ``` - -##### 2.3.3 输出数据 -###### 2.3.3.1 查询数据表 +#### 2.3.3 Output Data +##### 2.3.3.1 Querying Data Tables +To query output data tables, use the following command: ```shell flow output query-data-table -j $job_id -r $role -p $party_id -tn $task_name ``` -如使用上面的训练dag提交任务，可以使用`flow output query-data-table -j 202308211911505128750 -r host -p 9998 -tn binning_0`查询。 -查询结果如下： +For example, if you used the training DAG from above, you can use `flow output query-data-table -j 202308211911505128750 -r host -p 9998 -tn binning_0` to query data tables. +The query result will be similar to this: + ```json { "train_output_data": [ @@ -533,29 +303,29 @@ flow output query-data-table -j $job_id -r $role -p $party_id -tn $task_name } ``` -###### 2.3.3.2 预览数据 +##### 2.3.3.2 Preview Data ```shell flow output display-data -j $job_id -r $role -p $party_id -tn $task_name ``` -如使用上面的训练dag提交任务，可以使用`flow output display-data -j 202308211911505128750 -r host -p 9998 -tn binning_0`预览输出数据。 +To preview output data using the above training DAG submission, you can use the following command: `flow output display-data -j 202308211911505128750 -r host -p 9998 -tn binning_0`. -###### 2.3.3.3 下载数据 +##### 2.3.3.3 Download Data ```shell flow output download-data -j $job_id -r $role -p $party_id -tn $task_name -o $download_dir ``` -如使用上面的训练dag提交任务，可以使用`flow output download-data -j 202308211911505128750 -r guest -p 9999 -tn lr_0 -o ./`下载输出数据。 -下载结果如下： +To download output data using the above training DAG submission, you can use the following command: `flow output download-data -j 202308211911505128750 -r guest -p 9999 -tn lr_0 -o ./`. + +The download result will be as follows: ```json { "code": 0, "directory": "./output_data_202308211911505128750_guest_9999_lr_0", "message": "download success, please check the path: ./output_data_202308211911505128750_guest_9999_lr_0" } - ``` -### 3.更多文档 -- [Restful-api](./swagger/swagger.json) +## 3. More Documentation +- [Restful-api](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-beta/doc/swagger/swagger.yaml) - [CLI](https://github.com/FederatedAI/FATE-Client/blob/dev-2.0.0-beta/python/fate_client/flow_cli/build/doc) - [Pipeline](https://github.com/FederatedAI/FATE/tree/dev-2.0.0-beta/doc/tutorial) -- [FATE算法](https://github.com/FederatedAI/FATE/tree/dev-2.0.0-beta/doc/2.0/components) +- [FATE Algorithms](https://github.com/FederatedAI/FATE/tree/dev-2.0.0-beta/doc/2.0/components) \ No newline at end of file diff --git a/doc/quick_start.zh.md b/doc/quick_start.zh.md index 970e9288b..8363e0932 100644 --- a/doc/quick_start.zh.md +++ b/doc/quick_start.zh.md @@ -1,10 +1,10 @@ -## 快速入门 +# 快速入门 -### 1. 环境部署 +## 1. 环境部署 以下三种模式可根据需求自行选择一种 -#### 1.1 Pypi包安装 +### 1.1 Pypi包安装 说明：此方式的运行模式为单机模式 -##### 1.1.1 安装 +#### 1.1.1 安装 - [conda](https://docs.conda.io/projects/miniconda/en/latest/)环境准备及安装 - 创建虚拟环境 ```shell @@ -17,7 +17,7 @@ conda activate fate_env pip install fate_client[fate,fate_flow]==2.0.0.b0 ``` -##### 1.1.2 服务初始化 +#### 1.1.2 服务初始化 ```shell fate_flow init --ip 127.0.0.1 --port 9380 --home $HOME_DIR ``` @@ -25,39 +25,39 @@ fate_flow init --ip 127.0.0.1 --port 9380 --home $HOME_DIR - port：服务运行时的http端口 - home: 数据存储目录。主要包括：数据/模型/日志/作业配置/sqlite.db等内容 -##### 1.1.3 服务启停 +#### 1.1.3 服务启停 ```shell fate_flow status/start/stop/restart ``` -#### 1.2 单机版部署 +### 1.2 单机版部署 参考[单机版部署](https://github.com/FederatedAI/FATE/blob/v2.0.0-beta/deploy/standalone-deploy/README.md) -#### 1.3 集群部署 +### 1.3 集群部署 参考[allinone部署](https://github.com/FederatedAI/FATE/blob/v2.0.0-beta/deploy/cluster-deploy/allinone/fate-allinone_deployment_guide.md) -### 2. 使用指南 +## 2. 使用指南 fate提供的客户端包括SDK、CLI和Pipeline，若你的环境中没有部署FATE Client,可以使用`pip install fate_client`下载，以下的使用操作均基于cli编写。 -#### 2.1 数据上传 +### 2.1 数据上传 在2.0-beta版本中，数据上传分为两步： - upload: 将数据上传到FATE支持存储服务中 - transformer: 将数据转化成dataframe -##### 2.1.1 upload -##### 2.1.1.1 配置及数据 - - 上传配置位于[examples-upload](../examples/upload)，上传数据位于[upload-data](../examples/data) +#### 2.1.1 upload +#### 2.1.1.1 配置及数据 + - 上传配置位于[examples-upload](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-beta/examples/upload)，上传数据位于[upload-data](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-beta/examples/data) - 你也可以使用自己的数据，并修改upload配置中的"meta"信息。 -##### 2.1.1.2 上传guest方数据 +#### 2.1.1.2 上传guest方数据 ```shell flow data upload -c examples/upload/upload_guest.json ``` - 需要记录返回的name和namespace，作为transformer的参数。 -##### 2.1.1.3 上传host方数据 +#### 2.1.1.3 上传host方数据 ```shell flow data upload -c examples/upload/upload_host.json ``` - 需要记录返回的name和namespace，作为transformer的参数。 -##### 2.1.1.4 上传结果 +#### 2.1.1.4 上传结果 ```json { "code": 0, @@ -71,7 +71,7 @@ flow data upload -c examples/upload/upload_host.json ``` 其中"namespace"和"name"是这份数据在fate中的标识，以便下面后续transformer阶段使用时可直接引用。 -##### 2.1.1.5 数据查询 +#### 2.1.1.5 数据查询 因为upload为异步操作，需要确认是否上传成功才可进行后续操作。 ```shell flow table query --namespace upload --name 36491bc8-3fef-11ee-be05-16b977118319 @@ -115,22 +115,22 @@ flow table query --namespace upload --name 36491bc8-3fef-11ee-be05-16b977118319 ``` 若返回的code为0即为上传成功。 -##### 2.1.2 transformer -##### 2.1.2.1 配置 - - transformer配置位于[examples-transformer](../examples/transformer) -##### 2.1.2.2 transformer guest +#### 2.1.2 transformer +#### 2.1.2.1 配置 + - transformer配置位于[examples-transformer](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-beta/examples/transformer) +#### 2.1.2.2 transformer guest - 配置路径位于： examples/transformer/transformer_guest.json - 修改配置中"data_warehouse"的"namespace"和"name"：上面upload guest阶段的输出 ```shell flow data transformer -c examples/transformer/transformer_guest.json ``` -##### 2.1.2.3 transformer host +#### 2.1.2.3 transformer host - 配置路径位于： examples/transformer/transformer_host.json - 修改配置中"data_warehouse"的"namespace"和"name"：上面upload host阶段的输出 ```shell flow data transformer -c examples/transformer/transformer_host.json ``` -##### 2.1.2.4 transformer结果 +#### 2.1.2.4 transformer结果 ```json { "code": 0, @@ -144,7 +144,7 @@ flow data transformer -c examples/transformer/transformer_host.json ``` 其中"namespace"和"name"是这份数据在fate中的标识，后续建模作业中使用。 -##### 2.1.2.5 查看数据是否上传成功 +#### 2.1.2.5 查看数据是否上传成功 因为transformer也是异步操作，需要确认是否上传成功才可进行后续操作。 ```shell @@ -155,11 +155,11 @@ flow table query --namespace experiment --name breast_hetero_host ``` 若返回的code为0即为上传成功。 -#### 2.2 开始FATE作业 -##### 2.2.1 提交作业 +### 2.2 开始FATE作业 +#### 2.2.1 提交作业 当你的数据准备好后，可以开始提交作业给FATE Flow： -- 训练job配置example位于[lr-train](../examples/lr/train_lr.yaml); -- 预测job配置example位于[lr-predict](../examples/lr/predict_lr.yaml);预测任务需要修改"dag.conf.model_warehouse"成训练作业的输出模型。 +- 训练job配置example位于[lr-train](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-beta/examples/lr/train_lr.yaml); +- 预测job配置example位于[lr-predict](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-beta/examples/lr/predict_lr.yaml);预测任务需要修改"dag.conf.model_warehouse"成训练作业的输出模型。 - 训练和预测job配置中站点id为"9998"和"9999"。如果你的部署环境为集群版，需要替换成真实的站点id；单机版可使用默认配置。 - 如果想要使用自己的数据，可以更改配置中guest和host的data_warehouse的namespace和name - 提交作业的命令为: @@ -181,27 +181,27 @@ flow job submit -c examples/lr/train_lr.yaml ``` 这里的"data"内容即为该作业的输出模型。 -##### 2.2.2 查询作业 +#### 2.2.2 查询作业 在作业的运行过程时，你可以通过查询命令获取作业的运行状态 ```shell flow job query -j $job_id ``` -##### 2.2.3 停止作业 +#### 2.2.3 停止作业 在作业的运行过程时，你可以通过停止作业命令来终止当前作业 ```shell flow job stop -j $job_id ``` -##### 2.2.4 重跑作业 +#### 2.2.4 重跑作业 在作业的运行过程时，如果运行失败，你可以通过重跑命令来重跑当前作业 ```shell flow job rerun -j $job_id ``` -#### 2.3 获取作业输出结果 +### 2.3 获取作业输出结果 作业的输出包括数据、模型和指标 -##### 2.3.1 输出指标 +#### 2.3.1 输出指标 查询输出指标命令： ```shell flow output query-metric -j $job_id -r $role -p $party_id -tn $task_name @@ -267,8 +267,8 @@ flow output query-metric -j $job_id -r $role -p $party_id -tn $task_name ``` -##### 2.3.2 输出模型 -###### 2.3.2.1 查询模型 +#### 2.3.2 输出模型 +##### 2.3.2.1 查询模型 ```shell flow output query-model -j $job_id -r $role -p $party_id -tn $task_name ``` @@ -498,7 +498,7 @@ flow output query-model -j $job_id -r $role -p $party_id -tn $task_name ``` -###### 2.3.2.2 下载模型 +##### 2.3.2.2 下载模型 ```shell flow output download-model -j $job_id -r $role -p $party_id -tn $task_name -o $download_dir ``` @@ -515,8 +515,8 @@ flow output download-model -j $job_id -r $role -p $party_id -tn $task_name -o $d ``` -##### 2.3.3 输出数据 -###### 2.3.3.1 查询数据表 +#### 2.3.3 输出数据 +##### 2.3.3.1 查询数据表 ```shell flow output query-data-table -j $job_id -r $role -p $party_id -tn $task_name ``` @@ -533,13 +533,13 @@ flow output query-data-table -j $job_id -r $role -p $party_id -tn $task_name } ``` -###### 2.3.3.2 预览数据 +##### 2.3.3.2 预览数据 ```shell flow output display-data -j $job_id -r $role -p $party_id -tn $task_name ``` 如使用上面的训练dag提交任务，可以使用`flow output display-data -j 202308211911505128750 -r host -p 9998 -tn binning_0`预览输出数据。 -###### 2.3.3.3 下载数据 +##### 2.3.3.3 下载数据 ```shell flow output download-data -j $job_id -r $role -p $party_id -tn $task_name -o $download_dir ``` @@ -554,8 +554,8 @@ flow output download-data -j $job_id -r $role -p $party_id -tn $task_name -o $do ``` -### 3.更多文档 -- [Restful-api](./swagger/swagger.json) +## 3.更多文档 +- [Restful-api](https://github.com/FederatedAI/FATE-Flow/tree/dev-2.0-beta/doc/swagger/swagger.yaml) - [CLI](https://github.com/FederatedAI/FATE-Client/blob/dev-2.0.0-beta/python/fate_client/flow_cli/build/doc) - [Pipeline](https://github.com/FederatedAI/FATE/tree/dev-2.0.0-beta/doc/tutorial) - [FATE算法](https://github.com/FederatedAI/FATE/tree/dev-2.0.0-beta/doc/2.0/components) diff --git a/doc/system_conf.md b/doc/system_conf.md index 5d83c58cd..8df23fc64 100644 --- a/doc/system_conf.md +++ b/doc/system_conf.md @@ -1,35 +1,35 @@ -### 系统配置描述文档 -FATE Flow使用yaml定义系统配置，配置路径位于: conf/service_conf.yaml, 具体配置内容及其含义如下： +# System Configuration +FATE Flow uses YAML to define system configurations, and the configuration file is located at: `conf/service_conf.yaml`. The specific configuration contents and their meanings are as follows: -| 配置项 | 说明 | 值 | +| Configuration Item | Description | Values | |----------------------|------|------------------------------| -| party_id | 本方站点id | 如: "9999", "10000 | -| use_registry | 是否使用注册中心，当前仅支持zookeeper模式，需要保证zookeeper的配置正确；
注：若使用高可用模式，需保证该配置设置为true | true/false | -| encrypt | 加密模块 | 见[加密模块](#加密模块) | -| fateflow | FATE Flow服务的配置，主要包括端口、命令通道服务、代理等 | 见[FateFlow配置](#fateflow配置) | -| database | 数据库服务的配置信息 | 见[数据库配置](#数据库配置) | -| default_engines | 系统的引擎服务，主要包括计算、存储和通信引擎 | 见[引擎配置](#引擎配置) | -| default_provider | 组件的来源信息，主要包括提供方名称、组件版本和运行模式 | 见[默认注册算法配置](#默认注册算法配置) | -| federation | 通信服务池 | 见[通信引擎池](#通信引擎池) | -| computing | 计算服务池 | 见[计算引擎池](#计算引擎池) | -| storage | 存储服务池 | 见[存储引擎池](#存储配置) | -| hook_module | 钩子配置，当前支持客户端认证、站点端认证以及鉴权钩子 | 见[钩子模块配置](#钩子模块配置) | -| authentication | 认证&&鉴权开关 | 见[认证开关](#认证开关) | -| model_store | 模型存储配置 | 见[模型存储](#模型存储) | -| zookeeper | zookeeper服务的配置 | 见[zookeeper配置](#zookeeper配置) | - -#### 加密模块 +| party_id | Local site ID | For example, "9999", "10000" | +| use_registry | Whether to use a registry center; currently, only ZooKeeper mode is supported, and it requires correct ZooKeeper configuration. Note: If using high availability mode, ensure this configuration is set to true. | true/false | +| encrypt | Encryption module | See [Encryption Module](#encryption-module) | +| fateflow | Configuration for the FATE Flow service, including ports, command channel service, and proxy | See [FateFlow Configuration](#fateflow-configuration) | +| database | Configuration information for the database service | See [Database Configuration](#database-configuration) | +| default_engines | System's engine services, including computing, storage, and communication engines | See [Engine Configuration](#engine-configuration) | +| default_provider | Component source information, including provider name, component version, and execution mode | See [Default Registered Algorithm Configuration](#default-registered-algorithm-configuration) | +| federation | Communication service pool | See [Communication Engine Pool](#communication-engine-pool) | +| computing | Computing service pool | See [Computing Engine Pool](#computing-engine-pool) | +| storage | Storage service pool | See [Storage Engine Pool](#storage-configuration) | +| hook_module | Hook configuration, currently supports client authentication, site authentication, and authorization hooks | See [Hook Module Configuration](#hook-module-configuration) | +| authentication | Authentication and authorization switches | See [Authentication Switch](#authentication-switch) | +| model_store | Model storage configuration | See [Model Storage](#model-storage) | +| zookeeper | ZooKeeper service configuration | See [ZooKeeper Configuration](#zookeeper-configuration) | + +## Encryption Module ```yaml key_0: module: fate_flow.hub.encrypt.password_encrypt#pwdecrypt private_path: private_key.pem ``` -该加密模块主要用于密码(如mysql密码)等内容加密： -- 其中"key_0"为加密模块的key(可以自定义名字)，便于其它配置中直接引用，多套加密模式共存。 - - module: 加密模块，拼接规则为：加密模块 + "#" + 加密函数。 - - private_path：密钥路径。如填相对路径，其根目录位于fate_flow/conf/ +This encryption module is primarily used for encrypting passwords (e.g., MySQL passwords): +- "key_0" is the key for the encryption module (you can customize the name), making it easier to reference in other configurations when multiple encryption modes coexist. + - module: The encryption module, formatted as "encryption module" + "#" + "encryption function." + - private_path: The path to the encryption key. If you provide a relative path, its root directory is `fate_flow/conf/`. -#### FateFlow配置 +## FateFlow Configuration ```yaml host: 127.0.0.1 http_port: 9380 @@ -40,13 +40,13 @@ nginx: http_port: grpc_port: ``` -- host: 主机地址; -- http_port：http端口号; -- grpc_port: grpc端口号; -- proxy_name: 命令通道服务名，支持osx/rollsite/nginx。详细配置需要在[通信引擎池](#通信引擎池) 里面配置; -- nginx: 代理服务配置，用于负载均衡。 +- host: Host address. +- http_port: HTTP port number. +- grpc_port: gRPC port number. +- proxy_name: Command channel service name, supporting osx/rollsite/nginx. Detailed configurations need to be set within [Communication Engine Pool](#communication-engine-pool). +- nginx: Proxy service configuration for load balancing. -#### 数据库配置 +## Database Configuration ```yaml engine: sqlite decrypt_key: @@ -61,12 +61,12 @@ mysql: sqlite: path: ``` -- engine: 数据库引擎名字，如这里填"mysql"，则需要更新mysql的配置详细配置。 -- decrypt_key: 加密模块,需要从[加密模块](#加密模块)中选择。 若不配置，视为不使用密码加密；若使用，则需要将下面的passwd相应设置为密文。 -- mysql: mysql服务配置；若使用密码加密功能，需要将此配置中的"passwd"设置为密文，并在[加密模块](#加密模块)中配置密钥路径 -- sqlite: sqlite文件路径，默认路径为fate_flow/fate_flow_sqlite.db +- engine: Database engine name. If set to "mysql" here, update the detailed MySQL configuration. +- decrypt_key: Encryption module, selected from [Encryption Module](#encryption-module). If not configured, it's considered to not use password encryption. If used, you need to set the "passwd" below to ciphertext and configure the key path in [Encryption Module](#encryption-module). +- mysql: MySQL service configuration. If using password encryption functionality, set the "passwd" in this configuration to ciphertext and configure the key path in [Encryption Module](#encryption-module). +- sqlite: SQLite file path, default path is `fate_flow/fate_flow_sqlite.db`. -#### 引擎配置 +## Engine Configuration ```yaml default_engines: computing: standalone @@ -74,17 +74,17 @@ default_engines: storage: standalone ``` -- computing: 计算引擎，支持"standalone"、"eggroll"、"spark" -- federation: 通信引擎，支持"standalone"、"rollsite"、"osx"、"rabbitmq"、"pulsar" -- storage: 存储引擎，支持"standalone"、"eggroll"、"hdfs" +- computing: Computing engine, supports "standalone," "eggroll," "spark." +- federation: Communication engine, supports "standalone," "rollsite," "osx," "rabbitmq," "pulsar." +- storage: Storage engine, supports "standalone," "eggroll," "hdfs." -#### 默认注册算法配置 -- name: 算法名称 -- version: 算法版本，若不配置，则使用fateflow.env中的配置 -- device: 算法启动方式, local/docker/k8s等 +## Default Registered Algorithm Configuration +- name: Algorithm name. +- version: Algorithm version. If not configured, it uses the configuration in `fateflow.env`. +- device: Algorithm launch mode, local/docker/k8s, etc. -#### 通信引擎池 -##### pulsar +## Communication Engine Pool +### Pulsar ```yaml pulsar: host: 192.168.0.5 @@ -93,88 +93,84 @@ pulsar: cluster: standalone tenant: fl-tenant topic_ttl: 30 - # default conf/pulsar_route_table.yaml route_table: - # mode: replication / client, default: replication mode: replication max_message_size: 1048576 ``` -##### nginx: +### Nginx: ```yaml nginx: host: 127.0.0.1 http_port: 9300 grpc_port: 9310 - # http or grpc protocol: http ``` -##### rabbitmq +### RabbitMQ ```yaml nginx: host: 127.0.0.1 http_port: 9300 grpc_port: 9310 - # http or grpc protocol: http ``` -##### rollsite +### Rollsite ```yaml rollsite: host: 127.0.0.1 port: 9370 ``` -##### osx +### OSx ```yaml host: 127.0.0.1 port: 9370 ``` -#### 计算引擎池 -##### standalone +## Computing Engine Pool +### Standalone ```yaml cores: 32 ``` -- cores: 资源总数 +- cores: Total resources. -##### eggroll +### Eggroll ```yaml eggroll: cores: 32 nodes: 2 ``` -- cores: 集群资源总数 -- nodes: 集群node-manager数量 +- cores: Total cluster resources. +- nodes: Number of node managers in the cluster. -##### spark +### Spark ```yaml eggroll: home: cores: 32 ``` -- home: spark home目录，如果不填，将使用"pyspark"作为计算引擎。 -- cores: 资源总数 +- home: Spark home directory. If not filled, "pyspark" will be used as the computing engine. +- cores: Total resources. -#### 存储引擎池 +## Storage Engine Pool ```yaml hdfs: name_node: hdfs://fate-cluster ``` -## 钩子模块配置 +## Hook Module Configuration ```yaml hook_module: client_authentication: fate_flow.hook.flow.client_authentication site_authentication: fate_flow.hook.flow.site_authentication permission: fate_flow.hook.flow.permission ``` -- client_authentication: 客户端认证钩子 -- site_authentication: 站点认证钩子 -- permission: 权限认证钩子 +- client_authentication: Client authentication hook. +- site_authentication: Site authentication hook. +- permission: Permission authentication hook. -## 认证开关 +## Authentication Switch ```yaml authentication: client: false @@ -182,7 +178,7 @@ authentication: permission: false ``` -## 模型存储 +## Model Storage ```yaml model_store: engine: file @@ -203,14 +199,13 @@ model_store: SecretKey: Bucket: ``` -- engine: 模型存储引擎，支持"file"、"mysql"和"tencent_cos"。 -- decrypt_key: 加密模块,需要从[加密模块](#加密模块)中选择。 若不配置，视为不使用密码加密；若使用，则需要将下面的passwd相应设置为密文。 -- file: 模型存储目录，默认位于： fate_flow/model -- mysql: mysql服务配置；若使用密码加密功能，需要将此配置中的"passwd"设置为密文，并在[加密模块](#加密模块)中配置密钥路径 -- tencent_cos: 腾讯云密钥配置 +- engine: Model storage engine, supports "file," "mysql", and "tencent_cos". +- decrypt_key: Encryption module, needs to be selected from [Encryption Module](#encryption-module). If not configured, it is assumed to not use password encryption. If used, you need to set the "passwd" below accordingly to ciphertext and configure the key path in [Encryption Module](#encryption-module). +- file: Model storage directory, default location is `fate_flow/model`. +- mysql: MySQL service configuration; if using password encryption functionality, you need to set the "passwd" in this configuration to ciphertext and configure the key path in [Encryption Module](#encryption-module). +- tencent_cos: Tencent Cloud key configuration. - -#### zookeeper配置 +## ZooKeeper Configuration ```yaml zookeeper: hosts: diff --git a/doc/system_conf.zh.md b/doc/system_conf.zh.md index 5d83c58cd..c0b4ef846 100644 --- a/doc/system_conf.zh.md +++ b/doc/system_conf.zh.md @@ -1,4 +1,4 @@ -### 系统配置描述文档 +# 系统配置描述文档 FATE Flow使用yaml定义系统配置，配置路径位于: conf/service_conf.yaml, 具体配置内容及其含义如下： | 配置项 | 说明 | 值 | @@ -18,7 +18,7 @@ FATE Flow使用yaml定义系统配置，配置路径位于: conf/service_conf.ya | model_store | 模型存储配置 | 见[模型存储](#模型存储) | | zookeeper | zookeeper服务的配置 | 见[zookeeper配置](#zookeeper配置) | -#### 加密模块 +## 加密模块 ```yaml key_0: module: fate_flow.hub.encrypt.password_encrypt#pwdecrypt @@ -29,7 +29,7 @@ key_0: - module: 加密模块，拼接规则为：加密模块 + "#" + 加密函数。 - private_path：密钥路径。如填相对路径，其根目录位于fate_flow/conf/ -#### FateFlow配置 +## FateFlow配置 ```yaml host: 127.0.0.1 http_port: 9380 @@ -46,7 +46,7 @@ nginx: - proxy_name: 命令通道服务名，支持osx/rollsite/nginx。详细配置需要在[通信引擎池](#通信引擎池) 里面配置; - nginx: 代理服务配置，用于负载均衡。 -#### 数据库配置 +## 数据库配置 ```yaml engine: sqlite decrypt_key: @@ -66,7 +66,7 @@ sqlite: - mysql: mysql服务配置；若使用密码加密功能，需要将此配置中的"passwd"设置为密文，并在[加密模块](#加密模块)中配置密钥路径 - sqlite: sqlite文件路径，默认路径为fate_flow/fate_flow_sqlite.db -#### 引擎配置 +## 引擎配置 ```yaml default_engines: computing: standalone @@ -78,13 +78,13 @@ default_engines: - federation: 通信引擎，支持"standalone"、"rollsite"、"osx"、"rabbitmq"、"pulsar" - storage: 存储引擎，支持"standalone"、"eggroll"、"hdfs" -#### 默认注册算法配置 +## 默认注册算法配置 - name: 算法名称 - version: 算法版本，若不配置，则使用fateflow.env中的配置 - device: 算法启动方式, local/docker/k8s等 -#### 通信引擎池 -##### pulsar +## 通信引擎池 +### pulsar ```yaml pulsar: host: 192.168.0.5 @@ -99,7 +99,7 @@ pulsar: mode: replication max_message_size: 1048576 ``` -##### nginx: +### nginx: ```yaml nginx: host: 127.0.0.1 @@ -109,7 +109,7 @@ nginx: protocol: http ``` -##### rabbitmq +### rabbitmq ```yaml nginx: host: 127.0.0.1 @@ -119,27 +119,27 @@ nginx: protocol: http ``` -##### rollsite +### rollsite ```yaml rollsite: host: 127.0.0.1 port: 9370 ``` -##### osx +### osx ```yaml host: 127.0.0.1 port: 9370 ``` -#### 计算引擎池 -##### standalone +## 计算引擎池 +### standalone ```yaml cores: 32 ``` - cores: 资源总数 -##### eggroll +### eggroll ```yaml eggroll: cores: 32 @@ -148,7 +148,7 @@ eggroll: - cores: 集群资源总数 - nodes: 集群node-manager数量 -##### spark +### spark ```yaml eggroll: home: @@ -157,7 +157,7 @@ eggroll: - home: spark home目录，如果不填，将使用"pyspark"作为计算引擎。 - cores: 资源总数 -#### 存储引擎池 +## 存储引擎池 ```yaml hdfs: name_node: hdfs://fate-cluster @@ -210,7 +210,7 @@ model_store: - tencent_cos: 腾讯云密钥配置 -#### zookeeper配置 +## zookeeper配置 ```yaml zookeeper: hosts: