Merge pull request #6 from FederatedAI/dev-2.0.0-beta

Merge 2.0.0-beta

.gitignore

@@ -0,0 +1,29 @@
+# common file patterns
+.DS_STORE
+.idea
+*.iml
+*.pyc
+__pycache__
+*.jar
+*.class
+.project
+*.prefs
+_build
+venv
+*.log
+*.tar.gz
+*.db
+*.db-journal
+*.mdb
+.env
+.vscode/*
+Cargo.lock
+**/*.egg-info
+
+# excluded paths
+/data/
+/logs/
+/jobs/
+/metric/
+/python/dist/
+/python/build/

README.md

@@ -1 +1,5 @@
-# FATE-Test
+# FATE-Test
+
+A collection of useful tools to running FATE tasks, including tests and benchmark comparisons.
+For a tutorial on using FATE-Test command line tools and compiling test suites, please
+refer to this [document](./doc/fate_test.md).

README_zh.md

@@ -0,0 +1,4 @@
+# FATE-Test
+
+FATE-Test用于批量执行FATE任务，包括测试用例和性能及效果测试。
+相关命令行工具说明及使用方法请参考[文档](./doc/fate_test.md)。

RELEASE.md

@@ -0,0 +1,4 @@
+## Release 2.0.0-beta
+### Major Features and Improvements
+> Fate-Test: FATE Automated Testing Tool
+* Migrated automated testing for functionality, performance, and correctness

doc/fate_test.md

@@ -0,0 +1,91 @@
+# FATE Test Tutorial
+
+A collection of useful tools to running FATE tests and PipeLine tasks.
+
+## quick start
+
+1. install
+
+    ```bash
+    pip install -e python/fate_test
+    ```
+2. edit default fate\_test\_config.yaml
+
+   ```bash
+   # edit priority config file with system default editor
+   # filling some field according to comments
+   fate_test config edit
+   ```
+
+3. configure FATE-Flow Commandline server setting
+
+    ```bash
+    # configure FATE-Flow Commandline server setting
+    flow init --port 9380 --ip 127.0.0.1
+    ```
+
+4. run some fate\_test suite
+
+   ```bash
+   fate_test suite -i <path contains *testsuite.yaml>
+   ```
+
+5. run some fate\_test benchmark quality
+
+   ```bash
+   fate_test benchmark-quality -i <path contains *benchmark.yaml>
+   ```
+
+6. run some fate\_test benchmark performance
+
+   ```bash
+   fate_test benchmark-quality -i <path contains *performance.yaml>
+   ```
+
+7useful logs or exception will be saved to logs dir with namespace
+shown in last step
+
+## command types
+
+- [suite](../api/fate_test.md#testsuite): used for running [testsuites](../api/fate_test.md#testsuite-configuration),
+  collection of FATE jobs
+
+  ```bash
+  fate_test suite -i <path contains *testsuite.yaml>
+  ```
+
+- [data](../api/fate_test.md#data): used for upload, delete, and generate dataset
+
+    - [upload/delete data](../api/fate_test.md#data-command-options) command:
+
+      ```bash
+      fate_test data [upload|delete] -i <path1 contains *testsuite.yaml | *benchmark.yaml>
+      ```
+    - [upload example data of min_test/all_examples](../api/fate_test.md#data-command-options) command:
+
+      ```bash
+      fate_test data upload -t min_test
+      fate_test data upload -t all_examples
+      ```
+
+    - [generate data](../api/fate_test.md#generate-command-options) command:
+
+      ```bash
+      fate_test data generate -i <path1 contains *testsuite.yaml | *benchmark.yaml>
+      ```
+
+- [benchmark-quality](../api/fate_test.md#benchmark-quality): used for comparing modeling quality between FATE
+  and other machine learning systems, as specified
+  in [benchmark job configuration](../api/fate_test.md#benchmark-job-configuration)
+
+  ```bash
+  fate_test bq -i <path contains *benchmark.yaml>
+  ```
+
+- [benchmark-performance](../api/fate_test.md#benchmark-performance): used for checking FATE algorithm performance; user
+  should first generate and upload data before running performance testsuite
+
+  ```bash
+  fate_test data generate -i <path contains *performance.yaml> -ng 10000 -fg 10 -fh 10 -m 1.0 --upload-data
+  fate_test performance -i <path contains *performance.yaml> --skip-data
+  ```

python/fate_test/__init__.py

@@ -0,0 +1,15 @@
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#

python/fate_test/_ascii.py

@@ -0,0 +1,48 @@
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+
+HEAD = """\
+
+████████╗███████╗███████╗████████╗███████╗██╗   ██╗██╗████████╗███████╗
+╚══██╔══╝██╔════╝██╔════╝╚══██╔══╝██╔════╝██║   ██║██║╚══██╔══╝██╔════╝
+   ██║   █████╗  ███████╗   ██║   ███████╗██║   ██║██║   ██║   █████╗
+   ██║   ██╔══╝  ╚════██║   ██║   ╚════██║██║   ██║██║   ██║   ██╔══╝
+   ██║   ███████╗███████║   ██║   ███████║╚██████╔╝██║   ██║   ███████╗
+   ╚═╝   ╚══════╝╚══════╝   ╚═╝   ╚══════╝ ╚═════╝ ╚═╝   ╚═╝   ╚══════╝
+
+"""
+
+BENCHMARK = """\
+
+██████╗ ███████╗███╗   ██╗ ██████╗██╗  ██╗███╗   ███╗ █████╗ ██████╗ ██╗  ██╗
+██╔══██╗██╔════╝████╗  ██║██╔════╝██║  ██║████╗ ████║██╔══██╗██╔══██╗██║ ██╔╝
+██████╔╝█████╗  ██╔██╗ ██║██║     ███████║██╔████╔██║███████║██████╔╝█████╔╝
+██╔══██╗██╔══╝  ██║╚██╗██║██║     ██╔══██║██║╚██╔╝██║██╔══██║██╔══██╗██╔═██╗
+██████╔╝███████╗██║ ╚████║╚██████╗██║  ██║██║ ╚═╝ ██║██║  ██║██║  ██║██║  ██╗
+╚═════╝ ╚══════╝╚═╝  ╚═══╝ ╚═════╝╚═╝  ╚═╝╚═╝     ╚═╝╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═╝
+"""
+
+TAIL = """\
+
+    ██╗  ██╗ █████╗ ██╗   ██╗███████╗    ███████╗██╗   ██╗███╗   ██╗
+    ██║  ██║██╔══██╗██║   ██║██╔════╝    ██╔════╝██║   ██║████╗  ██║
+    ███████║███████║██║   ██║█████╗      █████╗  ██║   ██║██╔██╗ ██║
+    ██╔══██║██╔══██║╚██╗ ██╔╝██╔══╝      ██╔══╝  ██║   ██║██║╚██╗██║
+    ██║  ██║██║  ██║ ╚████╔╝ ███████╗    ██║     ╚██████╔╝██║ ╚████║
+    ╚═╝  ╚═╝╚═╝  ╚═╝  ╚═══╝  ╚══════╝    ╚═╝      ╚═════╝ ╚═╝  ╚═══╝
+
+"""

python/fate_test/_client.py

@@ -0,0 +1,44 @@
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from fate_test._flow_client import FLOWClient
+from fate_test._parser import Config
+
+
+class Clients(object):
+    def __init__(self, config: Config):
+        self._flow_clients = {}
+        # self._tunnel_id_to_flow_clients = {}
+        self._role_str_to_service_id = {}
+        for party, service_id in config.party_to_service_id.items():
+            for role_str in config.parties.party_to_role_string(party):
+                self._role_str_to_service_id[role_str] = service_id
+
+        for service_id, service in config.service_id_to_service.items():
+            if isinstance(service, Config.service):
+                self._flow_clients[service_id] = FLOWClient(
+                    service.address, config.data_base_dir, config.cache_directory)
+
+    def __getitem__(self, role_str: str) -> 'FLOWClient':
+        if role_str not in self._role_str_to_service_id:
+            raise RuntimeError(f"no flow client found binding to {role_str}")
+        return self._flow_clients[self._role_str_to_service_id[role_str]]
+
+    def contains(self, role_str):
+        return role_str in self._role_str_to_service_id
+
+    def all_roles(self):
+        return sorted(self._role_str_to_service_id.keys())