diff --git a/RELEASE.md b/RELEASE.md index eae44b7b..ffa5313d 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,13 @@ +# Release 2.0.0 + +#### Major Features and Improvements +**Major Features** + +* Refactoring DAG components, adding support for stage status, and displaying dynamic ports. +* Update the cache structure to optimize issues such as user timeout handling and duplicate storage of configuration information. +* Optimize some interactive functions. +* Update the style theme. + # Release 1.11.2 #### Major Features and Improvements diff --git a/package.xml b/package.xml new file mode 100644 index 00000000..d3c47f3f --- /dev/null +++ b/package.xml @@ -0,0 +1,72 @@ + + + release + + + zip + + + false + + + / + target + + *.jar + + + + + + /lib + target/lib + + *.jar + + + + + / + bin + + *.sh + + 755 + unix + + + + /dag + static + + *.yaml + + 755 + unix + + + + /conf + src/main/resources + + * + + + + + \ No newline at end of file diff --git a/resources-front-end/package.json b/resources-front-end/package.json index 0bb2726f..5ba2d8b6 100644 --- a/resources-front-end/package.json +++ b/resources-front-end/package.json @@ -1,98 +1,59 @@ { "name": "aisp-scvmadm-web", - "version": "3.9.0", + "version": "2.0.0", "license": "MIT", "description": "aisp-scvmadm-web", "author": "webank", - "scripts": { - "dev": "webpack-dev-server --inline --progress --config build/webpack.dev.conf.js", - "start": "npm run dev", - "build": "node build/build.js", - "build:report": "npm_config_report=true npm run build", - "lint": "eslint --ext .js,.vue src", - "test": "npm run lint", - "svgo": "svgo -f src/icons/svg --config=src/icons/svgo.yml" + "devDependencies": { + "@babel/core": "^7.23.2", + "@babel/preset-env": "^7.23.2", + "@types/lodash": "^4.14.200", + "@types/node": "^20.8.7", + "@typescript-eslint/eslint-plugin": "^6.8.0", + "@typescript-eslint/parser": "^6.8.0", + "babel-loader": "^9.1.3", + "css-loader": "^6.8.1", + "eslint": "^8.52.0", + "eslint-config-standard-with-typescript": "^39.1.1", + "eslint-plugin-import": "^2.28.1", + "eslint-plugin-n": "^16.2.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-vue": "^9.17.0", + "file-loader": "^6.2.0", + "fork-ts-checker-webpack-plugin": "^9.0.0", + "html-webpack-plugin": "^5.5.3", + "lerna": "^5.6.2", + "mini-css-extract-plugin": "^2.7.6", + "postcss": "^8.4.31", + "postcss-loader": "^7.3.3", + "postcss-preset-env": "^9.2.0", + "postcss-short": "^5.0.0", + "prettier": "^3.0.3", + "sass": "^1.69.4", + "sass-loader": "^13.3.2", + "style-loader": "^3.3.3", + "svg-inline-loader": "^0.8.2", + "ts-loader": "^9.5.0", + "typescript": "^5.2.2", + "url-loader": "^4.1.1", + "vue-loader": "^17.2.2", + "vue-template-compiler": "^2.7.14", + "webpack": "^5.89.0", + "webpack-cli": "^5.1.4", + "webpack-dev-server": "^4.15.1", + "webpack-merge": "^5.10.0" }, + "workspaces": [ + "packages/*" + ], "dependencies": { - "axios": "^0.27.2", - "babel-polyfill": "^6.26.0", - "clipboard": "^2.0.8", - "echarts": "^4.2.1", - "element-ui": "2.8.2", - "file-saver": "^2.0.1", - "hmac_sha1": "^0.1.1", - "js-cookie": "2.2.0", - "jszip": "^3.8.0", + "@element-plus/icons-vue": "^2.1.0", + "element-plus": "^2.4.1", "lodash": "^4.17.21", - "mockjs": "1.0.1-beta3", - "normalize.css": "7.0.0", - "nprogress": "0.2.0", - "sortablejs": "^1.8.4", - "vue": "2.5.17", - "vue-observe-visibility": "^0.4.6", - "vue-router": "3.0.1", - "vuex": "3.0.1", - "xlsx": "^0.18.5" - }, - "devDependencies": { - "autoprefixer": "8.5.0", - "babel-core": "6.26.0", - "babel-eslint": "8.2.6", - "babel-helper-vue-jsx-merge-props": "2.0.3", - "babel-loader": "7.1.5", - "babel-plugin-syntax-jsx": "6.18.0", - "babel-plugin-transform-runtime": "6.23.0", - "babel-plugin-transform-vue-jsx": "3.7.0", - "babel-preset-env": "1.7.0", - "babel-preset-stage-2": "6.24.1", - "chalk": "2.4.1", - "compression-webpack-plugin": "2.0.0", - "copy-webpack-plugin": "4.5.2", - "css-loader": "1.0.0", - "eslint": "4.19.1", - "eslint-friendly-formatter": "4.0.1", - "eslint-loader": "2.0.0", - "eslint-plugin-vue": "4.7.1", - "eventsource-polyfill": "0.9.6", - "file-loader": "1.1.11", - "friendly-errors-webpack-plugin": "1.7.0", - "html-webpack-plugin": "4.5.2", - "mini-css-extract-plugin": "0.4.1", - "node-notifier": "^8.0.2", - "node-sass": "^8.0.0", - "optimize-css-assets-webpack-plugin": "5.0.0", - "ora": "3.0.0", - "path-to-regexp": "2.4.0", - "portfinder": "1.0.16", - "postcss-import": "12.0.0", - "postcss-loader": "2.1.6", - "postcss-url": "7.3.2", - "rimraf": "2.6.2", - "sass-loader": "7.0.3", - "script-ext-html-webpack-plugin": "2.1.5", - "script-loader": "^0.7.2", - "semver": "5.5.0", - "shelljs": "0.8.5", - "svg-sprite-loader": "3.8.0", - "svgo": "^1.2.2", - "uglifyjs-webpack-plugin": "1.2.7", - "url-loader": "1.0.1", - "vue-loader": "15.3.0", - "vue-style-loader": "4.1.2", - "vue-template-compiler": "2.5.17", - "webpack": "4.16.5", - "webpack-bundle-analyzer": "3.3.2", - "webpack-cli": "3.1.0", - "webpack-dev-server": "3.1.14", - "webpack-merge": "4.1.4" - }, - "engines": { - "node": ">= 6.0.0", - "npm": ">= 3.0.0" + "vue": "^3.3.4" }, - "browserslist": [ - "> 1%", - "last 2 versions", - "not ie <= 8" - ] + "scripts": { + "dev": "lerna run dev --scope=fate-board", + "build": "lerna run build" + } } diff --git a/src/main/java/com/webank/ai/fate/board/services/FlowLogFeign.java b/src/main/java/com/webank/ai/fate/board/services/FlowLogFeign.java deleted file mode 100644 index 71bbf2cf..00000000 --- a/src/main/java/com/webank/ai/fate/board/services/FlowLogFeign.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.webank.ai.fate.board.services; - -import com.webank.ai.fate.board.conf.RouteTargeter; -import com.webank.ai.fate.board.intercept.FeignRequestInterceptor; -import com.webank.ai.fate.board.pojo.flow.*; -import org.springframework.cloud.openfeign.FeignClient; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; - -import java.util.List; - -@FeignClient(url = RouteTargeter.URL_PLACE_HOLDER + "/v1/log", name = "flowLogFeign", configuration = FeignRequestInterceptor.class) -public interface FlowLogFeign { - - @RequestMapping(value = "/cat", method = RequestMethod.POST) - FlowResponse> logCat(FlowLogCatReq request); - - @RequestMapping(value = "/size", method = RequestMethod.POST) - FlowResponse logSize(FlowLogSizeReq request); - -} diff --git a/src/main/java/com/webank/ai/fate/board/bootstrap/Bootstrap.java b/src/main/java/org/fedai/fate/board/bootstrap/Bootstrap.java similarity index 92% rename from src/main/java/com/webank/ai/fate/board/bootstrap/Bootstrap.java rename to src/main/java/org/fedai/fate/board/bootstrap/Bootstrap.java index 1c20acc0..e9b7f291 100644 --- a/src/main/java/com/webank/ai/fate/board/bootstrap/Bootstrap.java +++ b/src/main/java/org/fedai/fate/board/bootstrap/Bootstrap.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.bootstrap; +package org.fedai.fate.board.bootstrap; import org.springframework.boot.SpringApplication; @@ -31,11 +31,11 @@ @SpringBootApplication(exclude = {DataSourceAutoConfiguration.class, SessionAutoConfiguration.class}) @ServletComponentScan(basePackages = {"com.webank.ai.fate.board.conf"}) -@ComponentScan(basePackages = {"com.webank.ai.fate.*"}) +@ComponentScan(basePackages = {"org.fedai.fate.board.*"}) @PropertySource(value = "classpath:application.properties", ignoreResourceNotFound = true) @Configuration @EnableScheduling -@EnableFeignClients(basePackages = "com.webank.ai.fate.*") +@EnableFeignClients(basePackages = "org.fedai.fate.board.*") public class Bootstrap { public static void main(String[] args) { diff --git a/src/main/java/org/fedai/fate/board/conf/CaffeineCacheConfig.java b/src/main/java/org/fedai/fate/board/conf/CaffeineCacheConfig.java new file mode 100644 index 00000000..fd3bb438 --- /dev/null +++ b/src/main/java/org/fedai/fate/board/conf/CaffeineCacheConfig.java @@ -0,0 +1,19 @@ +package org.fedai.fate.board.conf; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.Map; + + +@Configuration +public class CaffeineCacheConfig { + + @Bean + public Cache> caffeineCache() { + return Caffeine.newBuilder() + .build(); + } +} diff --git a/src/main/java/com/webank/ai/fate/board/conf/Configurator.java b/src/main/java/org/fedai/fate/board/conf/Configurator.java similarity index 97% rename from src/main/java/com/webank/ai/fate/board/conf/Configurator.java rename to src/main/java/org/fedai/fate/board/conf/Configurator.java index 15a4dd81..f822c3ff 100644 --- a/src/main/java/com/webank/ai/fate/board/conf/Configurator.java +++ b/src/main/java/org/fedai/fate/board/conf/Configurator.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.conf; +package org.fedai.fate.board.conf; import javax.websocket.HandshakeResponse; import javax.websocket.server.HandshakeRequest; diff --git a/src/main/java/com/webank/ai/fate/board/conf/FeignConfig.java b/src/main/java/org/fedai/fate/board/conf/FeignConfig.java similarity index 99% rename from src/main/java/com/webank/ai/fate/board/conf/FeignConfig.java rename to src/main/java/org/fedai/fate/board/conf/FeignConfig.java index c64987f1..fcaa5a09 100644 --- a/src/main/java/com/webank/ai/fate/board/conf/FeignConfig.java +++ b/src/main/java/org/fedai/fate/board/conf/FeignConfig.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.conf; +package org.fedai.fate.board.conf; import feign.Logger; import feign.Request; diff --git a/src/main/java/com/webank/ai/fate/board/conf/RouteTargeter.java b/src/main/java/org/fedai/fate/board/conf/RouteTargeter.java similarity index 86% rename from src/main/java/com/webank/ai/fate/board/conf/RouteTargeter.java rename to src/main/java/org/fedai/fate/board/conf/RouteTargeter.java index 2eb986d4..d6d1d9c5 100644 --- a/src/main/java/com/webank/ai/fate/board/conf/RouteTargeter.java +++ b/src/main/java/org/fedai/fate/board/conf/RouteTargeter.java @@ -1,18 +1,24 @@ -package com.webank.ai.fate.board.conf; +package org.fedai.fate.board.conf; -import com.webank.ai.fate.board.utils.TelnetUtil; +import org.fedai.fate.board.services.FlowHighAvailableService; +import org.fedai.fate.board.utils.TelnetUtil; import feign.Feign; import feign.Request; import feign.RequestTemplate; import feign.Target; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cloud.openfeign.FeignClientFactoryBean; import org.springframework.cloud.openfeign.FeignContext; import org.springframework.cloud.openfeign.Targeter; import org.springframework.core.env.Environment; public class RouteTargeter implements Targeter { + + @Autowired + private FlowHighAvailableService zk; + private static Environment environment; public RouteTargeter(Environment environment) { @@ -27,12 +33,12 @@ public T target(FeignClientFactoryBean factory, Feign.Builder feign, FeignCo return feign.target(new RouteTarget<>(target)); } - public static class RouteTarget implements Target { + public class RouteTarget implements Target { Logger log = LoggerFactory.getLogger(getClass()); private Target realTarget; - private String availableFlow = null; private Long lastUpdateTime = null; + private FlowHighAvailableService flowHighAvailableService = RouteTargeter.this.zk; private static final long UPDATE_INTERVAL = 1000 * 30; public RouteTarget(Target realTarget) { @@ -75,7 +81,8 @@ private void updateAvailableFlow() { } else { flowUrl = flowUrl.replaceAll("http[s]?://", ""); } - String flowUrlList = environment.getProperty("fateflow.url-list"); +// String flowUrlList = environment.getProperty("fateflow.url-list"); + String flowUrlList = flowHighAvailableService.getFlowUrlsFromCache(); if (flowUrlList == null || flowUrlList.trim().isEmpty()) { availableFlow = flowUrl; } else if (flowUrlList.trim().split(";").length < 2) { diff --git a/src/main/java/com/webank/ai/fate/board/conf/SecurityFilter.java b/src/main/java/org/fedai/fate/board/conf/SecurityFilter.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/conf/SecurityFilter.java rename to src/main/java/org/fedai/fate/board/conf/SecurityFilter.java index f5d7c85a..6a60881b 100644 --- a/src/main/java/com/webank/ai/fate/board/conf/SecurityFilter.java +++ b/src/main/java/org/fedai/fate/board/conf/SecurityFilter.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.conf; +package org.fedai.fate.board.conf; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; diff --git a/src/main/java/com/webank/ai/fate/board/conf/TomcatConfig.java b/src/main/java/org/fedai/fate/board/conf/TomcatConfig.java similarity index 96% rename from src/main/java/com/webank/ai/fate/board/conf/TomcatConfig.java rename to src/main/java/org/fedai/fate/board/conf/TomcatConfig.java index 2cb2287f..e0cc629b 100644 --- a/src/main/java/com/webank/ai/fate/board/conf/TomcatConfig.java +++ b/src/main/java/org/fedai/fate/board/conf/TomcatConfig.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.conf; +package org.fedai.fate.board.conf; import org.apache.tomcat.util.descriptor.web.SecurityCollection; import org.apache.tomcat.util.descriptor.web.SecurityConstraint; diff --git a/src/main/java/com/webank/ai/fate/board/conf/WebConfiguration.java b/src/main/java/org/fedai/fate/board/conf/WebConfiguration.java similarity index 95% rename from src/main/java/com/webank/ai/fate/board/conf/WebConfiguration.java rename to src/main/java/org/fedai/fate/board/conf/WebConfiguration.java index 712c52f9..35dc5cce 100644 --- a/src/main/java/com/webank/ai/fate/board/conf/WebConfiguration.java +++ b/src/main/java/org/fedai/fate/board/conf/WebConfiguration.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.conf; +package org.fedai.fate.board.conf; -import com.webank.ai.fate.board.intercept.UserInterceptor; +import org.fedai.fate.board.intercept.UserInterceptor; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; diff --git a/src/main/java/com/webank/ai/fate/board/conf/WebsocketFilter.java b/src/main/java/org/fedai/fate/board/conf/WebsocketFilter.java similarity index 84% rename from src/main/java/com/webank/ai/fate/board/conf/WebsocketFilter.java rename to src/main/java/org/fedai/fate/board/conf/WebsocketFilter.java index c5ffa09f..582eaeea 100644 --- a/src/main/java/com/webank/ai/fate/board/conf/WebsocketFilter.java +++ b/src/main/java/org/fedai/fate/board/conf/WebsocketFilter.java @@ -1,10 +1,9 @@ -package com.webank.ai.fate.board.conf; +package org.fedai.fate.board.conf; import com.alibaba.fastjson.JSON; -import com.webank.ai.fate.board.global.ErrorCode; -import com.webank.ai.fate.board.global.ResponseResult; +import org.fedai.fate.board.global.ErrorCode; +import org.fedai.fate.board.global.ResponseResult; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.core.annotation.Order; import javax.servlet.*; @@ -12,7 +11,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; -import javax.websocket.server.HandshakeRequest; import java.io.IOException; @Slf4j diff --git a/src/main/java/com/webank/ai/fate/board/controller/JobDetailController.java b/src/main/java/org/fedai/fate/board/controller/JobDetailController.java similarity index 100% rename from src/main/java/com/webank/ai/fate/board/controller/JobDetailController.java rename to src/main/java/org/fedai/fate/board/controller/JobDetailController.java diff --git a/src/main/java/com/webank/ai/fate/board/controller/JobManagerController.java b/src/main/java/org/fedai/fate/board/controller/JobManagerController.java similarity index 100% rename from src/main/java/com/webank/ai/fate/board/controller/JobManagerController.java rename to src/main/java/org/fedai/fate/board/controller/JobManagerController.java diff --git a/src/main/java/com/webank/ai/fate/board/controller/SshController.java b/src/main/java/org/fedai/fate/board/controller/SshController.java similarity index 91% rename from src/main/java/com/webank/ai/fate/board/controller/SshController.java rename to src/main/java/org/fedai/fate/board/controller/SshController.java index 4e9aaa23..21383ae8 100644 --- a/src/main/java/com/webank/ai/fate/board/controller/SshController.java +++ b/src/main/java/org/fedai/fate/board/controller/SshController.java @@ -13,25 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.controller; +package org.fedai.fate.board.controller; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.jcraft.jsch.Session; -import com.webank.ai.fate.board.global.ErrorCode; -import com.webank.ai.fate.board.global.ResponseResult; -import com.webank.ai.fate.board.pojo.SshDTO; -import com.webank.ai.fate.board.pojo.SshInfo; -import com.webank.ai.fate.board.ssh.SshService; -import com.webank.ai.fate.board.global.Dict; +import org.fedai.fate.board.global.ErrorCode; +import org.fedai.fate.board.global.ResponseResult; +import org.fedai.fate.board.pojo.SshDTO; +import org.fedai.fate.board.pojo.SshInfo; +import org.fedai.fate.board.ssh.SshService; +import org.fedai.fate.board.global.Dict; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.ClassPathResource; -import org.springframework.stereotype.Component; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.*; diff --git a/src/main/java/com/webank/ai/fate/board/controller/UserController.java b/src/main/java/org/fedai/fate/board/controller/UserController.java similarity index 91% rename from src/main/java/com/webank/ai/fate/board/controller/UserController.java rename to src/main/java/org/fedai/fate/board/controller/UserController.java index b5d2c3d9..4dcaf5ca 100644 --- a/src/main/java/com/webank/ai/fate/board/controller/UserController.java +++ b/src/main/java/org/fedai/fate/board/controller/UserController.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.controller; +package org.fedai.fate.board.controller; -import com.webank.ai.fate.board.global.ErrorCode; -import com.webank.ai.fate.board.global.ResponseResult; -import com.webank.ai.fate.board.pojo.UserDTO; -import com.webank.ai.fate.board.services.UserService; +import org.fedai.fate.board.global.ErrorCode; +import org.fedai.fate.board.global.ResponseResult; +import org.fedai.fate.board.pojo.UserDTO; +import org.fedai.fate.board.services.UserService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.validation.BindingResult; diff --git a/src/main/java/com/webank/ai/fate/board/disruptor/LogFileEventHandler.java b/src/main/java/org/fedai/fate/board/disruptor/LogFileEventHandler.java similarity index 93% rename from src/main/java/com/webank/ai/fate/board/disruptor/LogFileEventHandler.java rename to src/main/java/org/fedai/fate/board/disruptor/LogFileEventHandler.java index 3d0e577b..2aff5be6 100644 --- a/src/main/java/com/webank/ai/fate/board/disruptor/LogFileEventHandler.java +++ b/src/main/java/org/fedai/fate/board/disruptor/LogFileEventHandler.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.disruptor; +package org.fedai.fate.board.disruptor; import com.lmax.disruptor.EventHandler; -import com.webank.ai.fate.board.ssh.SftpUtils; +import org.fedai.fate.board.ssh.SftpUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; diff --git a/src/main/java/com/webank/ai/fate/board/disruptor/LogFileTransferEvent.java b/src/main/java/org/fedai/fate/board/disruptor/LogFileTransferEvent.java similarity index 95% rename from src/main/java/com/webank/ai/fate/board/disruptor/LogFileTransferEvent.java rename to src/main/java/org/fedai/fate/board/disruptor/LogFileTransferEvent.java index a84422ad..06d93e3a 100644 --- a/src/main/java/com/webank/ai/fate/board/disruptor/LogFileTransferEvent.java +++ b/src/main/java/org/fedai/fate/board/disruptor/LogFileTransferEvent.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.disruptor; +package org.fedai.fate.board.disruptor; -import com.webank.ai.fate.board.pojo.SshInfo; +import org.fedai.fate.board.pojo.SshInfo; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; diff --git a/src/main/java/com/webank/ai/fate/board/disruptor/LogFileTransferEventFactory.java b/src/main/java/org/fedai/fate/board/disruptor/LogFileTransferEventFactory.java similarity index 95% rename from src/main/java/com/webank/ai/fate/board/disruptor/LogFileTransferEventFactory.java rename to src/main/java/org/fedai/fate/board/disruptor/LogFileTransferEventFactory.java index 76890144..b9e23e29 100644 --- a/src/main/java/com/webank/ai/fate/board/disruptor/LogFileTransferEventFactory.java +++ b/src/main/java/org/fedai/fate/board/disruptor/LogFileTransferEventFactory.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.disruptor; +package org.fedai.fate.board.disruptor; import com.lmax.disruptor.EventFactory; diff --git a/src/main/java/com/webank/ai/fate/board/disruptor/LogFileTransferEventProducer.java b/src/main/java/org/fedai/fate/board/disruptor/LogFileTransferEventProducer.java similarity index 93% rename from src/main/java/com/webank/ai/fate/board/disruptor/LogFileTransferEventProducer.java rename to src/main/java/org/fedai/fate/board/disruptor/LogFileTransferEventProducer.java index 19e550d0..f17d630b 100644 --- a/src/main/java/com/webank/ai/fate/board/disruptor/LogFileTransferEventProducer.java +++ b/src/main/java/org/fedai/fate/board/disruptor/LogFileTransferEventProducer.java @@ -13,15 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.disruptor; +package org.fedai.fate.board.disruptor; import com.lmax.disruptor.RingBuffer; import com.lmax.disruptor.dsl.Disruptor; import com.lmax.disruptor.util.DaemonThreadFactory; -import com.webank.ai.fate.board.pojo.SshInfo; +import org.fedai.fate.board.pojo.SshInfo; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; public class LogFileTransferEventProducer implements InitializingBean { diff --git a/src/main/java/com/webank/ai/fate/board/exceptions/LogicException.java b/src/main/java/org/fedai/fate/board/exceptions/LogicException.java similarity index 93% rename from src/main/java/com/webank/ai/fate/board/exceptions/LogicException.java rename to src/main/java/org/fedai/fate/board/exceptions/LogicException.java index 501f8f5c..dc915865 100644 --- a/src/main/java/com/webank/ai/fate/board/exceptions/LogicException.java +++ b/src/main/java/org/fedai/fate/board/exceptions/LogicException.java @@ -1,6 +1,6 @@ -package com.webank.ai.fate.board.exceptions; +package org.fedai.fate.board.exceptions; -import com.webank.ai.fate.board.global.ErrorCode; +import org.fedai.fate.board.global.ErrorCode; public class LogicException extends RuntimeException { diff --git a/src/main/java/com/webank/ai/fate/board/global/Dict.java b/src/main/java/org/fedai/fate/board/global/Dict.java similarity index 100% rename from src/main/java/com/webank/ai/fate/board/global/Dict.java rename to src/main/java/org/fedai/fate/board/global/Dict.java diff --git a/src/main/java/com/webank/ai/fate/board/global/ErrorCode.java b/src/main/java/org/fedai/fate/board/global/ErrorCode.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/global/ErrorCode.java rename to src/main/java/org/fedai/fate/board/global/ErrorCode.java index 14041099..079bf5fa 100644 --- a/src/main/java/com/webank/ai/fate/board/global/ErrorCode.java +++ b/src/main/java/org/fedai/fate/board/global/ErrorCode.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.global; +package org.fedai.fate.board.global; public enum ErrorCode { diff --git a/src/main/java/com/webank/ai/fate/board/global/GlobalExceptionHandler.java b/src/main/java/org/fedai/fate/board/global/GlobalExceptionHandler.java similarity index 96% rename from src/main/java/com/webank/ai/fate/board/global/GlobalExceptionHandler.java rename to src/main/java/org/fedai/fate/board/global/GlobalExceptionHandler.java index 31c64add..e074181d 100644 --- a/src/main/java/com/webank/ai/fate/board/global/GlobalExceptionHandler.java +++ b/src/main/java/org/fedai/fate/board/global/GlobalExceptionHandler.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.global; +package org.fedai.fate.board.global; -import com.webank.ai.fate.board.exceptions.LogicException; +import org.fedai.fate.board.exceptions.LogicException; import feign.FeignException; import org.apache.http.client.ClientProtocolException; import org.slf4j.Logger; diff --git a/src/main/java/com/webank/ai/fate/board/global/GlobalResponseController.java b/src/main/java/org/fedai/fate/board/global/GlobalResponseController.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/global/GlobalResponseController.java rename to src/main/java/org/fedai/fate/board/global/GlobalResponseController.java index b5ef1c29..dcd9eae1 100644 --- a/src/main/java/com/webank/ai/fate/board/global/GlobalResponseController.java +++ b/src/main/java/org/fedai/fate/board/global/GlobalResponseController.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.global; +package org.fedai.fate.board.global; import org.springframework.core.MethodParameter; diff --git a/src/main/java/com/webank/ai/fate/board/global/ResponseResult.java b/src/main/java/org/fedai/fate/board/global/ResponseResult.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/global/ResponseResult.java rename to src/main/java/org/fedai/fate/board/global/ResponseResult.java index d232c009..151fc2a4 100644 --- a/src/main/java/com/webank/ai/fate/board/global/ResponseResult.java +++ b/src/main/java/org/fedai/fate/board/global/ResponseResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.global; +package org.fedai.fate.board.global; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/src/main/java/com/webank/ai/fate/board/intercept/FeignRequestInterceptor.java b/src/main/java/org/fedai/fate/board/intercept/FeignRequestInterceptor.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/intercept/FeignRequestInterceptor.java rename to src/main/java/org/fedai/fate/board/intercept/FeignRequestInterceptor.java index 8f03a5df..86030143 100644 --- a/src/main/java/com/webank/ai/fate/board/intercept/FeignRequestInterceptor.java +++ b/src/main/java/org/fedai/fate/board/intercept/FeignRequestInterceptor.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.intercept; +package org.fedai.fate.board.intercept; import feign.RequestInterceptor; import feign.RequestTemplate; diff --git a/src/main/java/com/webank/ai/fate/board/intercept/UserInterceptor.java b/src/main/java/org/fedai/fate/board/intercept/UserInterceptor.java similarity index 92% rename from src/main/java/com/webank/ai/fate/board/intercept/UserInterceptor.java rename to src/main/java/org/fedai/fate/board/intercept/UserInterceptor.java index 6ecbfc63..6969f342 100644 --- a/src/main/java/com/webank/ai/fate/board/intercept/UserInterceptor.java +++ b/src/main/java/org/fedai/fate/board/intercept/UserInterceptor.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.intercept; +package org.fedai.fate.board.intercept; import com.alibaba.fastjson.JSON; -import com.webank.ai.fate.board.global.ErrorCode; -import com.webank.ai.fate.board.global.ResponseResult; +import org.fedai.fate.board.global.ErrorCode; +import org.fedai.fate.board.global.ResponseResult; import org.springframework.web.servlet.HandlerInterceptor; import org.springframework.web.servlet.ModelAndView; diff --git a/src/main/java/com/webank/ai/fate/board/log/LogFileService.java b/src/main/java/org/fedai/fate/board/log/LogFileService.java similarity index 96% rename from src/main/java/com/webank/ai/fate/board/log/LogFileService.java rename to src/main/java/org/fedai/fate/board/log/LogFileService.java index b6b939d3..cf4fafee 100644 --- a/src/main/java/com/webank/ai/fate/board/log/LogFileService.java +++ b/src/main/java/org/fedai/fate/board/log/LogFileService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.log; +package org.fedai.fate.board.log; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; @@ -22,12 +22,12 @@ import com.google.common.collect.Maps; import com.jcraft.jsch.Channel; import com.jcraft.jsch.Session; -import com.webank.ai.fate.board.global.Dict; -import com.webank.ai.fate.board.pojo.JobDO; -import com.webank.ai.fate.board.pojo.SshInfo; -import com.webank.ai.fate.board.services.FlowFeign; -import com.webank.ai.fate.board.services.JobManagerService; -import com.webank.ai.fate.board.ssh.SshService; +import org.fedai.fate.board.global.Dict; +import org.fedai.fate.board.pojo.JobDO; +import org.fedai.fate.board.pojo.SshInfo; +import org.fedai.fate.board.services.FlowFeign; +import org.fedai.fate.board.services.JobManagerService; +import org.fedai.fate.board.ssh.SshService; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -177,7 +177,7 @@ public String getLogPath(String jobId) throws Exception { e.printStackTrace(); } if (jsonObject != null && !jsonObject.isEmpty()) { - Integer retcode = jsonObject.getInteger(Dict.RETCODE); + Integer retcode = jsonObject.getInteger(Dict.CODE); String retmsg = jsonObject.getString(Dict.RETMSG); if (retcode == 0) { JSONObject data = jsonObject.getJSONObject(Dict.DATA); diff --git a/src/main/java/com/webank/ai/fate/board/log/LogScanner.java b/src/main/java/org/fedai/fate/board/log/LogScanner.java similarity index 95% rename from src/main/java/com/webank/ai/fate/board/log/LogScanner.java rename to src/main/java/org/fedai/fate/board/log/LogScanner.java index 1657362e..92417861 100644 --- a/src/main/java/com/webank/ai/fate/board/log/LogScanner.java +++ b/src/main/java/org/fedai/fate/board/log/LogScanner.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.log; +package org.fedai.fate.board.log; public interface LogScanner { diff --git a/src/main/java/com/webank/ai/fate/board/log/RandomFileScanner.java b/src/main/java/org/fedai/fate/board/log/RandomFileScanner.java similarity index 95% rename from src/main/java/com/webank/ai/fate/board/log/RandomFileScanner.java rename to src/main/java/org/fedai/fate/board/log/RandomFileScanner.java index e6a736b6..c1314701 100644 --- a/src/main/java/com/webank/ai/fate/board/log/RandomFileScanner.java +++ b/src/main/java/org/fedai/fate/board/log/RandomFileScanner.java @@ -13,14 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.log; +package org.fedai.fate.board.log; import com.alibaba.fastjson.JSON; import com.google.common.collect.Lists; -import com.webank.ai.fate.board.utils.LogHandle; +import org.fedai.fate.board.utils.LogHandle; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import javax.websocket.Session; import java.io.File; diff --git a/src/main/java/com/webank/ai/fate/board/log/TailFile.java b/src/main/java/org/fedai/fate/board/log/TailFile.java similarity index 99% rename from src/main/java/com/webank/ai/fate/board/log/TailFile.java rename to src/main/java/org/fedai/fate/board/log/TailFile.java index 6e8e7e6b..229a128f 100644 --- a/src/main/java/com/webank/ai/fate/board/log/TailFile.java +++ b/src/main/java/org/fedai/fate/board/log/TailFile.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.log; +package org.fedai.fate.board.log; import com.google.common.collect.Lists; import org.slf4j.Logger; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/BatchMetricDTO.java b/src/main/java/org/fedai/fate/board/pojo/BatchMetricDTO.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/pojo/BatchMetricDTO.java rename to src/main/java/org/fedai/fate/board/pojo/BatchMetricDTO.java index 2227057d..a0654186 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/BatchMetricDTO.java +++ b/src/main/java/org/fedai/fate/board/pojo/BatchMetricDTO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import javax.validation.constraints.NotNull; import java.io.Serializable; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/ComponentQueryDTO.java b/src/main/java/org/fedai/fate/board/pojo/ComponentQueryDTO.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/pojo/ComponentQueryDTO.java rename to src/main/java/org/fedai/fate/board/pojo/ComponentQueryDTO.java index 685a6ee2..099fc459 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/ComponentQueryDTO.java +++ b/src/main/java/org/fedai/fate/board/pojo/ComponentQueryDTO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import javax.validation.constraints.NotNull; import java.io.Serializable; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/DownloadQO.java b/src/main/java/org/fedai/fate/board/pojo/DownloadQO.java similarity index 96% rename from src/main/java/com/webank/ai/fate/board/pojo/DownloadQO.java rename to src/main/java/org/fedai/fate/board/pojo/DownloadQO.java index effa8c91..5448e073 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/DownloadQO.java +++ b/src/main/java/org/fedai/fate/board/pojo/DownloadQO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import lombok.*; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/FlowJobDO.java b/src/main/java/org/fedai/fate/board/pojo/FlowJobDO.java similarity index 97% rename from src/main/java/com/webank/ai/fate/board/pojo/FlowJobDO.java rename to src/main/java/org/fedai/fate/board/pojo/FlowJobDO.java index f8f17dcb..cab4f041 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/FlowJobDO.java +++ b/src/main/java/org/fedai/fate/board/pojo/FlowJobDO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import lombok.AllArgsConstructor; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/FlowJobQO.java b/src/main/java/org/fedai/fate/board/pojo/FlowJobQO.java similarity index 93% rename from src/main/java/com/webank/ai/fate/board/pojo/FlowJobQO.java rename to src/main/java/org/fedai/fate/board/pojo/FlowJobQO.java index 8c4d1468..6187f711 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/FlowJobQO.java +++ b/src/main/java/org/fedai/fate/board/pojo/FlowJobQO.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/FlowTaskDO.java b/src/main/java/org/fedai/fate/board/pojo/FlowTaskDO.java similarity index 97% rename from src/main/java/com/webank/ai/fate/board/pojo/FlowTaskDO.java rename to src/main/java/org/fedai/fate/board/pojo/FlowTaskDO.java index 90d2f8e1..e9d7619d 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/FlowTaskDO.java +++ b/src/main/java/org/fedai/fate/board/pojo/FlowTaskDO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/FuzzyLogQO.java b/src/main/java/org/fedai/fate/board/pojo/FuzzyLogQO.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/pojo/FuzzyLogQO.java rename to src/main/java/org/fedai/fate/board/pojo/FuzzyLogQO.java index 87dc19ce..778ad317 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/FuzzyLogQO.java +++ b/src/main/java/org/fedai/fate/board/pojo/FuzzyLogQO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import java.io.Serializable; import java.util.Objects; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/Job.java b/src/main/java/org/fedai/fate/board/pojo/Job.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/pojo/Job.java rename to src/main/java/org/fedai/fate/board/pojo/Job.java index 6fc7621b..aeca1e51 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/Job.java +++ b/src/main/java/org/fedai/fate/board/pojo/Job.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import com.alibaba.fastjson.JSON; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/JobDO.java b/src/main/java/org/fedai/fate/board/pojo/JobDO.java similarity index 99% rename from src/main/java/com/webank/ai/fate/board/pojo/JobDO.java rename to src/main/java/org/fedai/fate/board/pojo/JobDO.java index 78b9ad5c..821819d4 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/JobDO.java +++ b/src/main/java/org/fedai/fate/board/pojo/JobDO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import java.util.List; import java.util.Set; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/JobExample.java b/src/main/java/org/fedai/fate/board/pojo/JobExample.java similarity index 99% rename from src/main/java/com/webank/ai/fate/board/pojo/JobExample.java rename to src/main/java/org/fedai/fate/board/pojo/JobExample.java index 34aa2c06..4ada9611 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/JobExample.java +++ b/src/main/java/org/fedai/fate/board/pojo/JobExample.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/JobKey.java b/src/main/java/org/fedai/fate/board/pojo/JobKey.java similarity index 96% rename from src/main/java/com/webank/ai/fate/board/pojo/JobKey.java rename to src/main/java/org/fedai/fate/board/pojo/JobKey.java index 70e41f9f..d27787e4 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/JobKey.java +++ b/src/main/java/org/fedai/fate/board/pojo/JobKey.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; public class JobKey { private String fJobId; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/JobQueryDTO.java b/src/main/java/org/fedai/fate/board/pojo/JobQueryDTO.java similarity index 97% rename from src/main/java/com/webank/ai/fate/board/pojo/JobQueryDTO.java rename to src/main/java/org/fedai/fate/board/pojo/JobQueryDTO.java index a2c1eef5..357d2b89 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/JobQueryDTO.java +++ b/src/main/java/org/fedai/fate/board/pojo/JobQueryDTO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import javax.validation.constraints.NotNull; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/JobStopDTO.java b/src/main/java/org/fedai/fate/board/pojo/JobStopDTO.java similarity index 96% rename from src/main/java/com/webank/ai/fate/board/pojo/JobStopDTO.java rename to src/main/java/org/fedai/fate/board/pojo/JobStopDTO.java index 79108f20..1b6a6528 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/JobStopDTO.java +++ b/src/main/java/org/fedai/fate/board/pojo/JobStopDTO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import javax.validation.constraints.NotNull; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/JobWithBLOBs.java b/src/main/java/org/fedai/fate/board/pojo/JobWithBLOBs.java similarity index 97% rename from src/main/java/com/webank/ai/fate/board/pojo/JobWithBLOBs.java rename to src/main/java/org/fedai/fate/board/pojo/JobWithBLOBs.java index 1e8b1848..c1301203 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/JobWithBLOBs.java +++ b/src/main/java/org/fedai/fate/board/pojo/JobWithBLOBs.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; public class JobWithBLOBs extends Job { private String fDescription; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/MetricDTO.java b/src/main/java/org/fedai/fate/board/pojo/MetricDTO.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/pojo/MetricDTO.java rename to src/main/java/org/fedai/fate/board/pojo/MetricDTO.java index 8668cb32..18820f16 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/MetricDTO.java +++ b/src/main/java/org/fedai/fate/board/pojo/MetricDTO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import javax.validation.constraints.NotNull; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/PagedJobQO.java b/src/main/java/org/fedai/fate/board/pojo/PagedJobQO.java similarity index 97% rename from src/main/java/com/webank/ai/fate/board/pojo/PagedJobQO.java rename to src/main/java/org/fedai/fate/board/pojo/PagedJobQO.java index bbf69aff..7328ce6c 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/PagedJobQO.java +++ b/src/main/java/org/fedai/fate/board/pojo/PagedJobQO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.*; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/ReRunDTO.java b/src/main/java/org/fedai/fate/board/pojo/ReRunDTO.java similarity index 97% rename from src/main/java/com/webank/ai/fate/board/pojo/ReRunDTO.java rename to src/main/java/org/fedai/fate/board/pojo/ReRunDTO.java index 00a2605a..aa50f42d 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/ReRunDTO.java +++ b/src/main/java/org/fedai/fate/board/pojo/ReRunDTO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import javax.validation.constraints.NotNull; import java.io.Serializable; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/SshDTO.java b/src/main/java/org/fedai/fate/board/pojo/SshDTO.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/pojo/SshDTO.java rename to src/main/java/org/fedai/fate/board/pojo/SshDTO.java index 120c5731..1fdbca9b 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/SshDTO.java +++ b/src/main/java/org/fedai/fate/board/pojo/SshDTO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import java.io.Serializable; import java.util.Objects; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/SshInfo.java b/src/main/java/org/fedai/fate/board/pojo/SshInfo.java similarity index 97% rename from src/main/java/com/webank/ai/fate/board/pojo/SshInfo.java rename to src/main/java/org/fedai/fate/board/pojo/SshInfo.java index 657c9360..baac097e 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/SshInfo.java +++ b/src/main/java/org/fedai/fate/board/pojo/SshInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/Task.java b/src/main/java/org/fedai/fate/board/pojo/Task.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/pojo/Task.java rename to src/main/java/org/fedai/fate/board/pojo/Task.java index 4a0434b0..0f9ac255 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/Task.java +++ b/src/main/java/org/fedai/fate/board/pojo/Task.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/TaskDO.java b/src/main/java/org/fedai/fate/board/pojo/TaskDO.java similarity index 99% rename from src/main/java/com/webank/ai/fate/board/pojo/TaskDO.java rename to src/main/java/org/fedai/fate/board/pojo/TaskDO.java index 03dc85ad..fc73dc37 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/TaskDO.java +++ b/src/main/java/org/fedai/fate/board/pojo/TaskDO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import java.io.Serializable; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/TaskExample.java b/src/main/java/org/fedai/fate/board/pojo/TaskExample.java similarity index 99% rename from src/main/java/com/webank/ai/fate/board/pojo/TaskExample.java rename to src/main/java/org/fedai/fate/board/pojo/TaskExample.java index a0666ddc..873efd61 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/TaskExample.java +++ b/src/main/java/org/fedai/fate/board/pojo/TaskExample.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/UpdateNotesDTO.java b/src/main/java/org/fedai/fate/board/pojo/UpdateNotesDTO.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/pojo/UpdateNotesDTO.java rename to src/main/java/org/fedai/fate/board/pojo/UpdateNotesDTO.java index 794da327..f3260aae 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/UpdateNotesDTO.java +++ b/src/main/java/org/fedai/fate/board/pojo/UpdateNotesDTO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import javax.validation.constraints.NotNull; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/UserDTO.java b/src/main/java/org/fedai/fate/board/pojo/UserDTO.java similarity index 96% rename from src/main/java/com/webank/ai/fate/board/pojo/UserDTO.java rename to src/main/java/org/fedai/fate/board/pojo/UserDTO.java index e5061932..1f249900 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/UserDTO.java +++ b/src/main/java/org/fedai/fate/board/pojo/UserDTO.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.pojo; +package org.fedai.fate.board.pojo; import lombok.Data; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/flow/FlowLogCatReq.java b/src/main/java/org/fedai/fate/board/pojo/flow/FlowLogCatReq.java similarity index 80% rename from src/main/java/com/webank/ai/fate/board/pojo/flow/FlowLogCatReq.java rename to src/main/java/org/fedai/fate/board/pojo/flow/FlowLogCatReq.java index 1346f356..ab182987 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/flow/FlowLogCatReq.java +++ b/src/main/java/org/fedai/fate/board/pojo/flow/FlowLogCatReq.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.pojo.flow; +package org.fedai.fate.board.pojo.flow; import lombok.Data; @@ -10,6 +10,7 @@ public class FlowLogCatReq { private String role; private Integer party_id; private String component_name; + private String task_name; private String instance_id; private Integer begin; private Integer end; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/flow/FlowLogCatResp.java b/src/main/java/org/fedai/fate/board/pojo/flow/FlowLogCatResp.java similarity index 73% rename from src/main/java/com/webank/ai/fate/board/pojo/flow/FlowLogCatResp.java rename to src/main/java/org/fedai/fate/board/pojo/flow/FlowLogCatResp.java index 19ea672e..37c18eee 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/flow/FlowLogCatResp.java +++ b/src/main/java/org/fedai/fate/board/pojo/flow/FlowLogCatResp.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.pojo.flow; +package org.fedai.fate.board.pojo.flow; import lombok.Data; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/flow/FlowLogSizeReq.java b/src/main/java/org/fedai/fate/board/pojo/flow/FlowLogSizeReq.java similarity index 77% rename from src/main/java/com/webank/ai/fate/board/pojo/flow/FlowLogSizeReq.java rename to src/main/java/org/fedai/fate/board/pojo/flow/FlowLogSizeReq.java index 4767105b..8a6da60f 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/flow/FlowLogSizeReq.java +++ b/src/main/java/org/fedai/fate/board/pojo/flow/FlowLogSizeReq.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.pojo.flow; +package org.fedai.fate.board.pojo.flow; import lombok.Data; @@ -11,5 +11,6 @@ public class FlowLogSizeReq { private String party_id; private String component_name; private String instance_id; + private String task_name; } diff --git a/src/main/java/com/webank/ai/fate/board/pojo/flow/FlowLogSizeResp.java b/src/main/java/org/fedai/fate/board/pojo/flow/FlowLogSizeResp.java similarity index 66% rename from src/main/java/com/webank/ai/fate/board/pojo/flow/FlowLogSizeResp.java rename to src/main/java/org/fedai/fate/board/pojo/flow/FlowLogSizeResp.java index 326ff036..ce872b6c 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/flow/FlowLogSizeResp.java +++ b/src/main/java/org/fedai/fate/board/pojo/flow/FlowLogSizeResp.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.pojo.flow; +package org.fedai.fate.board.pojo.flow; import lombok.Data; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/flow/FlowResponse.java b/src/main/java/org/fedai/fate/board/pojo/flow/FlowResponse.java similarity index 75% rename from src/main/java/com/webank/ai/fate/board/pojo/flow/FlowResponse.java rename to src/main/java/org/fedai/fate/board/pojo/flow/FlowResponse.java index 4ef031e0..c87a5264 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/flow/FlowResponse.java +++ b/src/main/java/org/fedai/fate/board/pojo/flow/FlowResponse.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.pojo.flow; +package org.fedai.fate.board.pojo.flow; import lombok.Data; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/websocket/LogContentResponse.java b/src/main/java/org/fedai/fate/board/pojo/websocket/LogContentResponse.java similarity index 84% rename from src/main/java/com/webank/ai/fate/board/pojo/websocket/LogContentResponse.java rename to src/main/java/org/fedai/fate/board/pojo/websocket/LogContentResponse.java index 0941835a..995c1db3 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/websocket/LogContentResponse.java +++ b/src/main/java/org/fedai/fate/board/pojo/websocket/LogContentResponse.java @@ -1,6 +1,6 @@ -package com.webank.ai.fate.board.pojo.websocket; +package org.fedai.fate.board.pojo.websocket; -import com.webank.ai.fate.board.pojo.flow.FlowLogCatResp; +import org.fedai.fate.board.pojo.flow.FlowLogCatResp; import lombok.Data; import java.util.List; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/websocket/LogQuery.java b/src/main/java/org/fedai/fate/board/pojo/websocket/LogQuery.java similarity index 76% rename from src/main/java/com/webank/ai/fate/board/pojo/websocket/LogQuery.java rename to src/main/java/org/fedai/fate/board/pojo/websocket/LogQuery.java index 0b843cc8..3c6d4c07 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/websocket/LogQuery.java +++ b/src/main/java/org/fedai/fate/board/pojo/websocket/LogQuery.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.pojo.websocket; +package org.fedai.fate.board.pojo.websocket; import lombok.Data; diff --git a/src/main/java/com/webank/ai/fate/board/pojo/websocket/LogSizeResponse.java b/src/main/java/org/fedai/fate/board/pojo/websocket/LogSizeResponse.java similarity index 85% rename from src/main/java/com/webank/ai/fate/board/pojo/websocket/LogSizeResponse.java rename to src/main/java/org/fedai/fate/board/pojo/websocket/LogSizeResponse.java index 17ea33e1..61fd176c 100644 --- a/src/main/java/com/webank/ai/fate/board/pojo/websocket/LogSizeResponse.java +++ b/src/main/java/org/fedai/fate/board/pojo/websocket/LogSizeResponse.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.pojo.websocket; +package org.fedai.fate.board.pojo.websocket; import lombok.Data; diff --git a/src/main/java/com/webank/ai/fate/board/services/FlowFeign.java b/src/main/java/org/fedai/fate/board/services/FlowFeign.java similarity index 83% rename from src/main/java/com/webank/ai/fate/board/services/FlowFeign.java rename to src/main/java/org/fedai/fate/board/services/FlowFeign.java index 6083ad76..907eb912 100644 --- a/src/main/java/com/webank/ai/fate/board/services/FlowFeign.java +++ b/src/main/java/org/fedai/fate/board/services/FlowFeign.java @@ -1,7 +1,7 @@ -package com.webank.ai.fate.board.services; +package org.fedai.fate.board.services; -import com.webank.ai.fate.board.conf.RouteTargeter; -import com.webank.ai.fate.board.intercept.FeignRequestInterceptor; +import org.fedai.fate.board.conf.RouteTargeter; +import org.fedai.fate.board.intercept.FeignRequestInterceptor; import org.springframework.cloud.openfeign.FeignClient; import org.springframework.cloud.openfeign.SpringQueryMap; import org.springframework.web.bind.annotation.PathVariable; diff --git a/src/main/java/org/fedai/fate/board/services/FlowHighAvailableService.java b/src/main/java/org/fedai/fate/board/services/FlowHighAvailableService.java new file mode 100644 index 00000000..bb5640f8 --- /dev/null +++ b/src/main/java/org/fedai/fate/board/services/FlowHighAvailableService.java @@ -0,0 +1,145 @@ +package org.fedai.fate.board.services; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.github.benmanes.caffeine.cache.Cache; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.recipes.cache.*; +import org.apache.curator.framework.state.ConnectionState; +import org.apache.curator.framework.state.ConnectionStateListener; +import org.apache.curator.retry.RetryNTimes; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Id; +import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; +import org.fedai.fate.board.global.Dict; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.nio.charset.Charset; +import java.util.*; + +@Component +public class FlowHighAvailableService { + + private final Logger logger = LoggerFactory.getLogger(FlowHighAvailableService.class); + + @Value("${zookeeper.url_list}") + private String urls; + @Value("${zookeeper.time_out}") + private int timeout; + @Value("${zookeeper.acl_enable}") + private boolean aclEnable; + @Value("${zookeeper.acl_username}") + private String aclUsername; + @Value("${zookeeper.acl_password}") + private String aclPassword; + @Value("${zookeeper.scheme}") + private String scheme; + @Value("${zookeeper.node_path}") + private String path; + @Value("${zookeeper.enable}") + private boolean zookeeperEnable; + + private Charset CHARSET = Charset.forName("UTF-8"); + private CuratorFramework client; + private List acls = new ArrayList<>(); + + @Resource + private Cache cache; + + @PostConstruct + public void initServerConnect() { + if (!zookeeperEnable) { + return; + } + try { + CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() + .connectString(urls) + .retryPolicy(new RetryNTimes(1, 1000)) + .connectionTimeoutMs(timeout); + + if (aclEnable) { + if (!StringUtils.hasText(aclUsername) || !StringUtils.hasText(aclPassword)) { + aclEnable = false; + } else { + builder.authorization(scheme, (aclUsername + ":" + aclPassword).getBytes()); + Id allow = new Id(scheme, DigestAuthenticationProvider.generateDigest(aclUsername + ":" + aclPassword)); + acls.add(new ACL(ZooDefs.Perms.ALL, allow)); + } + } + client = builder.build(); + client.getConnectionStateListenable().addListener(new ConnectionStateListener() { + @Override + public void stateChanged(CuratorFramework client, ConnectionState state) { + if (state == ConnectionState.SUSPENDED) { + logger.warn("connection state has changed: state is {}", state); + } + } + }); + client.start(); + if (aclEnable) { + client.setACL().withACL(acls).forPath("/"); + } + PathChildrenCache childrenCache = new PathChildrenCache(client, path, true); + addPathChildrenCacheListener(childrenCache); + childrenCache.start(); + } catch (Exception e) { + logger.error("connect to zookeeper failed: {}", e.getMessage()); + } + } + + + private void addPathChildrenCacheListener(PathChildrenCache childrenCache) { + childrenCache.getListenable().addListener(new PathChildrenCacheListener() { + @Override + public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent event) throws Exception { + logger.info("event type: {}", event.getType()); + Map address = null; + Object ifPresent = cache.getIfPresent(path); + if (null != ifPresent) { + address = (Map) ifPresent; + } else { + address = new HashMap<>(); + cache.put(path, address); + } + if (event.getType() == PathChildrenCacheEvent.Type.CHILD_UPDATED || event.getType() == PathChildrenCacheEvent.Type.CHILD_ADDED) { + String result = new String(event.getData().getData()); + logger.info("=================addPathChildrenCacheListener============path: {}, data： {}", event.getData().getPath(), result); + address.put(event.getData().getPath(), result); + } else if (event.getType() == PathChildrenCacheEvent.Type.CHILD_REMOVED) { + address.remove(event.getData().getPath()); + } else { + return; + } + } + }); + } + + + public String getFlowUrlsFromCache() { + Object urlObj = cache.getIfPresent(path); + Map urlMap = null; + if (null != urlObj) { + urlMap = (Map) urlObj; + } + if (urlMap == null || urlMap.size() == 0) { + return null; + } + StringJoiner sj = new StringJoiner(";"); + for (String nodeName : urlMap.keySet()) { + String data = urlMap.get(nodeName); + JSONObject dataJson = JSON.parseObject(data); + String host = dataJson.getString(Dict.ZK_DATA_HOST); + Integer port = dataJson.getInteger(Dict.ZK_DATA_PORT); + sj.add(host + ":" + port); + } + return sj.toString(); + } +} diff --git a/src/main/java/org/fedai/fate/board/services/FlowLogFeign.java b/src/main/java/org/fedai/fate/board/services/FlowLogFeign.java new file mode 100644 index 00000000..c567b13c --- /dev/null +++ b/src/main/java/org/fedai/fate/board/services/FlowLogFeign.java @@ -0,0 +1,23 @@ +package org.fedai.fate.board.services; + +import org.fedai.fate.board.conf.RouteTargeter; +import org.fedai.fate.board.intercept.FeignRequestInterceptor; +import org.fedai.fate.board.pojo.flow.*; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.cloud.openfeign.SpringQueryMap; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; + +import java.util.List; +import java.util.Map; + +@FeignClient(url = RouteTargeter.URL_PLACE_HOLDER + "/v2/log", name = "flowLogFeign", configuration = FeignRequestInterceptor.class) +public interface FlowLogFeign { + + @RequestMapping(value = "/query", method = RequestMethod.GET) + String logCat( @SpringQueryMap(encoded = true) Map paramMap); + + @RequestMapping(value = "/count", method = RequestMethod.GET) + String logSize( @SpringQueryMap(encoded = true) Map paramMap); + +} diff --git a/src/main/java/com/webank/ai/fate/board/services/JobDetailService.java b/src/main/java/org/fedai/fate/board/services/JobDetailService.java similarity index 100% rename from src/main/java/com/webank/ai/fate/board/services/JobDetailService.java rename to src/main/java/org/fedai/fate/board/services/JobDetailService.java diff --git a/src/main/java/com/webank/ai/fate/board/services/JobManagerService.java b/src/main/java/org/fedai/fate/board/services/JobManagerService.java similarity index 96% rename from src/main/java/com/webank/ai/fate/board/services/JobManagerService.java rename to src/main/java/org/fedai/fate/board/services/JobManagerService.java index 1429c53c..2656f856 100644 --- a/src/main/java/com/webank/ai/fate/board/services/JobManagerService.java +++ b/src/main/java/org/fedai/fate/board/services/JobManagerService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.services; +package org.fedai.fate.board.services; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; @@ -22,15 +22,15 @@ import com.alibaba.fastjson.serializer.SerializerFeature; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; -import com.webank.ai.fate.board.exceptions.LogicException; -import com.webank.ai.fate.board.global.Dict; -import com.webank.ai.fate.board.global.ErrorCode; -import com.webank.ai.fate.board.global.ResponseResult; -import com.webank.ai.fate.board.log.LogFileService; -import com.webank.ai.fate.board.pojo.*; -import com.webank.ai.fate.board.utils.PageBean; -import com.webank.ai.fate.board.utils.ThreadPoolTaskExecutorUtil; +import org.fedai.fate.board.exceptions.LogicException; +import org.fedai.fate.board.global.Dict; +import org.fedai.fate.board.global.ErrorCode; +import org.fedai.fate.board.global.ResponseResult; +import org.fedai.fate.board.log.LogFileService; +import org.fedai.fate.board.utils.PageBean; +import org.fedai.fate.board.utils.ThreadPoolTaskExecutorUtil; import lombok.extern.slf4j.Slf4j; +import org.fedai.fate.board.pojo.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -222,12 +222,12 @@ public PageBean> queryPagedJobs(PagedJobQO pagedJobQO) { String result = flowFeign.get(Dict.URL_JOB_QUERY, jobParams); JSONObject resultObject = JSON.parseObject(result); - Integer retCode = resultObject.getInteger(Dict.RETCODE); + Integer retCode = resultObject.getInteger(Dict.CODE); if (400 == retCode || 401 == retCode || 425 == retCode || 403 == retCode) { logger.error(resultObject.getString(Dict.RETMSG)); LogicException.throwError(retCode, resultObject.getString(Dict.RETMSG)); } - JSONObject data = resultObject.getJSONObject(Dict.DATA); + JSONObject data = (JSONObject)resultObject.getJSONArray(Dict.DATA).get(0); return data; }, new int[]{500, 1000}, new int[]{3, 3}); jobWithBLOB.setfDsl(null); diff --git a/src/main/java/com/webank/ai/fate/board/services/JobWebSocketService.java b/src/main/java/org/fedai/fate/board/services/JobWebSocketService.java similarity index 95% rename from src/main/java/com/webank/ai/fate/board/services/JobWebSocketService.java rename to src/main/java/org/fedai/fate/board/services/JobWebSocketService.java index 9c91bc44..1d34560d 100644 --- a/src/main/java/com/webank/ai/fate/board/services/JobWebSocketService.java +++ b/src/main/java/org/fedai/fate/board/services/JobWebSocketService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.services; +package org.fedai.fate.board.services; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Maps; -import com.webank.ai.fate.board.controller.JobDetailController; -import com.webank.ai.fate.board.controller.JobManagerController; -import com.webank.ai.fate.board.global.Dict; -import com.webank.ai.fate.board.global.ResponseResult; -import com.webank.ai.fate.board.pojo.JobDO; +import org.fedai.fate.board.controller.JobDetailController; +import org.fedai.fate.board.controller.JobManagerController; +import org.fedai.fate.board.global.Dict; +import org.fedai.fate.board.global.ResponseResult; +import org.fedai.fate.board.pojo.JobDO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; diff --git a/src/main/java/com/webank/ai/fate/board/services/RSASecurityService.java b/src/main/java/org/fedai/fate/board/services/RSASecurityService.java similarity index 93% rename from src/main/java/com/webank/ai/fate/board/services/RSASecurityService.java rename to src/main/java/org/fedai/fate/board/services/RSASecurityService.java index ff380e93..c5d598f6 100644 --- a/src/main/java/com/webank/ai/fate/board/services/RSASecurityService.java +++ b/src/main/java/org/fedai/fate/board/services/RSASecurityService.java @@ -1,6 +1,6 @@ -package com.webank.ai.fate.board.services; +package org.fedai.fate.board.services; -import com.webank.ai.fate.board.utils.RSAUtils; +import org.fedai.fate.board.utils.RSAUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; diff --git a/src/main/java/com/webank/ai/fate/board/services/SecurityService.java b/src/main/java/org/fedai/fate/board/services/SecurityService.java similarity index 79% rename from src/main/java/com/webank/ai/fate/board/services/SecurityService.java rename to src/main/java/org/fedai/fate/board/services/SecurityService.java index a8d41d41..2513a2ac 100644 --- a/src/main/java/com/webank/ai/fate/board/services/SecurityService.java +++ b/src/main/java/org/fedai/fate/board/services/SecurityService.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.services; +package org.fedai.fate.board.services; public interface SecurityService { diff --git a/src/main/java/com/webank/ai/fate/board/services/TaskManagerService.java b/src/main/java/org/fedai/fate/board/services/TaskManagerService.java similarity index 100% rename from src/main/java/com/webank/ai/fate/board/services/TaskManagerService.java rename to src/main/java/org/fedai/fate/board/services/TaskManagerService.java diff --git a/src/main/java/com/webank/ai/fate/board/services/UserService.java b/src/main/java/org/fedai/fate/board/services/UserService.java similarity index 96% rename from src/main/java/com/webank/ai/fate/board/services/UserService.java rename to src/main/java/org/fedai/fate/board/services/UserService.java index d0786c1d..dffb58d3 100644 --- a/src/main/java/com/webank/ai/fate/board/services/UserService.java +++ b/src/main/java/org/fedai/fate/board/services/UserService.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.services; +package org.fedai.fate.board.services; -import com.webank.ai.fate.board.pojo.UserDTO; -import com.webank.ai.fate.board.utils.StandardRSAUtils; +import org.fedai.fate.board.pojo.UserDTO; +import org.fedai.fate.board.utils.StandardRSAUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/com/webank/ai/fate/board/ssh/SftpUtils.java b/src/main/java/org/fedai/fate/board/ssh/SftpUtils.java similarity index 99% rename from src/main/java/com/webank/ai/fate/board/ssh/SftpUtils.java rename to src/main/java/org/fedai/fate/board/ssh/SftpUtils.java index e663ce60..c0768862 100644 --- a/src/main/java/com/webank/ai/fate/board/ssh/SftpUtils.java +++ b/src/main/java/org/fedai/fate/board/ssh/SftpUtils.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.ssh; +package org.fedai.fate.board.ssh; import com.jcraft.jsch.*; import com.jcraft.jsch.ChannelSftp.LsEntry; -import com.webank.ai.fate.board.pojo.SshInfo; +import org.fedai.fate.board.pojo.SshInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/com/webank/ai/fate/board/ssh/SshConfigFileWatcher.java b/src/main/java/org/fedai/fate/board/ssh/SshConfigFileWatcher.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/ssh/SshConfigFileWatcher.java rename to src/main/java/org/fedai/fate/board/ssh/SshConfigFileWatcher.java index 13ef899d..a7f05640 100644 --- a/src/main/java/com/webank/ai/fate/board/ssh/SshConfigFileWatcher.java +++ b/src/main/java/org/fedai/fate/board/ssh/SshConfigFileWatcher.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.ssh; +package org.fedai.fate.board.ssh; import com.google.common.collect.Maps; -import com.webank.ai.fate.board.global.Dict; +import org.fedai.fate.board.global.Dict; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; diff --git a/src/main/java/com/webank/ai/fate/board/ssh/SshLogScanner.java b/src/main/java/org/fedai/fate/board/ssh/SshLogScanner.java similarity index 95% rename from src/main/java/com/webank/ai/fate/board/ssh/SshLogScanner.java rename to src/main/java/org/fedai/fate/board/ssh/SshLogScanner.java index 798a9fe0..abc1a9d3 100644 --- a/src/main/java/com/webank/ai/fate/board/ssh/SshLogScanner.java +++ b/src/main/java/org/fedai/fate/board/ssh/SshLogScanner.java @@ -13,19 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.ssh; +package org.fedai.fate.board.ssh; import com.alibaba.fastjson.JSON; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.jcraft.jsch.Channel; -import com.webank.ai.fate.board.log.LogFileService; -import com.webank.ai.fate.board.log.LogScanner; -import com.webank.ai.fate.board.pojo.SshInfo; -import com.webank.ai.fate.board.utils.LogHandle; +import org.fedai.fate.board.log.LogFileService; +import org.fedai.fate.board.log.LogScanner; +import org.fedai.fate.board.pojo.SshInfo; +import org.fedai.fate.board.utils.LogHandle; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import java.io.BufferedReader; import java.io.IOException; diff --git a/src/main/java/com/webank/ai/fate/board/ssh/SshService.java b/src/main/java/org/fedai/fate/board/ssh/SshService.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/ssh/SshService.java rename to src/main/java/org/fedai/fate/board/ssh/SshService.java index d325aa2a..2ae208bf 100644 --- a/src/main/java/com/webank/ai/fate/board/ssh/SshService.java +++ b/src/main/java/org/fedai/fate/board/ssh/SshService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.ssh; +package org.fedai.fate.board.ssh; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; @@ -21,8 +21,8 @@ import com.jcraft.jsch.ChannelExec; import com.jcraft.jsch.JSch; import com.jcraft.jsch.Session; -import com.webank.ai.fate.board.pojo.SshInfo; -import com.webank.ai.fate.board.global.Dict; +import org.fedai.fate.board.pojo.SshInfo; +import org.fedai.fate.board.global.Dict; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; diff --git a/src/main/java/com/webank/ai/fate/board/utils/CheckPathUtil.java b/src/main/java/org/fedai/fate/board/utils/CheckPathUtil.java similarity index 95% rename from src/main/java/com/webank/ai/fate/board/utils/CheckPathUtil.java rename to src/main/java/org/fedai/fate/board/utils/CheckPathUtil.java index 25a3a7ca..bbf083a6 100644 --- a/src/main/java/com/webank/ai/fate/board/utils/CheckPathUtil.java +++ b/src/main/java/org/fedai/fate/board/utils/CheckPathUtil.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.utils; +package org.fedai.fate.board.utils; public class CheckPathUtil { public static boolean checkPath(String... paths){ diff --git a/src/main/java/com/webank/ai/fate/board/utils/GetSystemInfo.java b/src/main/java/org/fedai/fate/board/utils/GetSystemInfo.java similarity index 99% rename from src/main/java/com/webank/ai/fate/board/utils/GetSystemInfo.java rename to src/main/java/org/fedai/fate/board/utils/GetSystemInfo.java index 30f67191..151e1bea 100644 --- a/src/main/java/com/webank/ai/fate/board/utils/GetSystemInfo.java +++ b/src/main/java/org/fedai/fate/board/utils/GetSystemInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.utils; +package org.fedai.fate.board.utils; import com.sun.management.OperatingSystemMXBean; import org.slf4j.Logger; diff --git a/src/main/java/com/webank/ai/fate/board/utils/LogHandle.java b/src/main/java/org/fedai/fate/board/utils/LogHandle.java similarity index 96% rename from src/main/java/com/webank/ai/fate/board/utils/LogHandle.java rename to src/main/java/org/fedai/fate/board/utils/LogHandle.java index 324105e5..1aeb682e 100644 --- a/src/main/java/com/webank/ai/fate/board/utils/LogHandle.java +++ b/src/main/java/org/fedai/fate/board/utils/LogHandle.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.utils; +package org.fedai.fate.board.utils; -import com.webank.ai.fate.board.global.Dict; +import org.fedai.fate.board.global.Dict; import org.springframework.boot.system.ApplicationHome; import java.io.File; diff --git a/src/main/java/com/webank/ai/fate/board/utils/PageBean.java b/src/main/java/org/fedai/fate/board/utils/PageBean.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/utils/PageBean.java rename to src/main/java/org/fedai/fate/board/utils/PageBean.java index 9f87ca90..54df7f90 100644 --- a/src/main/java/com/webank/ai/fate/board/utils/PageBean.java +++ b/src/main/java/org/fedai/fate/board/utils/PageBean.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.utils; +package org.fedai.fate.board.utils; import java.io.Serializable; import java.util.List; diff --git a/src/main/java/com/webank/ai/fate/board/utils/RSAUtils.java b/src/main/java/org/fedai/fate/board/utils/RSAUtils.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/utils/RSAUtils.java rename to src/main/java/org/fedai/fate/board/utils/RSAUtils.java index 749129c7..d5e5505a 100644 --- a/src/main/java/com/webank/ai/fate/board/utils/RSAUtils.java +++ b/src/main/java/org/fedai/fate/board/utils/RSAUtils.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.utils; +package org.fedai.fate.board.utils; import lombok.Data; import org.apache.commons.codec.binary.Base64; diff --git a/src/main/java/com/webank/ai/fate/board/utils/ReadJson.java b/src/main/java/org/fedai/fate/board/utils/ReadJson.java similarity index 97% rename from src/main/java/com/webank/ai/fate/board/utils/ReadJson.java rename to src/main/java/org/fedai/fate/board/utils/ReadJson.java index b0f6aadf..4c45fb7c 100644 --- a/src/main/java/com/webank/ai/fate/board/utils/ReadJson.java +++ b/src/main/java/org/fedai/fate/board/utils/ReadJson.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.utils; +package org.fedai.fate.board.utils; import java.io.*; diff --git a/src/main/java/com/webank/ai/fate/board/utils/ResponseUtil.java b/src/main/java/org/fedai/fate/board/utils/ResponseUtil.java similarity index 77% rename from src/main/java/com/webank/ai/fate/board/utils/ResponseUtil.java rename to src/main/java/org/fedai/fate/board/utils/ResponseUtil.java index fa204c44..7c92af8e 100644 --- a/src/main/java/com/webank/ai/fate/board/utils/ResponseUtil.java +++ b/src/main/java/org/fedai/fate/board/utils/ResponseUtil.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.utils; +package org.fedai.fate.board.utils; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; -import com.webank.ai.fate.board.global.Dict; -import com.webank.ai.fate.board.global.ErrorCode; -import com.webank.ai.fate.board.global.ResponseResult; +import org.fedai.fate.board.global.Dict; +import org.fedai.fate.board.global.ErrorCode; +import org.fedai.fate.board.global.ResponseResult; import org.apache.commons.lang3.StringUtils; public class ResponseUtil { @@ -33,8 +33,8 @@ public static ResponseResult buildResponse(String result, String dataName) { } JSONObject resultObject = JSON.parseObject(result); - Integer retcode = resultObject.getInteger(Dict.CODE); - String msg = resultObject.getString(Dict.REMOTE_RETURN_MSG); + Integer retcode = resultObject.getInteger(Dict.CODE) == null ? 200 : resultObject.getInteger(Dict.CODE); + String msg = resultObject.getString(Dict.RETMSG) == null ? "ok" : resultObject.getString(Dict.RETMSG); if (dataName != null) { JSONArray jsonArray = resultObject.getJSONArray(Dict.DATA); diff --git a/src/main/java/com/webank/ai/fate/board/utils/StandardRSAUtils.java b/src/main/java/org/fedai/fate/board/utils/StandardRSAUtils.java similarity index 99% rename from src/main/java/com/webank/ai/fate/board/utils/StandardRSAUtils.java rename to src/main/java/org/fedai/fate/board/utils/StandardRSAUtils.java index eb3406a7..cb87f23c 100644 --- a/src/main/java/com/webank/ai/fate/board/utils/StandardRSAUtils.java +++ b/src/main/java/org/fedai/fate/board/utils/StandardRSAUtils.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.utils; +package org.fedai.fate.board.utils; import javax.crypto.Cipher; import java.io.ByteArrayOutputStream; diff --git a/src/main/java/com/webank/ai/fate/board/utils/TelnetUtil.java b/src/main/java/org/fedai/fate/board/utils/TelnetUtil.java similarity index 93% rename from src/main/java/com/webank/ai/fate/board/utils/TelnetUtil.java rename to src/main/java/org/fedai/fate/board/utils/TelnetUtil.java index f267f9e7..4a857fef 100644 --- a/src/main/java/com/webank/ai/fate/board/utils/TelnetUtil.java +++ b/src/main/java/org/fedai/fate/board/utils/TelnetUtil.java @@ -1,4 +1,4 @@ -package com.webank.ai.fate.board.utils; +package org.fedai.fate.board.utils; import java.io.IOException; import java.net.InetSocketAddress; diff --git a/src/main/java/com/webank/ai/fate/board/utils/ThreadPoolTaskExecutorUtil.java b/src/main/java/org/fedai/fate/board/utils/ThreadPoolTaskExecutorUtil.java similarity index 98% rename from src/main/java/com/webank/ai/fate/board/utils/ThreadPoolTaskExecutorUtil.java rename to src/main/java/org/fedai/fate/board/utils/ThreadPoolTaskExecutorUtil.java index f61d1d0e..d74c6070 100644 --- a/src/main/java/com/webank/ai/fate/board/utils/ThreadPoolTaskExecutorUtil.java +++ b/src/main/java/org/fedai/fate/board/utils/ThreadPoolTaskExecutorUtil.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.utils; +package org.fedai.fate.board.utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/com/webank/ai/fate/board/websocket/JobWebSocketController.java b/src/main/java/org/fedai/fate/board/websocket/JobWebSocketController.java similarity index 79% rename from src/main/java/com/webank/ai/fate/board/websocket/JobWebSocketController.java rename to src/main/java/org/fedai/fate/board/websocket/JobWebSocketController.java index 06ea7039..75cdc7d7 100644 --- a/src/main/java/com/webank/ai/fate/board/websocket/JobWebSocketController.java +++ b/src/main/java/org/fedai/fate/board/websocket/JobWebSocketController.java @@ -13,42 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.webank.ai.fate.board.websocket; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; -import com.webank.ai.fate.board.conf.Configurator; -import com.webank.ai.fate.board.controller.JobDetailController; -import com.webank.ai.fate.board.controller.JobManagerController; -import com.webank.ai.fate.board.global.ResponseResult; -import com.webank.ai.fate.board.log.LogFileService; -import com.webank.ai.fate.board.pojo.Job; -import com.webank.ai.fate.board.pojo.JobDO; -import com.webank.ai.fate.board.services.JobManagerService; -import com.webank.ai.fate.board.global.Dict; -import com.webank.ai.fate.board.services.JobWebSocketService; +package org.fedai.fate.board.websocket; + +import org.fedai.fate.board.conf.Configurator; +import org.fedai.fate.board.controller.JobDetailController; +import org.fedai.fate.board.controller.JobManagerController; +import org.fedai.fate.board.log.LogFileService; +import org.fedai.fate.board.services.JobManagerService; +import org.fedai.fate.board.services.JobWebSocketService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeansException; import org.springframework.beans.factory.InitializingBean; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Component; -import org.springframework.util.concurrent.ListenableFuture; import javax.websocket.*; import javax.websocket.server.PathParam; import javax.websocket.server.ServerEndpoint; -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.*; @ServerEndpoint(value = "/websocket/progress/{jobId}/{role}/{partyId}", configurator = Configurator.class) @Component diff --git a/src/main/java/com/webank/ai/fate/board/websocket/LogWebSocketController.java b/src/main/java/org/fedai/fate/board/websocket/LogWebSocketController.java similarity index 100% rename from src/main/java/com/webank/ai/fate/board/websocket/LogWebSocketController.java rename to src/main/java/org/fedai/fate/board/websocket/LogWebSocketController.java diff --git a/src/main/resources/static b/src/main/resources/static deleted file mode 120000 index f1e308fd..00000000 --- a/src/main/resources/static +++ /dev/null @@ -1 +0,0 @@ -../../../resources-front-end/packages/fate-board/dist \ No newline at end of file diff --git a/static/coordinated_linr.yaml b/static/coordinated_linr.yaml new file mode 100644 index 00000000..c380b8b3 --- /dev/null +++ b/static/coordinated_linr.yaml @@ -0,0 +1,380 @@ +component: + name: coordinated_linr + description: '' + provider: fate + version: 2.0.0-beta + labels: [ ] + roles: + - guest + - host + - arbiter + parameters: + learning_rate_scheduler: + type: LRSchedulerParam + default: + method: linear + scheduler_params: + start_factor: 1.0 + optional: true + description: learning rate scheduler, select method from {'step', 'linear', + 'constant'}for list of configurable arguments, refer to torch.optim.lr_scheduler + type_meta: + title: LRSchedulerParam + $ref: '#/definitions/abc__LRSchedulerParam' + definitions: + abc__LRSchedulerParam: + title: LRSchedulerParam + type: object + properties: + method: + title: Method + default: constant + type: string + scheduler_params: + title: Scheduler Params + type: object + default: + method: linear + scheduler_params: + start_factor: 1.0 + description: learning rate scheduler, select method from {'step', 'linear', + 'constant'}for list of configurable arguments, refer to torch.optim.lr_scheduler + epochs: + type: ConstrainedNumberMeta + default: 20 + optional: true + description: max iteration num + type_meta: + title: ConstrainedNumberMeta + exclusiveMinimum: 0 + type: integer + batch_size: + type: ConstrainedNumberMeta + default: + optional: true + description: batch size, None means full batch, otherwise should be no less + than 10, default None + type_meta: + title: ConstrainedNumberMeta + minimum: 10 + type: integer + optimizer: + type: OptimizerParam + default: + method: sgd + penalty: l2 + alpha: 1.0 + optimizer_params: + lr: 0.01 + weight_decay: 0 + optional: true + description: '' + type_meta: + title: OptimizerParam + $ref: '#/definitions/abc__OptimizerParam' + definitions: + abc__OptimizerParam: + title: OptimizerParam + type: object + properties: + method: + title: Method + default: sgd + type: string + penalty: + title: Penalty + default: l2 + type: string + alpha: + title: Alpha + default: 1.0 + type: number + optimizer_params: + title: Optimizer Params + type: object + required: + - optimizer_params + default: + method: sgd + penalty: l2 + alpha: 1.0 + optimizer_params: + lr: 0.01 + weight_decay: 0 + description: '' + tol: + type: ConstrainedNumberMeta + default: 0.0001 + optional: true + description: '' + type_meta: + title: ConstrainedNumberMeta + minimum: 0 + type: number + early_stop: + type: type + default: diff + optional: true + description: early stopping criterion, choose from {weight_diff, diff, abs, + val_metrics} + type_meta: + title: type + type: string + init_param: + type: InitParam + default: + method: random_uniform + fill_val: 0.0 + fit_intercept: true + random_state: + optional: true + description: Model param init setting. + type_meta: + title: InitParam + $ref: '#/definitions/abc__InitParam' + definitions: + abc__InitParam: + title: InitParam + type: object + properties: + method: + title: Method + default: zeros + type: string + fill_val: + title: Fill Val + default: 0.0 + anyOf: + - type: integer + - type: number + fit_intercept: + title: Fit Intercept + default: true + type: boolean + random_state: + title: Random State + type: integer + default: + method: random_uniform + fill_val: 0.0 + fit_intercept: true + random_state: + description: Model param init setting. + he_param: + type: HEParam + default: + kind: paillier + key_length: 1024 + optional: true + description: homomorphic encryption param + type_meta: + title: HEParam + $ref: '#/definitions/abc__HEParam' + definitions: + abc__HEParam: + title: HEParam + type: object + properties: + kind: + title: Kind + type: string + key_length: + title: Key Length + default: 1024 + type: integer + required: + - kind + default: + kind: paillier + key_length: 1024 + description: homomorphic encryption param + floating_point_precision: + type: ConstrainedNumberMeta + default: 23 + optional: true + description: 'floating point precision, ' + type_meta: + title: ConstrainedNumberMeta + minimum: 0 + type: integer + cv_param: + type: CVParam + default: + n_splits: 5 + shuffle: false + random_state: + optional: true + description: cross validation param + type_meta: + title: CVParam + $ref: '#/definitions/abc__CVParam' + definitions: + abc__CVParam: + title: CVParam + type: object + properties: + n_splits: + title: N Splits + exclusiveMinimum: 1 + type: integer + shuffle: + title: Shuffle + default: false + type: boolean + random_state: + title: Random State + type: integer + required: + - n_splits + default: + n_splits: 5 + shuffle: false + random_state: + description: cross validation param + metrics: + type: type + default: + - mse + optional: true + description: '' + type_meta: + title: type + type: string + output_cv_data: + type: bool + default: true + optional: true + description: whether output prediction result per cv fold + type_meta: + title: bool + type: boolean + default: true + description: whether output prediction result per cv fold + input_artifacts: + data: + train_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + validate_data: + types: + - dataframe + optional: true + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + cv_data: + types: + - dataframe + optional: false + stages: + - cross_validation + roles: + - guest + - host + description: '' + is_multi: false + model: + warm_start_model: + types: + - json_model + optional: true + stages: + - train + roles: + - guest + - host + - arbiter + description: '' + is_multi: false + input_model: + types: + - json_model + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + output_artifacts: + data: + train_output_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_output_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + cv_output_datas: + types: + - dataframe + optional: true + stages: + - cross_validation + roles: + - guest + - host + description: '' + is_multi: true + model: + output_model: + types: + - json_model + optional: false + stages: + - train + roles: + - guest + - host + - arbiter + description: '' + is_multi: false + metric: + metric: + types: + - json_metric + optional: false + stages: [ ] + roles: [ ] + description: metric, invisible for user + is_multi: false +schema_version: v1 diff --git a/static/coordinated_lr.yaml b/static/coordinated_lr.yaml new file mode 100644 index 00000000..0c61cb1e --- /dev/null +++ b/static/coordinated_lr.yaml @@ -0,0 +1,390 @@ +component: + name: coordinated_lr + description: '' + provider: fate + version: 2.0.0-beta + labels: [ ] + roles: + - guest + - host + - arbiter + parameters: + learning_rate_scheduler: + type: LRSchedulerParam + default: + method: linear + scheduler_params: + start_factor: 1.0 + optional: true + description: learning rate scheduler, select method from {'step', 'linear', + 'constant'}for list of configurable arguments, refer to torch.optim.lr_scheduler + type_meta: + title: LRSchedulerParam + $ref: '#/definitions/abc__LRSchedulerParam' + definitions: + abc__LRSchedulerParam: + title: LRSchedulerParam + type: object + properties: + method: + title: Method + default: constant + type: string + scheduler_params: + title: Scheduler Params + type: object + default: + method: linear + scheduler_params: + start_factor: 1.0 + description: learning rate scheduler, select method from {'step', 'linear', + 'constant'}for list of configurable arguments, refer to torch.optim.lr_scheduler + epochs: + type: ConstrainedNumberMeta + default: 20 + optional: true + description: max iteration num + type_meta: + title: ConstrainedNumberMeta + exclusiveMinimum: 0 + type: integer + batch_size: + type: ConstrainedNumberMeta + default: + optional: true + description: batch size, None means full batch, otherwise should be no less + than 10, default None + type_meta: + title: ConstrainedNumberMeta + minimum: 10 + type: integer + optimizer: + type: OptimizerParam + default: + method: sgd + penalty: l2 + alpha: 1.0 + optimizer_params: + lr: 0.01 + weight_decay: 0 + optional: true + description: '' + type_meta: + title: OptimizerParam + $ref: '#/definitions/abc__OptimizerParam' + definitions: + abc__OptimizerParam: + title: OptimizerParam + type: object + properties: + method: + title: Method + default: sgd + type: string + penalty: + title: Penalty + default: l2 + type: string + alpha: + title: Alpha + default: 1.0 + type: number + optimizer_params: + title: Optimizer Params + type: object + required: + - optimizer_params + default: + method: sgd + penalty: l2 + alpha: 1.0 + optimizer_params: + lr: 0.01 + weight_decay: 0 + description: '' + floating_point_precision: + type: ConstrainedNumberMeta + default: 23 + optional: true + description: 'floating point precision, ' + type_meta: + title: ConstrainedNumberMeta + minimum: 0 + type: integer + tol: + type: ConstrainedNumberMeta + default: 0.0001 + optional: true + description: '' + type_meta: + title: ConstrainedNumberMeta + minimum: 0 + type: number + early_stop: + type: type + default: diff + optional: true + description: early stopping criterion, choose from {weight_diff, diff, abs, + val_metrics} + type_meta: + title: type + type: string + he_param: + type: HEParam + default: + kind: paillier + key_length: 1024 + optional: true + description: homomorphic encryption param + type_meta: + title: HEParam + $ref: '#/definitions/abc__HEParam' + definitions: + abc__HEParam: + title: HEParam + type: object + properties: + kind: + title: Kind + type: string + key_length: + title: Key Length + default: 1024 + type: integer + required: + - kind + default: + kind: paillier + key_length: 1024 + description: homomorphic encryption param + init_param: + type: InitParam + default: + method: random_uniform + fill_val: 0.0 + fit_intercept: true + random_state: + optional: true + description: Model param init setting. + type_meta: + title: InitParam + $ref: '#/definitions/abc__InitParam' + definitions: + abc__InitParam: + title: InitParam + type: object + properties: + method: + title: Method + default: zeros + type: string + fill_val: + title: Fill Val + default: 0.0 + anyOf: + - type: integer + - type: number + fit_intercept: + title: Fit Intercept + default: true + type: boolean + random_state: + title: Random State + type: integer + default: + method: random_uniform + fill_val: 0.0 + fit_intercept: true + random_state: + description: Model param init setting. + threshold: + type: ConstrainedNumberMeta + default: 0.5 + optional: true + description: predict threshold for binary data + type_meta: + title: ConstrainedNumberMeta + minimum: 0.0 + maximum: 1.0 + type: number + cv_param: + type: CVParam + default: + n_splits: 5 + shuffle: false + random_state: + optional: true + description: cross validation param + type_meta: + title: CVParam + $ref: '#/definitions/abc__CVParam' + definitions: + abc__CVParam: + title: CVParam + type: object + properties: + n_splits: + title: N Splits + exclusiveMinimum: 1 + type: integer + shuffle: + title: Shuffle + default: false + type: boolean + random_state: + title: Random State + type: integer + required: + - n_splits + default: + n_splits: 5 + shuffle: false + random_state: + description: cross validation param + metrics: + type: type + default: + - auc + optional: true + description: '' + type_meta: + title: type + type: string + output_cv_data: + type: bool + default: true + optional: true + description: whether output prediction result per cv fold + type_meta: + title: bool + type: boolean + default: true + description: whether output prediction result per cv fold + input_artifacts: + data: + train_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + validate_data: + types: + - dataframe + optional: true + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + cv_data: + types: + - dataframe + optional: false + stages: + - cross_validation + roles: + - guest + - host + description: '' + is_multi: false + model: + warm_start_model: + types: + - json_model + optional: true + stages: + - train + roles: + - guest + - host + - arbiter + description: '' + is_multi: false + input_model: + types: + - json_model + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + output_artifacts: + data: + train_output_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_output_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + cv_output_datas: + types: + - dataframe + optional: true + stages: + - cross_validation + roles: + - guest + - host + description: '' + is_multi: true + model: + output_model: + types: + - json_model + optional: false + stages: + - train + roles: + - guest + - host + - arbiter + description: '' + is_multi: false + metric: + metric: + types: + - json_metric + optional: false + stages: [ ] + roles: [ ] + description: metric, invisible for user + is_multi: false +schema_version: v1 diff --git a/static/data_split.yaml b/static/data_split.yaml new file mode 100644 index 00000000..e3cb85ce --- /dev/null +++ b/static/data_split.yaml @@ -0,0 +1,163 @@ +component: + name: data_split + description: '' + provider: fate + version: 2.0.0-beta + labels: [ ] + roles: + - guest + - host + parameters: + train_size: + type: typing.Union[fate.components.core.params._fields.ConstrainedFloatValue, + fate.components.core.params._fields.ConstrainedIntValue] + default: + optional: true + description: size of output training data, should be either int for exact sample + size or float for fraction + type_meta: + title: typing.Union[fate.components.core.params._fields.ConstrainedFloatValue, + fate.components.core.params._fields.ConstrainedIntValue] + anyOf: + - type: number + minimum: 0.0 + maximum: 1.0 + - type: integer + minimum: 0 + default: + description: size of output training data, should be either int for exact + sample size or float for fraction + validate_size: + type: typing.Union[fate.components.core.params._fields.ConstrainedFloatValue, + fate.components.core.params._fields.ConstrainedIntValue] + default: + optional: true + description: size of output validation data, should be either int for exact + sample size or float for fraction + type_meta: + title: typing.Union[fate.components.core.params._fields.ConstrainedFloatValue, + fate.components.core.params._fields.ConstrainedIntValue] + anyOf: + - type: number + minimum: 0.0 + maximum: 1.0 + - type: integer + minimum: 0 + default: + description: size of output validation data, should be either int for exact + sample size or float for fraction + test_size: + type: typing.Union[fate.components.core.params._fields.ConstrainedFloatValue, + fate.components.core.params._fields.ConstrainedIntValue] + default: + optional: true + description: size of output test data, should be either int for exact sample + size or float for fraction + type_meta: + title: typing.Union[fate.components.core.params._fields.ConstrainedFloatValue, + fate.components.core.params._fields.ConstrainedIntValue] + anyOf: + - type: number + minimum: 0.0 + maximum: 1.0 + - type: integer + minimum: 0 + default: + description: size of output test data, should be either int for exact sample + size or float for fraction + stratified: + type: bool + default: false + optional: true + description: whether sample with stratification, should not use this for data + with continuous label values + type_meta: + title: bool + type: boolean + default: true + description: whether guest sync data set sids with host, default True for + hetero scenario, should set to False for local and homo scenario + random_state: + type: ConstrainedNumberMeta + default: + optional: true + description: random state + type_meta: + title: ConstrainedNumberMeta + minimum: 0 + type: integer + hetero_sync: + type: bool + default: true + optional: true + description: whether guest sync data set sids with host, default True for hetero + scenario, should set to False for local and homo scenario + type_meta: + title: bool + type: boolean + default: true + description: whether guest sync data set sids with host, default True for + hetero scenario, should set to False for local and homo scenario + input_artifacts: + data: + input_data: + types: + - dataframe + optional: false + stages: + - default + roles: + - guest + - host + description: '' + is_multi: false + model: { } + output_artifacts: + data: + train_output_data: + types: + - dataframe + optional: true + stages: + - default + roles: + - guest + - host + description: '' + is_multi: false + validate_output_data: + types: + - dataframe + optional: true + stages: + - default + roles: + - guest + - host + description: '' + is_multi: false + test_output_data: + types: + - dataframe + optional: true + stages: + - default + roles: + - guest + - host + description: '' + is_multi: false + model: { } + metric: + metric: + types: + - json_metric + optional: false + stages: + - default + roles: + - guest + - host + description: metric, invisible for user + is_multi: false +schema_version: v1 diff --git a/static/dataframe_io_test.yaml b/static/dataframe_io_test.yaml new file mode 100644 index 00000000..17815106 --- /dev/null +++ b/static/dataframe_io_test.yaml @@ -0,0 +1,97 @@ +component: + name: dataframe_io_test + description: '' + provider: fate + version: 2.0.0-beta + labels: [] + roles: + - local + parameters: {} + input_artifacts: + data: + dataframe_input: + types: + - dataframe + optional: false + stages: + - default + roles: + - local + description: '' + is_multi: false + dataframe_inputs: + types: + - dataframe + optional: false + stages: + - default + roles: + - local + description: '' + is_multi: true + model: {} + output_artifacts: + data: + dataframe_output: + types: + - dataframe + optional: false + stages: + - default + roles: + - local + description: '' + is_multi: false + dataframe_outputs: + types: + - dataframe + optional: false + stages: + - default + roles: + - local + description: '' + is_multi: true + model: + json_model_output: + types: + - json_model + optional: false + stages: + - default + roles: + - local + description: '' + is_multi: false + json_model_outputs: + types: + - json_model + optional: false + stages: + - default + roles: + - local + description: '' + is_multi: true + model_directory_output: + types: + - model_directory + optional: false + stages: + - default + roles: + - local + description: '' + is_multi: false + model_directory_outputs: + types: + - model_directory + optional: false + stages: + - default + roles: + - local + description: '' + is_multi: true + metric: {} +schema_version: v1 \ No newline at end of file diff --git a/static/dataframe_transformer.yaml b/static/dataframe_transformer.yaml new file mode 100644 index 00000000..64452ebe --- /dev/null +++ b/static/dataframe_transformer.yaml @@ -0,0 +1,93 @@ +component: + name: dataframe_transformer + description: '' + provider: fate + version: 2.0.0-beta + labels: [] + roles: + - local + parameters: + namespace: + type: parameter_namespace + default: + optional: true + description: '' + type_meta: + title: parameter_namespace + type: object + properties: + data: + title: Data + type: string + required: + - data + name: + type: parameter_name + default: + optional: true + description: '' + type_meta: + title: parameter_name + type: object + properties: + data: + title: Data + type: string + required: + - data + anonymous_role: + type: parameter_anonymous_role + default: + optional: true + description: '' + type_meta: + title: parameter_anonymous_role + type: object + properties: + data: + title: Data + type: string + required: + - data + anonymous_party_id: + type: parameter_anonymous_party_id + default: + optional: true + description: '' + type_meta: + title: parameter_anonymous_party_id + type: object + properties: + data: + title: Data + type: string + required: + - data + input_artifacts: + data: + table: + types: + - table + optional: false + stages: + - default + roles: + - local + description: '' + is_multi: false + model: {} + output_artifacts: + data: + dataframe_output: + types: + - dataframe + optional: false + stages: + - default + roles: + - local + description: '' + is_multi: false + model: {} + metric: {} +schema_version: v1 diff --git a/static/evaluation.yaml b/static/evaluation.yaml new file mode 100644 index 00000000..a98f7fe0 --- /dev/null +++ b/static/evaluation.yaml @@ -0,0 +1,85 @@ +component: + name: evaluation + description: '' + provider: fate + version: 2.0.0-beta + labels: [] + roles: + - guest + - host + parameters: + default_eval_setting: + type: type + default: binary + optional: true + description: '' + type_meta: + title: type + type: string + metrics: + type: list + default: + optional: true + description: '' + type_meta: + title: list + type: array + items: {} + default: + description: '' + predict_column_name: + type: str + default: + optional: true + description: predict data column name, if None(default), will use 'predict_score' + when use binary and regression default setting, and + use 'predict_result' on multi classification default setting + type_meta: + title: str + type: string + default: + description: label data column namem if None(default), will + use 'label' in the input dataframe + label_column_name: + type: str + default: + optional: true + description: label data column namem if None(default), will + use 'label' in the input dataframe + type_meta: + title: str + type: string + default: + description: label data column namem if None(default), will + use 'label' in the input dataframe + input_artifacts: + data: + input_data: + types: + - dataframe + optional: false + stages: + - default + roles: + - guest + - host + description: '' + is_multi: true + model: {} + output_artifacts: + data: {} + model: {} + metric: + metric: + types: + - json_metric + optional: false + stages: + - default + roles: + - guest + - host + description: metric, invisible for user + is_multi: false +schema_version: v1 + diff --git a/static/feature_scale.yaml b/static/feature_scale.yaml new file mode 100644 index 00000000..cb104fe6 --- /dev/null +++ b/static/feature_scale.yaml @@ -0,0 +1,171 @@ +component: + name: feature_scale + description: '' + provider: fate + version: 2.0.0-beta + labels: [] + roles: + - guest + - host + parameters: + method: + type: type + default: standard + optional: false + description: '' + type_meta: + title: type + type: string + feature_range: + type: typing.Union[list, dict] + default: + - 0 + - 1 + optional: true + description: 'Result feature value range for `min_max` method, take either dict + in format: {col_name: [min, max]} for specific columns or [min, max] for all + columns. Columns unspecified will be scaled to default range [0,1]' + type_meta: + title: typing.Union[list, dict] + anyOf: + - type: array + items: { } + - type: object + default: + - 0 + - 1 + description: 'Result feature value range for `min_max` method, take either + dict in format: {col_name: [min, max]} for specific columns or [min, max] + for all columns. Columns unspecified will be scaled to default range [0,1]' + scale_col: + type: typing.List[str] + default: + optional: true + description: list of column names to be scaled, if None, all columns will be + scaled; only one of {scale_col, scale_idx} should be specified + type_meta: + title: typing.List[str] + type: array + items: + type: string + default: + description: list of column names to be scaled, if None, all columns will + be scaled; only one of {scale_col, scale_idx} should be specified + scale_idx: + type: typing.List[fate.components.core.params._fields.ConstrainedIntValue] + default: + optional: true + description: list of column index to be scaled, if None, all columns will be + scaled; only one of {scale_col, scale_idx} should be specified + type_meta: + title: typing.List[fate.components.core.params._fields.ConstrainedIntValue] + type: array + items: + type: integer + minimum: 0 + default: + description: list of column index to be scaled, if None, all columns will + be scaled; only one of {scale_col, scale_idx} should be specified + strict_range: + type: bool + default: true + optional: true + description: whether transformed value to be strictly restricted within given + range; effective for 'min_max' scale method only + type_meta: + title: bool + type: boolean + default: false + description: bool, whether interpret `scale_col` as anonymous column names + use_anonymous: + type: bool + default: false + optional: true + description: bool, whether interpret `scale_col` as anonymous column names + type_meta: + title: bool + type: boolean + default: false + description: bool, whether interpret `scale_col` as anonymous column names + input_artifacts: + data: + train_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + model: + input_model: + types: + - json_model + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + output_artifacts: + data: + train_output_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_output_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + model: + output_model: + types: + - json_model + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + metric: + metric: + types: + - json_metric + optional: false + stages: [ ] + roles: [ ] + description: metric, invisible for user + is_multi: false +schema_version: v1 diff --git a/static/hetero_feature_binning.yaml b/static/hetero_feature_binning.yaml new file mode 100644 index 00000000..1cd9a5d8 --- /dev/null +++ b/static/hetero_feature_binning.yaml @@ -0,0 +1,273 @@ +component: + name: hetero_feature_binning + description: '' + provider: fate + version: 2.0.0-beta + labels: [ ] + roles: + - guest + - host + parameters: + method: + type: type + default: quantile + optional: false + description: 'binning method, options: {quantile, bucket, manual}' + type_meta: + title: type + type: string + n_bins: + type: ConstrainedNumberMeta + default: 10 + optional: true + description: max number of bins, should be no less than 2 + type_meta: + title: ConstrainedNumberMeta + exclusiveMinimum: 1 + type: integer + split_pt_dict: + type: dict + default: + optional: true + description: dict, manually provided split points, only effective when `method`='manual' + type_meta: + title: dict + type: object + default: + description: dict, manually provided split points, only effective when `method`='manual' + bin_col: + type: typing.List[str] + default: + optional: true + description: list of column names to be binned, if None, all columns will be + binned; only one of {bin_col, bin_idx} should be specified + type_meta: + title: typing.List[str] + type: array + items: + type: string + default: + description: list of column names to be treated as categorical features and + will not be binned; only one of {category_col, category_idx} should be specifiednote + that metrics will be computed over categorical features if this param is + specified + bin_idx: + type: typing.List[fate.components.core.params._fields.ConstrainedIntValue] + default: + optional: true + description: list of column index to be binned, if None, all columns will be + binned; only one of {bin_col, bin_idx} should be specified + type_meta: + title: typing.List[fate.components.core.params._fields.ConstrainedIntValue] + type: array + items: + type: integer + minimum: 0 + default: + description: list of column index to be binned, if None, all columns will + be binned; only one of {bin_col, bin_idx} should be specified + category_col: + type: typing.List[str] + default: + optional: true + description: list of column names to be treated as categorical features and + will not be binned; only one of {category_col, category_idx} should be specifiednote + that metrics will be computed over categorical features if this param is specified + type_meta: + title: typing.List[str] + type: array + items: + type: string + default: + description: list of column names to be treated as categorical features and + will not be binned; only one of {category_col, category_idx} should be specifiednote + that metrics will be computed over categorical features if this param is + specified + category_idx: + type: typing.List[fate.components.core.params._fields.ConstrainedIntValue] + default: + optional: true + description: list of column index to be treated as categorical features and + will not be binned; only one of {category_col, category_idx} should be specifiednote + that metrics will be computed over categorical features if this param is specified + type_meta: + title: typing.List[fate.components.core.params._fields.ConstrainedIntValue] + type: array + items: + type: integer + minimum: 0 + default: + description: list of column index to be treated as categorical features and + will not be binned; only one of {category_col, category_idx} should be specifiednote + that metrics will be computed over categorical features if this param is + specified + use_anonymous: + type: bool + default: false + optional: true + description: bool, whether interpret `bin_col` & `category_col` as anonymous + column names + type_meta: + title: bool + type: boolean + default: false + description: bool, whether compute host's metrics or not + transform_method: + type: type + default: + optional: true + description: str, values to which binned data will be transformed, select from + {'woe', 'bin_idx'}; note that host will not transform features to woe values + regardless of setting + type_meta: + title: type + type: string + skip_metrics: + type: bool + default: false + optional: true + description: bool, whether compute host's metrics or not + type_meta: + title: bool + type: boolean + default: false + description: bool, whether compute host's metrics or not + local_only: + type: bool + default: false + optional: true + description: bool, whether compute host's metrics or not + type_meta: + title: bool + type: boolean + default: false + description: bool, whether compute host's metrics or not + relative_error: + type: ConstrainedNumberMeta + default: 1e-06 + optional: true + description: float, error rate for quantile + type_meta: + title: ConstrainedNumberMeta + exclusiveMinimum: 0 + maximum: 1 + type: number + adjustment_factor: + type: ConstrainedNumberMeta + default: 0.5 + optional: true + description: float, useful when here is no event or non-event in a bin + type_meta: + title: ConstrainedNumberMeta + exclusiveMinimum: 0 + type: number + he_param: + type: HEParam + default: + kind: paillier + key_length: 1024 + optional: true + description: homomorphic encryption param + type_meta: + title: HEParam + $ref: '#/definitions/abc__HEParam' + definitions: + abc__HEParam: + title: HEParam + type: object + properties: + kind: + title: Kind + type: string + key_length: + title: Key Length + default: 1024 + type: integer + required: + - kind + default: + kind: paillier + key_length: 1024 + description: homomorphic encryption param + input_artifacts: + data: + train_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + model: + input_model: + types: + - json_model + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + output_artifacts: + data: + train_output_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_output_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + model: + output_model: + types: + - json_model + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + metric: + metric: + types: + - json_metric + optional: false + stages: [ ] + roles: [ ] + description: metric, invisible for user + is_multi: false +schema_version: v1 diff --git a/static/hetero_feature_selection.yaml b/static/hetero_feature_selection.yaml new file mode 100644 index 00000000..f37f2f01 --- /dev/null +++ b/static/hetero_feature_selection.yaml @@ -0,0 +1,316 @@ +component: + name: hetero_feature_selection + description: '' + provider: fate + version: 2.0.0-beta + labels: [ ] + roles: + - guest + - host + parameters: + method: + type: typing.List[fate.components.core.params._fields.StringChoice] + default: + - manual + optional: false + description: 'selection method, options: {manual, binning, statistics}' + type_meta: + title: typing.List[fate.components.core.params._fields.StringChoice] + type: array + items: + type: string + default: + - manual + description: 'selection method, options: {manual, binning, statistics}' + select_col: + type: typing.List[str] + default: + optional: true + description: list of column names to be selected, if None, all columns will + be considered + type_meta: + title: typing.List[str] + type: array + items: + type: string + default: + description: list of column names to be selected, if None, all columns will + be considered + iv_param: + type: ModelMetaclass + default: + metrics: + - iv + filter_type: + - threshold + threshold: + - 1.0 + take_high: + - true + host_filter_type: + - threshold + host_threshold: + - 1.0 + host_take_high: + - true + select_federated: true + optional: true + description: iv filter param + type_meta: + title: IVFilterParam + type: object + properties: + metrics: + title: Metrics + default: + - iv + type: array + items: + type: string + filter_type: + title: Filter Type + default: + - threshold + type: array + items: + type: string + threshold: + title: Threshold + default: + - 1.0 + type: array + items: + anyOf: + - type: number + minimum: 0.0 + maximum: 1.0 + - type: integer + minimum: 1 + take_high: + title: Take High + default: + - true + type: array + items: + type: boolean + host_filter_type: + title: Host Filter Type + default: + - threshold + type: array + items: + type: string + host_threshold: + title: Host Threshold + default: + - 1.0 + type: array + items: + anyOf: + - type: number + minimum: 0.0 + maximum: 1.0 + - type: integer + minimum: 1 + host_take_high: + title: Host Take High + default: + - true + type: array + items: + type: boolean + select_federated: + title: Select Federated + default: true + type: boolean + statistic_param: + type: ModelMetaclass + default: + metrics: + - mean + filter_type: + - threshold + threshold: + - 1.0 + take_high: + - true + optional: true + description: statistic filter param + type_meta: + title: StatisticFilterParam + type: object + properties: + metrics: + title: Metrics + default: + - mean + type: array + items: + anyOf: + - type: string + - type: string + filter_type: + title: Filter Type + default: + - threshold + type: array + items: + type: string + threshold: + title: Threshold + default: + - 1.0 + type: array + items: + anyOf: + - type: number + minimum: 0.0 + maximum: 1.0 + - type: integer + minimum: 1 + take_high: + title: Take High + default: + - true + type: array + items: + type: boolean + manual_param: + type: ModelMetaclass + default: + keep_col: [ ] + filter_out_col: [ ] + optional: true + description: manual filter param + type_meta: + title: ManualFilterParam + type: object + properties: + keep_col: + title: Keep Col + default: [ ] + type: array + items: + type: string + filter_out_col: + title: Filter Out Col + default: [ ] + type: array + items: + type: string + keep_one: + type: bool + default: true + optional: true + description: whether to keep at least one feature among `select_col` + type_meta: + title: bool + type: boolean + default: false + description: bool, whether interpret `select_col` & `filter_out_col` & `keep_col` + as anonymous column names + use_anonymous: + type: bool + default: false + optional: true + description: bool, whether interpret `select_col` & `filter_out_col` & `keep_col` + as anonymous column names + type_meta: + title: bool + type: boolean + default: false + description: bool, whether interpret `select_col` & `filter_out_col` & `keep_col` + as anonymous column names + input_artifacts: + data: + train_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + model: + input_models: + types: + - json_model + optional: true + stages: + - train + roles: + - guest + - host + description: '' + is_multi: true + input_model: + types: + - json_model + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + output_artifacts: + data: + train_output_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_output_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + model: + train_output_model: + types: + - json_model + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + metric: + metric: + types: + - json_metric + optional: false + stages: [ ] + roles: [ ] + description: metric, invisible for user + is_multi: false +schema_version: v1 diff --git a/static/homo_lr.yaml b/static/homo_lr.yaml new file mode 100644 index 00000000..ef99bb1a --- /dev/null +++ b/static/homo_lr.yaml @@ -0,0 +1,272 @@ +component: + name: homo_lr + description: '' + provider: fate + version: 2.0.0-beta + labels: [] + roles: + - guest + - host + - arbiter + parameters: + learning_rate_scheduler: + type: LRSchedulerParam + default: + method: linear + scheduler_params: + start_factor: 1.0 + optional: true + description: learning rate scheduler, select method from {'step', 'linear', + 'constant'}for list of configurable arguments, refer to torch.optim.lr_scheduler + type_meta: + title: LRSchedulerParam + $ref: '#/definitions/abc__LRSchedulerParam' + definitions: + abc__LRSchedulerParam: + title: LRSchedulerParam + type: object + properties: + method: + title: Method + default: constant + type: string + scheduler_params: + title: Scheduler Params + type: object + default: + method: linear + scheduler_params: + start_factor: 1.0 + description: learning rate scheduler, select method from {'step', 'linear', + 'constant'}for list of configurable arguments, refer to torch.optim.lr_scheduler + epochs: + type: ConstrainedNumberMeta + default: 20 + optional: true + description: max iteration num + type_meta: + title: ConstrainedNumberMeta + exclusiveMinimum: 0 + type: integer + batch_size: + type: ConstrainedNumberMeta + default: 100 + optional: true + description: batch size, value less or equals to 0 means full batch + type_meta: + title: ConstrainedNumberMeta + minimum: -1 + type: integer + optimizer: + type: OptimizerParam + default: + method: sgd + penalty: l2 + alpha: 1.0 + optimizer_params: + lr: 0.01 + weight_decay: 0 + optional: true + description: '' + type_meta: + title: OptimizerParam + $ref: '#/definitions/abc__OptimizerParam' + definitions: + abc__OptimizerParam: + title: OptimizerParam + type: object + properties: + method: + title: Method + default: sgd + type: string + penalty: + title: Penalty + default: l2 + type: string + alpha: + title: Alpha + default: 1.0 + type: number + optimizer_params: + title: Optimizer Params + type: object + required: + - optimizer_params + default: + method: sgd + penalty: l2 + alpha: 1.0 + optimizer_params: + lr: 0.01 + weight_decay: 0 + description: '' + init_param: + type: InitParam + default: + method: zeros + fill_val: 0.0 + fit_intercept: true + optional: true + description: Model param init setting. + type_meta: + title: InitParam + $ref: '#/definitions/abc__InitParam' + definitions: + abc__InitParam: + title: InitParam + type: object + properties: + method: + title: Method + default: zeros + type: string + fill_val: + title: Fill Val + default: 0.0 + anyOf: + - type: integer + - type: number + fit_intercept: + title: Fit Intercept + default: true + type: boolean + default: + method: zeros + fill_val: 0.0 + fit_intercept: true + description: Model param init setting. + threshold: + type: ConstrainedNumberMeta + default: 0.5 + optional: true + description: predict threshold for binary data + type_meta: + title: ConstrainedNumberMeta + minimum: 0.0 + maximum: 1.0 + type: number + ovr: + type: bool + default: false + optional: true + description: predict threshold for binary data + type_meta: + title: bool + type: boolean + default: false + description: predict threshold for binary data + label_num: + type: ConstrainedNumberMeta + default: + optional: true + description: '' + type_meta: + title: ConstrainedNumberMeta + minimum: 2 + type: integer + input_artifacts: + data: + train_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + validate_data: + types: + - dataframe + optional: true + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + model: + train_input_model: + types: + - json_model + optional: true + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + predict_input_model: + types: + - json_model + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + output_artifacts: + data: + train_output_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_output_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + model: + train_output_model: + types: + - json_model + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + metric: + metric: + types: + - json_metric + optional: false + stages: [] + roles: [] + description: metric, invisible for user + is_multi: false +schema_version: v1 + diff --git a/static/homo_nn.yaml b/static/homo_nn.yaml new file mode 100644 index 00000000..8ecc8ce1 --- /dev/null +++ b/static/homo_nn.yaml @@ -0,0 +1,156 @@ +component: + name: homo_nn + description: '' + provider: fate + version: 2.0.0-beta + labels: [] + roles: + - guest + - host + - arbiter + parameters: + runner_module: + type: str + default: default_runner + optional: true + description: name of your runner script + type_meta: + title: str + type: string + default: + description: path to your runner script folder + runner_class: + type: str + default: DefaultRunner + optional: true + description: class name of your runner class + type_meta: + title: str + type: string + default: + description: path to your runner script folder + runner_conf: + type: dict + default: {} + optional: true + description: the parameter dict of the NN runner class + type_meta: + title: dict + type: object + default: {} + description: the parameter dict of the NN runner class + source: + type: str + default: + optional: true + description: path to your runner script folder + type_meta: + title: str + type: string + default: + description: path to your runner script folder + input_artifacts: + data: + train_data: + types: + - dataframe + optional: false + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + validate_data: + types: + - dataframe + optional: true + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + test_data: + types: + - dataframe + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + model: + train_model_input: + types: + - model_directory + optional: true + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + predict_model_input: + types: + - model_directory + optional: false + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + output_artifacts: + data: + train_data_output: + types: + - dataframe + optional: true + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + predict_data_output: + types: + - dataframe + optional: true + stages: + - predict + roles: + - guest + - host + description: '' + is_multi: false + model: + train_model_output: + types: + - model_directory + optional: true + stages: + - train + roles: + - guest + - host + description: '' + is_multi: false + metric: + metric: + types: + - json_metric + optional: false + stages: [] + roles: [] + description: metric, invisible for user + is_multi: false +schema_version: v1 + diff --git a/static/psi.yaml b/static/psi.yaml new file mode 100644 index 00000000..7f93b1b8 --- /dev/null +++ b/static/psi.yaml @@ -0,0 +1,71 @@ +component: + name: psi + description: '' + provider: fate + version: 2.0.0-beta + labels: [] + roles: + - guest + - host + parameters: + protocol: + type: str + default: ecdh_psi + optional: true + description: '' + type_meta: + title: str + type: string + default: curve25519 + description: '' + curve_type: + type: str + default: curve25519 + optional: true + description: '' + type_meta: + title: str + type: string + default: curve25519 + description: '' + input_artifacts: + data: + input_data: + types: + - dataframe + optional: false + stages: + - default + roles: + - guest + - host + description: '' + is_multi: false + model: {} + output_artifacts: + data: + output_data: + types: + - dataframe + optional: false + stages: + - default + roles: + - guest + - host + description: '' + is_multi: false + model: {} + metric: + metric: + types: + - json_metric + optional: false + stages: + - default + roles: + - guest + - host + description: metric, invisible for user + is_multi: false +schema_version: v1 \ No newline at end of file diff --git a/static/sample.yaml b/static/sample.yaml new file mode 100644 index 00000000..42489214 --- /dev/null +++ b/static/sample.yaml @@ -0,0 +1,119 @@ +component: + name: sample + description: '' + provider: fate + version: 2.0.0-beta + labels: [ ] + roles: + - guest + - host + parameters: + replace: + type: bool + default: false + optional: true + description: whether allow sampling with replacement, default False + type_meta: + title: bool + type: boolean + default: true + description: whether guest sync sampled data sids with host, default True + for hetero scenario, should set to False for local and homo scenario + frac: + type: typing.Union[fate.components.core.params._fields.ConstrainedFloatValue, + typing.Mapping[typing.Union[fate.components.core.params._fields.ConstrainedIntValue, + fate.components.core.params._fields.ConstrainedFloatValue], fate.components.core.params._fields.ConstrainedFloatValue]] + default: + optional: true + description: 'if mode equals to random, it should be a float number greater + than 0,otherwise a dict of pairs like [label_i, sample_rate_i],e.g. {0: 0.5, + 1: 0.8, 2: 0.3}, any label unspecified in dict will not be sampled,default: + 1.0, cannot be used with n' + type_meta: + title: typing.Union[fate.components.core.params._fields.ConstrainedFloatValue, + typing.Mapping[typing.Union[fate.components.core.params._fields.ConstrainedIntValue, + fate.components.core.params._fields.ConstrainedFloatValue], fate.components.core.params._fields.ConstrainedFloatValue]] + anyOf: + - type: number + exclusiveMinimum: 0.0 + - type: object + additionalProperties: + type: number + exclusiveMinimum: 0.0 + default: + description: 'if mode equals to random, it should be a float number greater + than 0,otherwise a dict of pairs like [label_i, sample_rate_i],e.g. {0: + 0.5, 1: 0.8, 2: 0.3}, any label unspecified in dict will not be sampled,default: + 1.0, cannot be used with n' + n: + type: ConstrainedNumberMeta + default: + optional: true + description: 'exact sample size, it should be an int greater than 0, default: + None, cannot be used with frac' + type_meta: + title: ConstrainedNumberMeta + exclusiveMinimum: 0 + type: integer + random_state: + type: ConstrainedNumberMeta + default: + optional: true + description: random state + type_meta: + title: ConstrainedNumberMeta + minimum: 0 + type: integer + hetero_sync: + type: bool + default: true + optional: true + description: whether guest sync sampled data sids with host, default True for + hetero scenario, should set to False for local and homo scenario + type_meta: + title: bool + type: boolean + default: true + description: whether guest sync sampled data sids with host, default True + for hetero scenario, should set to False for local and homo scenario + input_artifacts: + data: + input_data: + types: + - dataframe + optional: false + stages: + - default + roles: + - guest + - host + description: '' + is_multi: false + model: { } + output_artifacts: + data: + output_data: + types: + - dataframe + optional: false + stages: + - default + roles: + - guest + - host + description: '' + is_multi: false + model: { } + metric: + metric: + types: + - json_metric + optional: false + stages: + - default + roles: + - guest + - host + description: metric, invisible for user + is_multi: false +schema_version: v1 diff --git a/static/statistics.yaml b/static/statistics.yaml new file mode 100644 index 00000000..955643f4 --- /dev/null +++ b/static/statistics.yaml @@ -0,0 +1,136 @@ +component: + name: statistics + description: '' + provider: fate + version: 2.0.0-beta + labels: [ ] + roles: + - guest + - host + parameters: + metrics: + type: typing.Union[typing.List[typing.Union[fate.components.core.params._metrics.StatisticMetrics, + fate.components.core.params._metrics.LegalPercentile]], fate.components.core.params._metrics.StatisticMetrics, + fate.components.core.params._metrics.LegalPercentile] + default: + - mean + - std + - min + - max + optional: true + description: metrics to be computed, default ['count', 'mean', 'std', 'min', + 'max'] + type_meta: + title: typing.Union[typing.List[typing.Union[fate.components.core.params._metrics.StatisticMetrics, + fate.components.core.params._metrics.LegalPercentile]], fate.components.core.params._metrics.StatisticMetrics, + fate.components.core.params._metrics.LegalPercentile] + anyOf: + - type: array + items: + anyOf: + - type: string + - type: string + - type: string + - type: string + default: + - mean + - std + - min + - max + description: metrics to be computed, default ['count', 'mean', 'std', 'min', + 'max'] + ddof: + type: ConstrainedNumberMeta + default: 1 + optional: true + description: Delta Degrees of Freedom for std and var, default 1 + type_meta: + title: ConstrainedNumberMeta + minimum: 0 + type: integer + bias: + type: bool + default: true + optional: true + description: If False, the calculations of skewness and kurtosis are corrected + for statistical bias. + type_meta: + title: bool + type: boolean + default: false + description: bool, whether interpret `skip_col` as anonymous column names + relative_error: + type: ConstrainedNumberMeta + default: 0.001 + optional: true + description: float, error rate for quantile + type_meta: + title: ConstrainedNumberMeta + exclusiveMinimum: 0 + maximum: 1 + type: number + skip_col: + type: typing.List[str] + default: + optional: true + description: columns to be skipped, default None; if None, statistics will be + computed over all columns + type_meta: + title: typing.List[str] + type: array + items: + type: string + default: + description: columns to be skipped, default None; if None, statistics will + be computed over all columns + use_anonymous: + type: bool + default: false + optional: true + description: bool, whether interpret `skip_col` as anonymous column names + type_meta: + title: bool + type: boolean + default: false + description: bool, whether interpret `skip_col` as anonymous column names + input_artifacts: + data: + input_data: + types: + - dataframe + optional: false + stages: + - default + roles: + - guest + - host + description: '' + is_multi: false + model: { } + output_artifacts: + data: { } + model: + output_model: + types: + - json_model + optional: false + stages: + - default + roles: + - guest + - host + description: '' + is_multi: false + metric: + metric: + types: + - json_metric + optional: false + stages: + - default + roles: + - guest + - host + description: metric, invisible for user + is_multi: false +schema_version: v1 diff --git a/static/union.yaml b/static/union.yaml new file mode 100644 index 00000000..3a5d340a --- /dev/null +++ b/static/union.yaml @@ -0,0 +1,51 @@ +component: + name: union + description: '' + provider: fate + version: 2.0.0-beta + labels: [ ] + roles: + - guest + - host + parameters: { } + input_artifacts: + data: + input_data_list: + types: + - dataframe + optional: false + stages: + - default + roles: + - guest + - host + description: '' + is_multi: true + model: { } + output_artifacts: + data: + output_data: + types: + - dataframe + optional: false + stages: + - default + roles: + - guest + - host + description: '' + is_multi: false + model: { } + metric: + metric: + types: + - json_metric + optional: false + stages: + - default + roles: + - guest + - host + description: metric, invisible for user + is_multi: false +schema_version: v1