Skip to content

Commit

Permalink
Merge pull request #48 from FederatedAI/develop-1.4.1
Browse files Browse the repository at this point in the history 
Develop 1.4.1
  • Loading branch information
@mgqa34
mgqa34 authored Jun 11, 2020
2 parents 3247082 + ce700d1 commit f8ef200
Show file tree
Hide file tree
Showing 16 changed files with 360 additions and 19 deletions.
7 changes: 7 additions & 0 deletions RELEASE.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
# Release 1.4.1

#### Major Features and Improvements
- support PSI display in evaluation component
- Support confusion matrix display
- update logs interface
- Firefox basic compatibility
# Release 1.4.0

#### Major Features and Improvements
Expand Down
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>fateboard</groupId>
<artifactId>fateboard</artifactId>
<version>1.4.0</version>
<version>1.4.1</version>

<parent>
<groupId>org.springframework.boot</groupId>
Expand Down Expand Up @@ -90,7 +90,7 @@
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.58</version>
<version>1.2.70</version>
<exclusions>
<exclusion>
<artifactId>fastjson</artifactId>
Expand Down
95 changes: 87 additions & 8 deletions src/main/java/com/webank/ai/fate/board/controller/LogController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.webank.ai.fate.board.global.ErrorCode;
import com.webank.ai.fate.board.global.ResponseResult;
import com.webank.ai.fate.board.log.LogFileService;
import com.webank.ai.fate.board.pojo.FuzzyLogQO;
import com.webank.ai.fate.board.pojo.SshInfo;
import com.webank.ai.fate.board.ssh.SshService;
import com.webank.ai.fate.board.utils.GetSystemInfo;
Expand All @@ -28,11 +29,9 @@
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.*;

import javax.xml.ws.Response;
import java.io.*;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -75,7 +74,6 @@ public ResponseResult queryLogWithSizeSSH(@PathVariable String componentId,

}


@RequestMapping(value = "/queryLogSize/{jobId}/{role}/{partyId}/{componentId}/{type}", method = RequestMethod.GET)
@ResponseBody
public ResponseResult queryLogSize(@PathVariable String componentId,
Expand Down Expand Up @@ -145,7 +143,7 @@ List<Map> queryLog(String componentId, String jobId, String type, String role, S
if (begin > end || begin <= 0) {
throw new Exception();
}
String[] cmd = {"sh", "-c", "tail -n +" + begin + " " + filePath + " | head -n " + (end - begin)};
String[] cmd = {"sh", "-c", "tail -n +" + begin + " " + filePath + " | head -n " + (end - begin + 1)};
Process process = Runtime.getRuntime().exec(cmd);
InputStream inputStream = process.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
Expand All @@ -163,11 +161,20 @@ List<Map> queryLog(String componentId, String jobId, String type, String role, S
logger.debug("execute cmd {} return count {}", cmd, index);
}
} finally {
if (inputStream != null) {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
if (process != null) {
try {
process.destroyForcibly();
} catch (Exception e) {
e.printStackTrace();
}
}
return result;
Expand All @@ -184,6 +191,62 @@ List<Map> queryLog(String componentId, String jobId, String type, String role, S

}

private List<Map> queryFuzzyLog(String componentId, String jobId, String type, String role, String partyId, String condition, Integer begin, Integer end) throws Exception {
String filePath = logFileService.buildFilePath(jobId, componentId, type, role, partyId);
// Preconditions.checkArgument(filePath != null && 0!=filePath.trim().length());
Preconditions.checkArgument(StringUtils.isNoneEmpty(condition, filePath));
Preconditions.checkArgument(begin != null && end != null);
Preconditions.checkArgument(end > begin && begin > 0);

if (LogFileService.checkFileIsExist(filePath)) {
List<Map> result = Lists.newArrayList();

String[] cmd = {"sh", "-c", "grep -n " + condition + " " + filePath + " | tail -n +" + begin + " | head -n " + (end - begin + 1)};
Process process = Runtime.getRuntime().exec(cmd);
InputStream inputStream = process.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
try {
String content = null;
do {
content = reader.readLine();
if (content != null) {
int i = content.indexOf(":");
String lineNumber = content.substring(0, i);
String lineContent = content.substring(i + 1);
result.add(LogFileService.toLogMap(lineContent, Long.parseLong(lineNumber)));
}
} while (content != null);
logger.info("execute cmd : {} ", (Object) cmd);
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
process.destroyForcibly();
} catch (Exception e) {
e.printStackTrace();
}
}
return result;
} else {
String ip = logFileService.getJobTaskInfo(jobId, componentId, role, partyId).ip;
if (StringUtils.isEmpty(ip)) {
return null;
}
logFileService.checkSshInfo(ip);
List<Map> remoteFuzzyLog = logFileService.getRemoteFuzzyLog(filePath, ip, condition, begin, end);
return remoteFuzzyLog;

}

}

@RequestMapping(value = "/queryLogWithSize/{jobId}/{role}/{partyId}/{componentId}/{type}/{begin}/{end}", method = RequestMethod.GET)
@ResponseBody
Expand All @@ -202,4 +265,20 @@ public ResponseResult queryLogWithSize(@PathVariable String componentId,
return new ResponseResult<>(ErrorCode.SUCCESS, result);
}

@RequestMapping(value = "/queryFuzzyLog", method = RequestMethod.POST)
@ResponseBody
public ResponseResult queryFuzzyLog(@RequestBody FuzzyLogQO fuzzyLogQO) throws Exception {
logger.info("RequestBody: queryFuzzyLog {}", fuzzyLogQO);
List<Map> result = this.queryFuzzyLog(fuzzyLogQO.getComponentId(), fuzzyLogQO.getJobId(), fuzzyLogQO.getType(), fuzzyLogQO.getRole(), fuzzyLogQO.getPartyId(), fuzzyLogQO.getCondition(), fuzzyLogQO.getBegin(), fuzzyLogQO.getEnd());
return new ResponseResult<>(ErrorCode.SUCCESS, result);
}

@RequestMapping(value = "/queryFuzzyLogSize", method = RequestMethod.POST)
@ResponseBody
public ResponseResult queryFuzzyLogSize(@RequestBody FuzzyLogQO fuzzyLogQO) throws Exception {
logger.info("RequestBody: queryFuzzyLogSize {}", fuzzyLogQO);
long size = logFileService.queryFuzzyLogSize(fuzzyLogQO.getComponentId(), fuzzyLogQO.getJobId(), fuzzyLogQO.getType(), fuzzyLogQO.getRole(), fuzzyLogQO.getPartyId(), fuzzyLogQO.getCondition());
return new ResponseResult<>(ErrorCode.SUCCESS, size);
}

}
133 changes: 130 additions & 3 deletions src/main/java/com/webank/ai/fate/board/log/LogFileService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,14 +216,66 @@ public List<Map> getRemoteLogWithFixSize(String jobId, String componentId, Strin

} while (content != null);
} finally {
if (channel != null) {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
channel.disconnect();
} catch (Exception e) {
e.printStackTrace();
}

}
return results;
}

public List<Map> getRemoteFuzzyLog(String filePath, String ip, String condition, Integer begin, Integer end) throws Exception {
List<Map> results = Lists.newArrayList();
// JobTaskInfo jobTaskInfo = this.getJobTaskInfo(jobId, componentId, role, partyId);
SshInfo sshInfo = this.sshService.getSSHInfo(ip);
// String filePath = this.buildFilePath(jobId, componentId, type, role, partyId);
Session session = this.sshService.connect(sshInfo);
String cmd = "grep -n " + condition + " " + filePath + " | tail -n +" + begin + " | head -n " + (end - begin + 1);
Channel channel = this.sshService.executeCmd(session, cmd);
logger.info("cmd:{}", cmd);
InputStream inputStream = channel.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
try {
String content = null;
do {
content = reader.readLine();
if (content != null) {
int i = content.indexOf(":");
String lineNumber = content.substring(0, i);
String lineContent = content.substring(i + 1);
results.add(LogFileService.toLogMap(lineContent, Long.parseLong(lineNumber)));
}
} while (content != null);
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
channel.disconnect();
} catch (Exception e) {
e.printStackTrace();
}
}
return results;
}

public Channel getRemoteLogStream(String jobId, String componentId, String role, String partyId, String cmd) throws Exception {

Expand Down Expand Up @@ -313,7 +365,82 @@ public void afterPropertiesSet() throws Exception {
}
}

;
public long queryFuzzyLogSize(String componentId, String jobId, String type, String role, String partyId, String condition) throws Exception {
String filePath = this.buildFilePath(jobId, componentId, type, role, partyId);
Preconditions.checkArgument(StringUtils.isNoneEmpty(condition, filePath));
long size;
if (LogFileService.checkFileIsExist(filePath)) {
String[] cmd = {"sh", "-c", "grep -c " + condition + " " + filePath};
Process process = Runtime.getRuntime().exec(cmd);
InputStream inputStream = process.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
try {
String content = reader.readLine();
size = Long.parseLong(content);
logger.info("execute cmd : {} ----result : {}", (Object) cmd, content);
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
process.destroyForcibly();
} catch (Exception e) {
e.printStackTrace();
}
}
return size;
} else {
String ip = this.getJobTaskInfo(jobId, componentId, role, partyId).ip;
if (StringUtils.isEmpty(ip)) {
return 0;
}
this.checkSshInfo(ip);
size = this.getRemoteFuzzyLogSize(filePath, ip, condition);
return size;

}
}

private long getRemoteFuzzyLogSize(String filePath, String ip, String condition) throws Exception {
SshInfo sshInfo = this.sshService.getSSHInfo(ip);
Session session = this.sshService.connect(sshInfo);
String cmd = "grep -c " + condition + " " + filePath;
Channel channel = this.sshService.executeCmd(session, cmd);
logger.info("cmd:{}", cmd);
InputStream inputStream = channel.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
long size;
try {
String content = reader.readLine();
size = Long.parseLong(content);
logger.info("execute cmd : {} ----result : {}", (Object) cmd, content);
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
channel.disconnect();
} catch (Exception e) {
e.printStackTrace();
}
}
return size;
}


public static class JobTaskInfo {

Expand Down
Loading

0 comments on commit f8ef200

Please sign in to comment.