diff --git a/application.template.yml b/application.template.yml index f02609e9a..02fcdbe9a 100644 --- a/application.template.yml +++ b/application.template.yml @@ -47,10 +47,10 @@ storage: root-dir: data # Needed to reallocate and for download (defaults 'logs/audit.log') -#audit.log.path: 'logs/audit.log' +#audit.log.path: logs/audit.log # Needed to download ... this file is reallocated by OPS (defaults 'logs/armadillo.log') -#stdout.log.path: 'logs/armadillo.log' +#stdout.log.path: logs/armadillo.log logging: level: diff --git a/armadillo/src/main/java/org/molgenis/armadillo/controller/InsightController.java b/armadillo/src/main/java/org/molgenis/armadillo/controller/InsightController.java index 1f16f33fb..778bbddbf 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/controller/InsightController.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/controller/InsightController.java @@ -13,6 +13,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; +import java.io.FileInputStream; import java.security.Principal; import java.util.List; import java.util.Map; @@ -20,10 +21,9 @@ import org.molgenis.armadillo.metadata.FileDetails; import org.molgenis.armadillo.metadata.FileInfo; import org.molgenis.armadillo.metadata.InsightService; -import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.Resource; import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; @@ -77,9 +77,14 @@ public List filesList(Principal principal) { }) @GetMapping(path = "files/{file_id}", produces = APPLICATION_JSON_VALUE) @ResponseStatus(OK) - public FileDetails fileDetails(Principal principal, @PathVariable String file_id) { + public FileDetails fileDetails( + Principal principal, + @PathVariable String file_id, + @RequestParam(name = "page_num", required = false, defaultValue = "0") int pageNum, + @RequestParam(name = "page_size", required = false, defaultValue = "1000") int pageSize, + @RequestParam(name = "direction", required = false, defaultValue = "end") String direction) { return auditor.audit( - () -> insightService.fileDetails(file_id), + () -> insightService.fileDetails(file_id, pageNum, pageSize, direction), principal, FILE_DETAILS, Map.of("FILE_ID", file_id)); @@ -100,14 +105,14 @@ public ResponseEntity downloadDetails( } public ResponseEntity createDownloadFile(String file_id) { - FileDetails fileDetails = insightService.downloadFile(file_id); - String data = fileDetails.getContent(); - Resource file = new ByteArrayResource(data.getBytes()); + FileInputStream file = insightService.downloadFile(file_id); + InputStreamResource inputStreamResource = new InputStreamResource(file); HttpHeaders headers = new HttpHeaders(); headers.add( - HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileDetails.getName() + "\""); - - return new ResponseEntity<>(file, headers, HttpStatus.OK); + HttpHeaders.CONTENT_DISPOSITION, + "attachment; filename=\"" + insightService.getDownloadName(file_id) + ".txt\""); + headers.add(HttpHeaders.CONTENT_TYPE, "text/text"); + return new ResponseEntity<>(inputStreamResource, headers, OK); } } diff --git a/armadillo/src/main/java/org/molgenis/armadillo/metadata/FileDetails.java b/armadillo/src/main/java/org/molgenis/armadillo/metadata/FileDetails.java index e041d4372..02acf0c07 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/metadata/FileDetails.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/metadata/FileDetails.java @@ -18,6 +18,10 @@ public abstract class FileDetails { @NotEmpty public abstract String getName(); + @JsonProperty("content_type") + @NotEmpty + public abstract String getContentType(); + @JsonProperty("content") @NotEmpty public abstract String getContent(); @@ -26,12 +30,24 @@ public abstract class FileDetails { @NotEmpty public abstract String getFetched(); + @JsonProperty("page_num") + @NotEmpty + public abstract int getPageNum(); + + @JsonProperty("page_size") + @NotEmpty + public abstract int getPageSize(); + @JsonCreator public static FileDetails create( @JsonProperty("id") String newId, @JsonProperty("name") String newName, + @JsonProperty("content_type") String newContentType, @JsonProperty("content") String newContent, - @JsonProperty("fetched") String newFetched) { - return new AutoValue_FileDetails(newId, newName, newContent, newFetched); + @JsonProperty("fetched") String newFetched, + @JsonProperty("page_num") int newPageNum, + @JsonProperty("page_size") int newPageSize) { + return new AutoValue_FileDetails( + newId, newName, newContentType, newContent, newFetched, newPageNum, newPageSize); } } diff --git a/armadillo/src/main/java/org/molgenis/armadillo/metadata/InsightService.java b/armadillo/src/main/java/org/molgenis/armadillo/metadata/InsightService.java index 8daff1fa6..886008015 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/metadata/InsightService.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/metadata/InsightService.java @@ -1,11 +1,13 @@ package org.molgenis.armadillo.metadata; +import java.io.FileInputStream; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import org.molgenis.armadillo.service.FileService; import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Service; @@ -13,11 +15,7 @@ @PreAuthorize("hasRole('ROLE_SU')") public class InsightService { public static final String AUDIT_FILE = "AUDIT_FILE"; - public static final String AUDIT_FILE_NAME = "armadillo-audit.log"; - public static final String AUDIT_FILE_DISPLAY_NAME = "Audit file"; public static final String LOG_FILE = "LOG_FILE"; - public static final String LOG_FILE_NAME = "armadillo-log.log"; - public static final String LOG_FILE_DISPLAY_NAME = "Log file"; private final FileService fileService; @@ -27,8 +25,13 @@ public InsightService(FileService fileService) { public List filesInfo() { ArrayList list = new ArrayList<>(); - list.add(FileInfo.create(AUDIT_FILE, AUDIT_FILE_DISPLAY_NAME)); - list.add(FileInfo.create(LOG_FILE, LOG_FILE_DISPLAY_NAME)); + list.add( + FileInfo.create( + InsightServiceFiles.AUDIT_FILE.getKey(), + InsightServiceFiles.AUDIT_FILE.getDisplayName())); + list.add( + FileInfo.create( + InsightServiceFiles.LOG_FILE.getKey(), InsightServiceFiles.LOG_FILE.getDisplayName())); return list; } @@ -45,32 +48,106 @@ private String getServerTime() { @Value("${audit.log.path:./logs/audit.log}") private String auditFilePath; - public FileDetails fileDetails(String file_id) { - return switch (file_id) { - case LOG_FILE -> FileDetails.create( - LOG_FILE, - LOG_FILE_DISPLAY_NAME, - this.fileService.readLogFile(logFilePath), - getServerTime()); - case AUDIT_FILE -> FileDetails.create( - AUDIT_FILE, - AUDIT_FILE_DISPLAY_NAME, - this.fileService.readLogFile(auditFilePath), - getServerTime()); - default -> FileDetails.create(file_id, file_id, file_id, getServerTime()); - }; + /** + * Map file_id to injected file paths. + * + * @param file_id one of injected names above. + * @return File path or given file id. + */ + public String getFileName(String file_id) { + InsightServiceFiles c = InsightServiceFiles.getConstantByKey(file_id); + if (!(c == null)) { + return switch (file_id) { + case LOG_FILE -> logFilePath; + case AUDIT_FILE -> auditFilePath; + default -> "Unregistered file name mapping: " + file_id; + }; + } + return file_id; + } + + public String getDownloadName(String file_id) { + String requestTime = getServerTime().replace(" ", "T").replace(":", "").replace("-", ""); + + InsightServiceFiles c = InsightServiceFiles.getConstantByKey(file_id); + if (!(c == null)) { + return c.getDownloadName() + "-" + requestTime; + } + return file_id; + } + + public FileDetails fileDetails(String file_id, int pageNum, int pageSize, String direction) { + InsightServiceFiles insightServiceFiles = InsightServiceFiles.getConstantByKey(file_id); + if (!(insightServiceFiles == null)) { + String filePath = getFileName(file_id); + String content = this.fileService.readLogFileBiz(filePath, pageNum, pageSize, direction); + return FileDetails.create( + insightServiceFiles.getKey(), + insightServiceFiles.getDisplayName(), + insightServiceFiles.getContentType(), + content, + getServerTime() + ": " + fileService.getFileSize(filePath), + pageNum, + pageSize); + } + return FileDetails.create(file_id, file_id, "text/plain", file_id, getServerTime(), -1, -1); } - public FileDetails downloadFile(String file_id) { + public FileInputStream downloadFile(String file_id) { return switch (file_id) { - case LOG_FILE -> FileDetails.create( - LOG_FILE, LOG_FILE_NAME, this.fileService.readLogFile(logFilePath), getServerTime()); - case AUDIT_FILE -> FileDetails.create( - AUDIT_FILE, - AUDIT_FILE_NAME, - this.fileService.readLogFile(auditFilePath), - getServerTime()); - default -> FileDetails.create(file_id, file_id, file_id, getServerTime()); + case LOG_FILE, AUDIT_FILE -> this.fileService.streamLogFile(getFileName(file_id)); + default -> (FileInputStream) FileInputStream.nullInputStream(); }; } } + +enum InsightServiceFiles { + AUDIT_FILE("AUDIT_FILE", MediaType.APPLICATION_NDJSON_VALUE, "Audit file", "armadillo-audit"), + LOG_FILE("LOG_FILE", MediaType.TEXT_PLAIN_VALUE, "Log file", "armadillo-log"); + + private final String key; + private final String contentType; + private final String displayName; + private final String downloadName; + + InsightServiceFiles(String key, String contentType, String displayName, String downloadName) { + this.key = key; + this.contentType = contentType; + this.displayName = displayName; + this.downloadName = downloadName; + } + + public String getKey() { + return key; + } + + public String getContentType() { + return contentType; + } + + public String getDisplayName() { + return displayName; + } + + public String getDownloadName() { + return downloadName; + } + + public static boolean hasKey(String key) { + for (InsightServiceFiles constant : InsightServiceFiles.values()) { + if (constant.getKey().equals(key)) { + return true; + } + } + return false; + } + + public static InsightServiceFiles getConstantByKey(String key) { + for (InsightServiceFiles constant : InsightServiceFiles.values()) { + if (constant.getKey().equals(key)) { + return constant; + } + } + return null; + } +} diff --git a/armadillo/src/main/java/org/molgenis/armadillo/metadata/TextBlockReader.java b/armadillo/src/main/java/org/molgenis/armadillo/metadata/TextBlockReader.java new file mode 100644 index 000000000..c7a5334c3 --- /dev/null +++ b/armadillo/src/main/java/org/molgenis/armadillo/metadata/TextBlockReader.java @@ -0,0 +1,63 @@ +package org.molgenis.armadillo.metadata; + +import java.io.*; + +/** + * Reads a block or frame between 2 '\n' from given position. + * + *

It does this by seeking previous and next new lines. + */ +public class TextBlockReader { + private RandomAccessFile raf; + + public TextBlockReader(String filePath) throws FileNotFoundException { + this.raf = new RandomAccessFile(filePath, "r"); + } + + long[] updatePositions(long startPosition, long endPosition) throws IOException { + long updatedStartPosition = startPosition; + long updatedEndPosition = endPosition; + + // Seek back for first occurrence of '\n' + raf.seek(updatedStartPosition); + while (updatedStartPosition > 0 && raf.readByte() != '\n') { + updatedStartPosition--; + raf.seek(updatedStartPosition); + } + + // Seek forward for first occurrence of '\n' + raf.seek(updatedEndPosition); + while (updatedEndPosition < raf.length() && raf.readByte() != '\n') { + updatedEndPosition++; + raf.seek(updatedEndPosition); + } + + return new long[] {updatedStartPosition, updatedEndPosition}; + } + + public BufferedReader readBlock(long startPosition, long endPosition) throws IOException { + long[] positions = updatePositions(startPosition, endPosition); + long start = positions[0]; + long end = positions[1]; + + InputStream is = + new InputStream() { + long current = start; + + @Override + public int read() throws IOException { + if (current < end) { + raf.seek(current++); + return raf.readByte(); + } else { + return -1; // end of stream + } + } + }; + + // Create a BufferedReader from the InputStream + BufferedReader reader = new BufferedReader(new InputStreamReader(is)); + + return reader; + } +} diff --git a/armadillo/src/main/java/org/molgenis/armadillo/service/FileService.java b/armadillo/src/main/java/org/molgenis/armadillo/service/FileService.java index 9bd055c1f..9ca29fa35 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/service/FileService.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/service/FileService.java @@ -1,6 +1,11 @@ package org.molgenis.armadillo.service; import java.io.*; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Collectors; +import org.molgenis.armadillo.metadata.TextBlockReader; import org.molgenis.armadillo.profile.annotation.ProfileScope; import org.springframework.stereotype.Service; @@ -8,16 +13,120 @@ @ProfileScope public class FileService { - public String readLogFile(String logFilePath) { + public static int pageNumFromDirection(int pageNum, String direction) { + if (direction.equals("end")) { + pageNum = -pageNum - 1; + } + return pageNum; + } + + /** + * Read page of lines from given pageNum and pageSize from direction. + * + *

The pageSize and pageNum for direction asked for can fall out of file lines range. + * + * @param logFilePath file to read from + * @param pageNum page num depends on pageSize + * @param pageSize number of lines to read + * @return lines falling in the asked frame + */ + public String readLogFile(String logFilePath, int pageNum, int pageSize, String direction) { StringBuilder stringBuilder = new StringBuilder(); String line; + pageNum = pageNumFromDirection(pageNum, direction); + try (BufferedReader reader = new BufferedReader(new FileReader(logFilePath))) { + long totalLines = new BufferedReader(new FileReader(logFilePath)).lines().count(); + long startLine; + long endLine; + + if (pageSize == 0) { + // Read the entire file + startLine = 0; + endLine = Long.MAX_VALUE; + } else { + if (pageNum < 0) { + // NOTE: page is negative + startLine = totalLines + (long) pageNum * pageSize; + } else { + startLine = (long) pageNum * pageSize; + } + + endLine = startLine + pageSize; + + // Restrict frame bounds + startLine = Math.max(0, startLine); + endLine = Math.min(endLine, totalLines); + } + + long lineRead = 0; while ((line = reader.readLine()) != null) { - stringBuilder.append(line + "\n"); + if (startLine <= lineRead && lineRead < endLine) { + stringBuilder.append(line).append("\n"); + } + if (endLine < lineRead) { + break; + } + lineRead += 1; } } catch (IOException e) { return "Error reading log file on '" + logFilePath + "'"; } + return stringBuilder.toString(); } + + public String readLogFileBiz(String logFilePath, int pageNum, int pageSize, String direction) { + long fileSize = getFileSize(logFilePath); + + // file does not exist OR trying to read past file size + if (fileSize == -1 || (long) pageNum * pageSize > fileSize) { + return ""; + } + + // From end makes negative pageNum ie -1 means 1 pageSize from end + pageNum = pageNumFromDirection(pageNum, direction); + + try { + TextBlockReader textBlockReader = new TextBlockReader(logFilePath); + long startPosition = (long) pageNum * pageSize + (direction.equals("end") ? fileSize : 0); + long endPosition = startPosition + pageSize; + + if (startPosition > fileSize) return ""; + + if (startPosition < 0) startPosition = 0; + if (endPosition > fileSize) endPosition = fileSize; + + if (endPosition < startPosition) return ""; + + try { + BufferedReader bufferedReader = textBlockReader.readBlock(startPosition, endPosition); + return bufferedReader.lines().collect(Collectors.joining(System.lineSeparator())); + } catch (IOException e) { + e.printStackTrace(); + return ""; + } + } catch (IOException e) { + e.printStackTrace(); + return ""; + } + } + + public FileInputStream streamLogFile(String logFilePath) { + try { + Path path = Path.of(logFilePath); + return new FileInputStream(path.toFile()); + } catch (IOException e) { + e.printStackTrace(); + return (FileInputStream) FileInputStream.nullInputStream(); + } + } + + public long getFileSize(String filePath) { + try (FileChannel fileChannel = FileChannel.open(Paths.get(filePath))) { + return fileChannel.size(); + } catch (IOException e) { + return -1; + } + } } diff --git a/ui/.gitignore b/ui/.gitignore index c3232071e..23e4f458c 100644 --- a/ui/.gitignore +++ b/ui/.gitignore @@ -13,6 +13,10 @@ dist dist-ssr *.local +# Test artifacts +## from expect(wrapper.vm.$el).toMatchSnapshot(); +__snapshots__/ + # Editor directories and files .vscode/* !.vscode/extensions.json diff --git a/ui/README.md b/ui/README.md index 534c36fb5..466c88856 100644 --- a/ui/README.md +++ b/ui/README.md @@ -26,15 +26,23 @@ it here makes is useable in CI (as an exercise for now). ## Test as a UI developer -From the terminal: -- `cd ui/` -- make sure you have `yarn` installed. See below for some caveats. -- `yarn test --watch` +From within the UI directory run: + +```bash +.gradle/yarn/yarn-v1.22.19/bin/yarn test --watch +``` Do your develop stuff and watch your test fail or succeed on the go. That is: - change files in `src/**` - change files in `tests/unit/**` +### Testing references + +- https://vuejs.org/guide/scaling-up/testing.html +- https://jestjs.io/docs/testing-frameworks +- https://alexjover.com/blog/write-the-first-vue-js-component-unit-test-in-jest/ + - https://leanpub.com/testingvuejscomponentswithjest + ## Run the UI locally Make sure Armadillo is running on 8080 as it needs an endpoint to talk with. diff --git a/ui/src/api/api.ts b/ui/src/api/api.ts index d6233a88f..a853f07d3 100644 --- a/ui/src/api/api.ts +++ b/ui/src/api/api.ts @@ -132,9 +132,14 @@ export async function getFiles(): Promise { } export async function getFileDetail( - file_id: string + file_id: string, + page_num: number, + page_size: number, + direction: string ): Promise { - return get(`/insight/files/${file_id}`); + return get( + `/insight/files/${file_id}?page_num=${page_num}&page_size=${page_size}&direction=${direction}` + ); } export async function getPrincipal(): Promise { diff --git a/ui/src/components/RemoteFile.vue b/ui/src/components/RemoteFile.vue index 9791602a6..82531a091 100644 --- a/ui/src/components/RemoteFile.vue +++ b/ui/src/components/RemoteFile.vue @@ -1,3 +1,131 @@ + + - -