Merge pull request #668 from molgenis/bug/add-streaming-pager

fix: add pager to insight files
molgenis · Mar 13, 2024 · e73c1fb · e73c1fb
2 parents 866f558 + 03914d5
commit e73c1fb
Show file tree

Hide file tree

Showing 14 changed files with 612 additions and 216 deletions.
diff --git a/application.template.yml b/application.template.yml
@@ -47,10 +47,10 @@ storage:
   root-dir: data
 
 # Needed to reallocate and for download (defaults 'logs/audit.log')
-#audit.log.path: 'logs/audit.log'
+#audit.log.path: logs/audit.log
 
 # Needed to download ... this file is reallocated by OPS (defaults 'logs/armadillo.log')
-#stdout.log.path: 'logs/armadillo.log'
+#stdout.log.path: logs/armadillo.log
 
 logging:
   level:

diff --git a/armadillo/src/main/java/org/molgenis/armadillo/controller/InsightController.java b/armadillo/src/main/java/org/molgenis/armadillo/controller/InsightController.java
@@ -13,17 +13,17 @@
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
+import java.io.FileInputStream;
 import java.security.Principal;
 import java.util.List;
 import java.util.Map;
 import org.molgenis.armadillo.audit.AuditEventPublisher;
 import org.molgenis.armadillo.metadata.FileDetails;
 import org.molgenis.armadillo.metadata.FileInfo;
 import org.molgenis.armadillo.metadata.InsightService;
-import org.springframework.core.io.ByteArrayResource;
+import org.springframework.core.io.InputStreamResource;
 import org.springframework.core.io.Resource;
 import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.*;
@@ -77,9 +77,14 @@ public List<FileInfo> filesList(Principal principal) {
       })
   @GetMapping(path = "files/{file_id}", produces = APPLICATION_JSON_VALUE)
   @ResponseStatus(OK)
-  public FileDetails fileDetails(Principal principal, @PathVariable String file_id) {
+  public FileDetails fileDetails(
+      Principal principal,
+      @PathVariable String file_id,
+      @RequestParam(name = "page_num", required = false, defaultValue = "0") int pageNum,
+      @RequestParam(name = "page_size", required = false, defaultValue = "1000") int pageSize,
+      @RequestParam(name = "direction", required = false, defaultValue = "end") String direction) {
     return auditor.audit(
-        () -> insightService.fileDetails(file_id),
+        () -> insightService.fileDetails(file_id, pageNum, pageSize, direction),
         principal,
         FILE_DETAILS,
         Map.of("FILE_ID", file_id));
@@ -100,14 +105,14 @@ public ResponseEntity<Resource> downloadDetails(
   }
 
   public ResponseEntity<Resource> createDownloadFile(String file_id) {
-    FileDetails fileDetails = insightService.downloadFile(file_id);
-    String data = fileDetails.getContent();
-    Resource file = new ByteArrayResource(data.getBytes());
+    FileInputStream file = insightService.downloadFile(file_id);
+    InputStreamResource inputStreamResource = new InputStreamResource(file);
 
     HttpHeaders headers = new HttpHeaders();
     headers.add(
-        HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileDetails.getName() + "\"");
-
-    return new ResponseEntity<>(file, headers, HttpStatus.OK);
+        HttpHeaders.CONTENT_DISPOSITION,
+        "attachment; filename=\"" + insightService.getDownloadName(file_id) + ".txt\"");
+    headers.add(HttpHeaders.CONTENT_TYPE, "text/text");
+    return new ResponseEntity<>(inputStreamResource, headers, OK);
   }
 }
diff --git a/armadillo/src/main/java/org/molgenis/armadillo/metadata/FileDetails.java b/armadillo/src/main/java/org/molgenis/armadillo/metadata/FileDetails.java
@@ -18,6 +18,10 @@ public abstract class FileDetails {
   @NotEmpty
   public abstract String getName();
 
+  @JsonProperty("content_type")
+  @NotEmpty
+  public abstract String getContentType();
+
   @JsonProperty("content")
   @NotEmpty
   public abstract String getContent();
@@ -26,12 +30,24 @@ public abstract class FileDetails {
   @NotEmpty
   public abstract String getFetched();
 
+  @JsonProperty("page_num")
+  @NotEmpty
+  public abstract int getPageNum();
+
+  @JsonProperty("page_size")
+  @NotEmpty
+  public abstract int getPageSize();
+
   @JsonCreator
   public static FileDetails create(
       @JsonProperty("id") String newId,
       @JsonProperty("name") String newName,
+      @JsonProperty("content_type") String newContentType,
       @JsonProperty("content") String newContent,
-      @JsonProperty("fetched") String newFetched) {
-    return new AutoValue_FileDetails(newId, newName, newContent, newFetched);
+      @JsonProperty("fetched") String newFetched,
+      @JsonProperty("page_num") int newPageNum,
+      @JsonProperty("page_size") int newPageSize) {
+    return new AutoValue_FileDetails(
+        newId, newName, newContentType, newContent, newFetched, newPageNum, newPageSize);
   }
 }
diff --git a/armadillo/src/main/java/org/molgenis/armadillo/metadata/InsightService.java b/armadillo/src/main/java/org/molgenis/armadillo/metadata/InsightService.java
@@ -1,23 +1,21 @@
 package org.molgenis.armadillo.metadata;
 
+import java.io.FileInputStream;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.List;
 import org.molgenis.armadillo.service.FileService;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.MediaType;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Service;
 
 @Service
 @PreAuthorize("hasRole('ROLE_SU')")
 public class InsightService {
   public static final String AUDIT_FILE = "AUDIT_FILE";
-  public static final String AUDIT_FILE_NAME = "armadillo-audit.log";
-  public static final String AUDIT_FILE_DISPLAY_NAME = "Audit file";
   public static final String LOG_FILE = "LOG_FILE";
-  public static final String LOG_FILE_NAME = "armadillo-log.log";
-  public static final String LOG_FILE_DISPLAY_NAME = "Log file";
 
   private final FileService fileService;
 
@@ -27,8 +25,13 @@ public InsightService(FileService fileService) {
 
   public List<FileInfo> filesInfo() {
     ArrayList<FileInfo> list = new ArrayList<>();
-    list.add(FileInfo.create(AUDIT_FILE, AUDIT_FILE_DISPLAY_NAME));
-    list.add(FileInfo.create(LOG_FILE, LOG_FILE_DISPLAY_NAME));
+    list.add(
+        FileInfo.create(
+            InsightServiceFiles.AUDIT_FILE.getKey(),
+            InsightServiceFiles.AUDIT_FILE.getDisplayName()));
+    list.add(
+        FileInfo.create(
+            InsightServiceFiles.LOG_FILE.getKey(), InsightServiceFiles.LOG_FILE.getDisplayName()));
     return list;
   }
 
@@ -45,32 +48,106 @@ private String getServerTime() {
   @Value("${audit.log.path:./logs/audit.log}")
   private String auditFilePath;
 
-  public FileDetails fileDetails(String file_id) {
-    return switch (file_id) {
-      case LOG_FILE -> FileDetails.create(
-          LOG_FILE,
-          LOG_FILE_DISPLAY_NAME,
-          this.fileService.readLogFile(logFilePath),
-          getServerTime());
-      case AUDIT_FILE -> FileDetails.create(
-          AUDIT_FILE,
-          AUDIT_FILE_DISPLAY_NAME,
-          this.fileService.readLogFile(auditFilePath),
-          getServerTime());
-      default -> FileDetails.create(file_id, file_id, file_id, getServerTime());
-    };
+  /**
+   * Map file_id to injected file paths.
+   *
+   * @param file_id one of injected names above.
+   * @return File path or given file id.
+   */
+  public String getFileName(String file_id) {
+    InsightServiceFiles c = InsightServiceFiles.getConstantByKey(file_id);
+    if (!(c == null)) {
+      return switch (file_id) {
+        case LOG_FILE -> logFilePath;
+        case AUDIT_FILE -> auditFilePath;
+        default -> "Unregistered file name mapping: " + file_id;
+      };
+    }
+    return file_id;
+  }
+
+  public String getDownloadName(String file_id) {
+    String requestTime = getServerTime().replace(" ", "T").replace(":", "").replace("-", "");
+
+    InsightServiceFiles c = InsightServiceFiles.getConstantByKey(file_id);
+    if (!(c == null)) {
+      return c.getDownloadName() + "-" + requestTime;
+    }
+    return file_id;
+  }
+
+  public FileDetails fileDetails(String file_id, int pageNum, int pageSize, String direction) {
+    InsightServiceFiles insightServiceFiles = InsightServiceFiles.getConstantByKey(file_id);
+    if (!(insightServiceFiles == null)) {
+      String filePath = getFileName(file_id);
+      String content = this.fileService.readLogFileBiz(filePath, pageNum, pageSize, direction);
+      return FileDetails.create(
+          insightServiceFiles.getKey(),
+          insightServiceFiles.getDisplayName(),
+          insightServiceFiles.getContentType(),
+          content,
+          getServerTime() + ": " + fileService.getFileSize(filePath),
+          pageNum,
+          pageSize);
+    }
+    return FileDetails.create(file_id, file_id, "text/plain", file_id, getServerTime(), -1, -1);
   }
 
-  public FileDetails downloadFile(String file_id) {
+  public FileInputStream downloadFile(String file_id) {
     return switch (file_id) {
-      case LOG_FILE -> FileDetails.create(
-          LOG_FILE, LOG_FILE_NAME, this.fileService.readLogFile(logFilePath), getServerTime());
-      case AUDIT_FILE -> FileDetails.create(
-          AUDIT_FILE,
-          AUDIT_FILE_NAME,
-          this.fileService.readLogFile(auditFilePath),
-          getServerTime());
-      default -> FileDetails.create(file_id, file_id, file_id, getServerTime());
+      case LOG_FILE, AUDIT_FILE -> this.fileService.streamLogFile(getFileName(file_id));
+      default -> (FileInputStream) FileInputStream.nullInputStream();
     };
   }
 }
+
+enum InsightServiceFiles {
+  AUDIT_FILE("AUDIT_FILE", MediaType.APPLICATION_NDJSON_VALUE, "Audit file", "armadillo-audit"),
+  LOG_FILE("LOG_FILE", MediaType.TEXT_PLAIN_VALUE, "Log file", "armadillo-log");
+
+  private final String key;
+  private final String contentType;
+  private final String displayName;
+  private final String downloadName;
+
+  InsightServiceFiles(String key, String contentType, String displayName, String downloadName) {
+    this.key = key;
+    this.contentType = contentType;
+    this.displayName = displayName;
+    this.downloadName = downloadName;
+  }
+
+  public String getKey() {
+    return key;
+  }
+
+  public String getContentType() {
+    return contentType;
+  }
+
+  public String getDisplayName() {
+    return displayName;
+  }
+
+  public String getDownloadName() {
+    return downloadName;
+  }
+
+  public static boolean hasKey(String key) {
+    for (InsightServiceFiles constant : InsightServiceFiles.values()) {
+      if (constant.getKey().equals(key)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public static InsightServiceFiles getConstantByKey(String key) {
+    for (InsightServiceFiles constant : InsightServiceFiles.values()) {
+      if (constant.getKey().equals(key)) {
+        return constant;
+      }
+    }
+    return null;
+  }
+}
diff --git a/armadillo/src/main/java/org/molgenis/armadillo/metadata/TextBlockReader.java b/armadillo/src/main/java/org/molgenis/armadillo/metadata/TextBlockReader.java
@@ -0,0 +1,63 @@
+package org.molgenis.armadillo.metadata;
+
+import java.io.*;
+
+/**
+ * Reads a block or frame between 2 '\n' from given position.
+ *
+ * <p>It does this by seeking previous and next new lines.
+ */
+public class TextBlockReader {
+  private RandomAccessFile raf;
+
+  public TextBlockReader(String filePath) throws FileNotFoundException {
+    this.raf = new RandomAccessFile(filePath, "r");
+  }
+
+  long[] updatePositions(long startPosition, long endPosition) throws IOException {
+    long updatedStartPosition = startPosition;
+    long updatedEndPosition = endPosition;
+
+    // Seek back for first occurrence of '\n'
+    raf.seek(updatedStartPosition);
+    while (updatedStartPosition > 0 && raf.readByte() != '\n') {
+      updatedStartPosition--;
+      raf.seek(updatedStartPosition);
+    }
+
+    // Seek forward for first occurrence of '\n'
+    raf.seek(updatedEndPosition);
+    while (updatedEndPosition < raf.length() && raf.readByte() != '\n') {
+      updatedEndPosition++;
+      raf.seek(updatedEndPosition);
+    }
+
+    return new long[] {updatedStartPosition, updatedEndPosition};
+  }
+
+  public BufferedReader readBlock(long startPosition, long endPosition) throws IOException {
+    long[] positions = updatePositions(startPosition, endPosition);
+    long start = positions[0];
+    long end = positions[1];
+
+    InputStream is =
+        new InputStream() {
+          long current = start;
+
+          @Override
+          public int read() throws IOException {
+            if (current < end) {
+              raf.seek(current++);
+              return raf.readByte();
+            } else {
+              return -1; // end of stream
+            }
+          }
+        };
+
+    // Create a BufferedReader from the InputStream
+    BufferedReader reader = new BufferedReader(new InputStreamReader(is));
+
+    return reader;
+  }
+}