Initial reimplementation of multipart file upload

pom.xml

@@ -44,6 +44,11 @@
                 <artifactId>jackson-databind</artifactId>
                 <version>2.12.4</version>
             </dependency>
+            <dependency>
+                <groupId>com.squareup.okhttp3</groupId>
+                <artifactId>okhttp</artifactId>
+                <version>4.12.0</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -57,10 +62,22 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-security</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.minio</groupId>
+            <artifactId>minio</artifactId>
+            <version>8.5.10</version>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>minio</artifactId>
+            <version>1.19.7</version>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>edu.stanford.protege</groupId>
             <artifactId>webprotege-ipc</artifactId>
-            <version>1.0.1</version>
+            <version>1.0.3</version>
         </dependency>
 
         <dependency>

src/main/java/edu/stanford/protege/webprotege/gateway/FileStorageService.java

@@ -0,0 +1,81 @@
+package edu.stanford.protege.webprotege.gateway;
+
+import io.minio.BucketExistsArgs;
+import io.minio.MakeBucketArgs;
+import io.minio.MinioClient;
+import io.minio.UploadObjectArgs;
+import io.minio.errors.MinioException;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.util.UUID;
+
+/**
+ * Matthew Horridge
+ * Stanford Center for Biomedical Informatics Research
+ * 2024-06-10
+ */
+@Component
+public class FileStorageService {
+
+
+    private static final Logger logger = LoggerFactory.getLogger(FileStorageService.class);
+
+    private final MinioClient minioClient;
+
+    private final MinioProperties minioProperties;
+
+    public FileStorageService(MinioClient minioClient,
+                          MinioProperties minioProperties) {
+        this.minioClient = minioClient;
+        this.minioProperties = minioProperties;
+    }
+
+    public FileSubmissionId storeFile(Path tempFile) {
+        var fileIdentifier = UUID.randomUUID().toString();
+        logger.info("Storing file ({}) in {} bucket with an object id of {}", getFileSizeInMB(tempFile), minioProperties.getBucketName(), fileIdentifier);
+        createBucketIfNecessary();
+        uploadObject(tempFile, fileIdentifier);
+        return new FileSubmissionId(fileIdentifier);
+    }
+
+    private String getFileSizeInMB(Path tempFile) {
+        try {
+            return FileUtils.byteCountToDisplaySize(Files.size(tempFile));
+        } catch (IOException e) {
+            return "";
+        }
+    }
+
+    private void uploadObject(Path tempFile, String fileIdentifier) {
+        try {
+            minioClient.uploadObject(UploadObjectArgs.builder()
+                                                     .bucket(minioProperties.getBucketName())
+                                                     .object(fileIdentifier)
+                                                     .filename(tempFile.toString())
+                                                     .build());
+        } catch (MinioException | NoSuchAlgorithmException |
+                 InvalidKeyException | IOException e) {
+            throw new StorageException(e);
+        }
+    }
+
+    private void createBucketIfNecessary() {
+        try {
+            if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket(minioProperties.getBucketName()).build())) {
+                minioClient.makeBucket(MakeBucketArgs.builder().bucket(minioProperties.getBucketName()).build());
+            }
+        } catch (MinioException | IOException | NoSuchAlgorithmException | IllegalArgumentException |
+                 InvalidKeyException e) {
+            throw new StorageException(e);
+        }
+    }
+
+}

src/main/java/edu/stanford/protege/webprotege/gateway/FileSubmissionId.java

@@ -0,0 +1,18 @@
+package edu.stanford.protege.webprotege.gateway;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public record FileSubmissionId(String value) {
+
+    @JsonCreator
+    public static FileSubmissionId valueOf(String value) {
+        return new FileSubmissionId(value);
+    }
+
+    @JsonValue
+    @Override
+    public String value() {
+        return value;
+    }
+}

src/main/java/edu/stanford/protege/webprotege/gateway/FileUploadController.java

@@ -0,0 +1,40 @@
+package edu.stanford.protege.webprotege.gateway;
+
+import org.slf4j.*;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.*;
+import org.springframework.security.core.annotation.AuthenticationPrincipal;
+import org.springframework.security.oauth2.jwt.Jwt;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.nio.file.Files;
+
+/**
+ * Matthew Horridge
+ * Stanford Center for Biomedical Informatics Research
+ * 2024-06-10
+ */
+@RestController
+public class FileUploadController {
+
+    private final Logger logger = LoggerFactory.getLogger(GatewayController.class);
+
+    private final FileStorageService fileStorageService;
+
+    public FileUploadController(FileStorageService fileStorageService) {
+        this.fileStorageService = fileStorageService;
+    }
+
+    @PostMapping(path = "/files/submit")
+    public FileSubmissionId execute(@RequestParam MultipartFile file,
+                                    @AuthenticationPrincipal Jwt principal) throws java.io.IOException {
+        var userId = principal.getClaimAsString("preferred_username");
+        logger.info("Received a multipart file from {} with a size of {} bytes", userId, file.getSize());
+        var tempFile = Files.createTempFile("webprotege-file-upload", null);
+        file.transferTo(tempFile);
+        return fileStorageService.storeFile(tempFile);
+    }
+}

src/main/java/edu/stanford/protege/webprotege/gateway/MinioProperties.java

@@ -0,0 +1,53 @@
+package edu.stanford.protege.webprotege.gateway;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Matthew Horridge
+ * Stanford Center for Biomedical Informatics Research
+ * 2024-06-10
+ */
+@ConfigurationProperties(prefix = "webprotege.minio")
+public class MinioProperties {
+
+    private String accessKey;
+
+    private String secretKey;
+
+    private String endPoint;
+
+    private String bucketName;
+
+    public void setAccessKey(String accessKey) {
+        this.accessKey = accessKey;
+    }
+
+    public void setSecretKey(String secretKey) {
+        this.secretKey = secretKey;
+    }
+
+    public void setEndPoint(String endPoint) {
+        this.endPoint = endPoint;
+    }
+
+    public String getAccessKey() {
+        return accessKey;
+    }
+
+    public String getSecretKey() {
+        return secretKey;
+    }
+
+    public String getEndPoint() {
+        return endPoint;
+    }
+
+    public void setBucketName(String bucketName) {
+        this.bucketName = bucketName;
+    }
+
+    public String getBucketName() {
+        return bucketName;
+    }
+}

src/main/java/edu/stanford/protege/webprotege/gateway/StorageException.java

@@ -0,0 +1,13 @@
+package edu.stanford.protege.webprotege.gateway;
+
+/**
+ * Matthew Horridge
+ * Stanford Center for Biomedical Informatics Research
+ * 2024-06-10
+ */
+public class StorageException extends RuntimeException {
+
+    public StorageException(Throwable cause) {
+        super(cause);
+    }
+}

src/main/java/edu/stanford/protege/webprotege/gateway/WebprotegeGwtApiGatewayApplication.java

@@ -2,18 +2,14 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import edu.stanford.protege.webprotege.ipc.WebProtegeIpcApplication;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import io.minio.MinioClient;
+import org.slf4j.*;
 import org.springframework.amqp.rabbit.AsyncRabbitTemplate;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.CommandLineRunner;
-import org.springframework.boot.SpringApplication;
+import org.springframework.boot.*;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Lazy;
+import org.springframework.boot.context.properties.*;
+import org.springframework.context.annotation.*;
 
 @EnableConfigurationProperties
 @ConfigurationPropertiesScan
@@ -38,6 +34,15 @@ public void run(String... args) {
 		}
 	}
 
+
+	@Bean
+	MinioClient minioClient(MinioProperties properties) {
+		return MinioClient.builder()
+								   .credentials(properties.getAccessKey(), properties.getSecretKey())
+								   .endpoint(properties.getEndPoint())
+								   .build();
+	}
+
 	@Bean
 	@Lazy
 	RpcRequestProcessor rpcRequestProcessor(ObjectMapper objectMapper,

src/test/java/edu/stanford/protege/webprotege/gateway/FileStorageServiceIntegrationTest.java

@@ -0,0 +1,78 @@
+package edu.stanford.protege.webprotege.gateway;
+
+import io.minio.*;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+
+import java.nio.file.Files;
+
+import static org.assertj.core.api.Assertions.*;
+
+/**
+ * Matthew Horridge
+ * Stanford Center for Biomedical Informatics Research
+ * 2024-06-12
+ */
+
+@ExtendWith(MinioTestExtension.class)
+@ActiveProfiles("test")
+public class FileStorageServiceIntegrationTest {
+
+    private FileStorageService storageService;
+
+    private MinioClient client;
+
+    @BeforeEach
+    void setUp() {
+        var properties = new MinioProperties();
+        properties.setBucketName("webprotege-uploads");
+        properties.setSecretKey("minioadmin");
+        properties.setAccessKey("minioadmin");
+        properties.setEndPoint(System.getProperty("webprotege.minio.endPoint"));
+        client = MinioClient.builder()
+                                   .credentials(properties.getAccessKey(), properties.getSecretKey())
+                                   .endpoint(properties.getEndPoint())
+                                   .build();
+        storageService = new FileStorageService(client, properties);
+    }
+
+    @Test
+    public void testStoreFile() throws Exception {
+        // Setup a temporary file
+        var tempFile = Files.createTempFile("test-file-", ".txt");
+        Files.write(tempFile, "This is a test file".getBytes());
+
+        // Test storing the file
+        var fileSubmissionId = storageService.storeFile(tempFile);
+        assertThat(fileSubmissionId).isNotNull();
+
+        // Cleanup
+        Files.deleteIfExists(tempFile);
+
+        var exists = client.bucketExists(BucketExistsArgs.builder().bucket("webprotege-uploads").build());
+        assertThat(exists).isTrue();
+    }
+
+    @Test
+    public void testStoreFileWhenMinioIsDown() throws Exception {
+        // Simulate Minio down by incorrect port or server address
+        storageService = new FileStorageService(MinioClient.builder()
+                                                       .endpoint("http://localhost:9999")
+                                                       .credentials("minioadmin", "minioadmin")
+                                                       .build(),
+                                            new MinioProperties());
+
+        var tempFile = Files.createTempFile("test-file-", ".txt");
+        Files.write(tempFile, "Content".getBytes());
+
+        // Expect an exception since Minio is down
+        assertThatThrownBy(() -> storageService.storeFile(tempFile))
+                .isInstanceOf(StorageException.class);
+
+        // Cleanup
+        Files.deleteIfExists(tempFile);
+    }
+}