From 036effac4589e2fe163133f51ae473ac12420468 Mon Sep 17 00:00:00 2001 From: Matthew Horridge Date: Wed, 12 Jun 2024 12:42:49 -0700 Subject: [PATCH] Initial reimplementation of multipart file upload --- pom.xml | 19 ++++- .../gateway/FileStorageService.java | 81 +++++++++++++++++++ .../webprotege/gateway/FileSubmissionId.java | 18 +++++ .../gateway/FileUploadController.java | 40 +++++++++ .../webprotege/gateway/MinioProperties.java | 53 ++++++++++++ .../webprotege/gateway/StorageException.java | 13 +++ .../WebprotegeGwtApiGatewayApplication.java | 23 +++--- .../FileStorageServiceIntegrationTest.java | 78 ++++++++++++++++++ .../gateway/FileSubmissionIdTest.java | 48 +++++++++++ .../gateway/MinioPropertiesTest.java | 28 +++++++ .../gateway/MinioTestExtension.java | 34 ++++++++ src/test/resources/application.yaml | 5 ++ 12 files changed, 430 insertions(+), 10 deletions(-) create mode 100644 src/main/java/edu/stanford/protege/webprotege/gateway/FileStorageService.java create mode 100644 src/main/java/edu/stanford/protege/webprotege/gateway/FileSubmissionId.java create mode 100644 src/main/java/edu/stanford/protege/webprotege/gateway/FileUploadController.java create mode 100644 src/main/java/edu/stanford/protege/webprotege/gateway/MinioProperties.java create mode 100644 src/main/java/edu/stanford/protege/webprotege/gateway/StorageException.java create mode 100644 src/test/java/edu/stanford/protege/webprotege/gateway/FileStorageServiceIntegrationTest.java create mode 100644 src/test/java/edu/stanford/protege/webprotege/gateway/FileSubmissionIdTest.java create mode 100644 src/test/java/edu/stanford/protege/webprotege/gateway/MinioPropertiesTest.java create mode 100644 src/test/java/edu/stanford/protege/webprotege/gateway/MinioTestExtension.java diff --git a/pom.xml b/pom.xml index e9a582a..8bb94d3 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,11 @@ jackson-databind 2.12.4 + + com.squareup.okhttp3 + okhttp + 4.12.0 + @@ -57,10 +62,22 @@ org.springframework.boot spring-boot-starter-security + + io.minio + minio + 8.5.10 + + + org.testcontainers + minio + 1.19.7 + test + + edu.stanford.protege webprotege-ipc - 1.0.1 + 1.0.3 diff --git a/src/main/java/edu/stanford/protege/webprotege/gateway/FileStorageService.java b/src/main/java/edu/stanford/protege/webprotege/gateway/FileStorageService.java new file mode 100644 index 0000000..ee92be8 --- /dev/null +++ b/src/main/java/edu/stanford/protege/webprotege/gateway/FileStorageService.java @@ -0,0 +1,81 @@ +package edu.stanford.protege.webprotege.gateway; + +import io.minio.BucketExistsArgs; +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; +import io.minio.UploadObjectArgs; +import io.minio.errors.MinioException; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.UUID; + +/** + * Matthew Horridge + * Stanford Center for Biomedical Informatics Research + * 2024-06-10 + */ +@Component +public class FileStorageService { + + + private static final Logger logger = LoggerFactory.getLogger(FileStorageService.class); + + private final MinioClient minioClient; + + private final MinioProperties minioProperties; + + public FileStorageService(MinioClient minioClient, + MinioProperties minioProperties) { + this.minioClient = minioClient; + this.minioProperties = minioProperties; + } + + public FileSubmissionId storeFile(Path tempFile) { + var fileIdentifier = UUID.randomUUID().toString(); + logger.info("Storing file ({}) in {} bucket with an object id of {}", getFileSizeInMB(tempFile), minioProperties.getBucketName(), fileIdentifier); + createBucketIfNecessary(); + uploadObject(tempFile, fileIdentifier); + return new FileSubmissionId(fileIdentifier); + } + + private String getFileSizeInMB(Path tempFile) { + try { + return FileUtils.byteCountToDisplaySize(Files.size(tempFile)); + } catch (IOException e) { + return ""; + } + } + + private void uploadObject(Path tempFile, String fileIdentifier) { + try { + minioClient.uploadObject(UploadObjectArgs.builder() + .bucket(minioProperties.getBucketName()) + .object(fileIdentifier) + .filename(tempFile.toString()) + .build()); + } catch (MinioException | NoSuchAlgorithmException | + InvalidKeyException | IOException e) { + throw new StorageException(e); + } + } + + private void createBucketIfNecessary() { + try { + if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket(minioProperties.getBucketName()).build())) { + minioClient.makeBucket(MakeBucketArgs.builder().bucket(minioProperties.getBucketName()).build()); + } + } catch (MinioException | IOException | NoSuchAlgorithmException | IllegalArgumentException | + InvalidKeyException e) { + throw new StorageException(e); + } + } + +} diff --git a/src/main/java/edu/stanford/protege/webprotege/gateway/FileSubmissionId.java b/src/main/java/edu/stanford/protege/webprotege/gateway/FileSubmissionId.java new file mode 100644 index 0000000..a28c8f9 --- /dev/null +++ b/src/main/java/edu/stanford/protege/webprotege/gateway/FileSubmissionId.java @@ -0,0 +1,18 @@ +package edu.stanford.protege.webprotege.gateway; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +public record FileSubmissionId(String value) { + + @JsonCreator + public static FileSubmissionId valueOf(String value) { + return new FileSubmissionId(value); + } + + @JsonValue + @Override + public String value() { + return value; + } +} diff --git a/src/main/java/edu/stanford/protege/webprotege/gateway/FileUploadController.java b/src/main/java/edu/stanford/protege/webprotege/gateway/FileUploadController.java new file mode 100644 index 0000000..be4a585 --- /dev/null +++ b/src/main/java/edu/stanford/protege/webprotege/gateway/FileUploadController.java @@ -0,0 +1,40 @@ +package edu.stanford.protege.webprotege.gateway; + +import org.slf4j.*; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.*; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.security.oauth2.jwt.Jwt; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.nio.file.Files; + +/** + * Matthew Horridge + * Stanford Center for Biomedical Informatics Research + * 2024-06-10 + */ +@RestController +public class FileUploadController { + + private final Logger logger = LoggerFactory.getLogger(GatewayController.class); + + private final FileStorageService fileStorageService; + + public FileUploadController(FileStorageService fileStorageService) { + this.fileStorageService = fileStorageService; + } + + @PostMapping(path = "/files/submit") + public FileSubmissionId execute(@RequestParam MultipartFile file, + @AuthenticationPrincipal Jwt principal) throws java.io.IOException { + var userId = principal.getClaimAsString("preferred_username"); + logger.info("Received a multipart file from {} with a size of {} bytes", userId, file.getSize()); + var tempFile = Files.createTempFile("webprotege-file-upload", null); + file.transferTo(tempFile); + return fileStorageService.storeFile(tempFile); + } +} diff --git a/src/main/java/edu/stanford/protege/webprotege/gateway/MinioProperties.java b/src/main/java/edu/stanford/protege/webprotege/gateway/MinioProperties.java new file mode 100644 index 0000000..3ed58a3 --- /dev/null +++ b/src/main/java/edu/stanford/protege/webprotege/gateway/MinioProperties.java @@ -0,0 +1,53 @@ +package edu.stanford.protege.webprotege.gateway; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +/** + * Matthew Horridge + * Stanford Center for Biomedical Informatics Research + * 2024-06-10 + */ +@ConfigurationProperties(prefix = "webprotege.minio") +public class MinioProperties { + + private String accessKey; + + private String secretKey; + + private String endPoint; + + private String bucketName; + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public void setEndPoint(String endPoint) { + this.endPoint = endPoint; + } + + public String getAccessKey() { + return accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public String getEndPoint() { + return endPoint; + } + + public void setBucketName(String bucketName) { + this.bucketName = bucketName; + } + + public String getBucketName() { + return bucketName; + } +} diff --git a/src/main/java/edu/stanford/protege/webprotege/gateway/StorageException.java b/src/main/java/edu/stanford/protege/webprotege/gateway/StorageException.java new file mode 100644 index 0000000..5181822 --- /dev/null +++ b/src/main/java/edu/stanford/protege/webprotege/gateway/StorageException.java @@ -0,0 +1,13 @@ +package edu.stanford.protege.webprotege.gateway; + +/** + * Matthew Horridge + * Stanford Center for Biomedical Informatics Research + * 2024-06-10 + */ +public class StorageException extends RuntimeException { + + public StorageException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/edu/stanford/protege/webprotege/gateway/WebprotegeGwtApiGatewayApplication.java b/src/main/java/edu/stanford/protege/webprotege/gateway/WebprotegeGwtApiGatewayApplication.java index 6c91ae6..a7467ba 100644 --- a/src/main/java/edu/stanford/protege/webprotege/gateway/WebprotegeGwtApiGatewayApplication.java +++ b/src/main/java/edu/stanford/protege/webprotege/gateway/WebprotegeGwtApiGatewayApplication.java @@ -2,18 +2,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import edu.stanford.protege.webprotege.ipc.WebProtegeIpcApplication; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import io.minio.MinioClient; +import org.slf4j.*; import org.springframework.amqp.rabbit.AsyncRabbitTemplate; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.SpringApplication; +import org.springframework.boot.*; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.properties.ConfigurationPropertiesScan; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Lazy; +import org.springframework.boot.context.properties.*; +import org.springframework.context.annotation.*; @EnableConfigurationProperties @ConfigurationPropertiesScan @@ -38,6 +34,15 @@ public void run(String... args) { } } + + @Bean + MinioClient minioClient(MinioProperties properties) { + return MinioClient.builder() + .credentials(properties.getAccessKey(), properties.getSecretKey()) + .endpoint(properties.getEndPoint()) + .build(); + } + @Bean @Lazy RpcRequestProcessor rpcRequestProcessor(ObjectMapper objectMapper, diff --git a/src/test/java/edu/stanford/protege/webprotege/gateway/FileStorageServiceIntegrationTest.java b/src/test/java/edu/stanford/protege/webprotege/gateway/FileStorageServiceIntegrationTest.java new file mode 100644 index 0000000..2e3949d --- /dev/null +++ b/src/test/java/edu/stanford/protege/webprotege/gateway/FileStorageServiceIntegrationTest.java @@ -0,0 +1,78 @@ +package edu.stanford.protege.webprotege.gateway; + +import io.minio.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; + +import java.nio.file.Files; + +import static org.assertj.core.api.Assertions.*; + +/** + * Matthew Horridge + * Stanford Center for Biomedical Informatics Research + * 2024-06-12 + */ + +@ExtendWith(MinioTestExtension.class) +@ActiveProfiles("test") +public class FileStorageServiceIntegrationTest { + + private FileStorageService storageService; + + private MinioClient client; + + @BeforeEach + void setUp() { + var properties = new MinioProperties(); + properties.setBucketName("webprotege-uploads"); + properties.setSecretKey("minioadmin"); + properties.setAccessKey("minioadmin"); + properties.setEndPoint(System.getProperty("webprotege.minio.endPoint")); + client = MinioClient.builder() + .credentials(properties.getAccessKey(), properties.getSecretKey()) + .endpoint(properties.getEndPoint()) + .build(); + storageService = new FileStorageService(client, properties); + } + + @Test + public void testStoreFile() throws Exception { + // Setup a temporary file + var tempFile = Files.createTempFile("test-file-", ".txt"); + Files.write(tempFile, "This is a test file".getBytes()); + + // Test storing the file + var fileSubmissionId = storageService.storeFile(tempFile); + assertThat(fileSubmissionId).isNotNull(); + + // Cleanup + Files.deleteIfExists(tempFile); + + var exists = client.bucketExists(BucketExistsArgs.builder().bucket("webprotege-uploads").build()); + assertThat(exists).isTrue(); + } + + @Test + public void testStoreFileWhenMinioIsDown() throws Exception { + // Simulate Minio down by incorrect port or server address + storageService = new FileStorageService(MinioClient.builder() + .endpoint("http://localhost:9999") + .credentials("minioadmin", "minioadmin") + .build(), + new MinioProperties()); + + var tempFile = Files.createTempFile("test-file-", ".txt"); + Files.write(tempFile, "Content".getBytes()); + + // Expect an exception since Minio is down + assertThatThrownBy(() -> storageService.storeFile(tempFile)) + .isInstanceOf(StorageException.class); + + // Cleanup + Files.deleteIfExists(tempFile); + } +} diff --git a/src/test/java/edu/stanford/protege/webprotege/gateway/FileSubmissionIdTest.java b/src/test/java/edu/stanford/protege/webprotege/gateway/FileSubmissionIdTest.java new file mode 100644 index 0000000..e612609 --- /dev/null +++ b/src/test/java/edu/stanford/protege/webprotege/gateway/FileSubmissionIdTest.java @@ -0,0 +1,48 @@ +package edu.stanford.protege.webprotege.gateway; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.*; +import org.springframework.boot.test.json.JacksonTester; +import org.springframework.core.ResolvableType; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Matthew Horridge + * Stanford Center for Biomedical Informatics Research + * 2024-06-12 + */ +public class FileSubmissionIdTest { + + private static final String THE_SUBMITTED_FILE_ID = "TheSubmittedFileId"; + + private JacksonTester tester; + + @BeforeEach + void setUp() { + tester = new JacksonTester<>(FileSubmissionIdTest.class, + ResolvableType.forClass(FileSubmissionId.class), + new ObjectMapper()); + } + + @Test + void shouldSerializeToJson() throws IOException { + var value = new FileSubmissionId(THE_SUBMITTED_FILE_ID); + var content = tester.write(value); + assertThat(content.getJson()).isEqualTo(""" + "TheSubmittedFileId" + """.strip()); + } + + @Test + void shouldDeserializeFromJson() throws IOException { + var json = """ + "TheSubmittedFileId" + """; + var parsed = tester.parse(json); + assertThat(parsed.getObject()).isEqualTo(new FileSubmissionId(THE_SUBMITTED_FILE_ID)); + } +} + diff --git a/src/test/java/edu/stanford/protege/webprotege/gateway/MinioPropertiesTest.java b/src/test/java/edu/stanford/protege/webprotege/gateway/MinioPropertiesTest.java new file mode 100644 index 0000000..abc2cd8 --- /dev/null +++ b/src/test/java/edu/stanford/protege/webprotege/gateway/MinioPropertiesTest.java @@ -0,0 +1,28 @@ +package edu.stanford.protege.webprotege.gateway; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Matthew Horridge + * Stanford Center for Biomedical Informatics Research + * 2024-06-10 + */ +@SpringBootTest(classes = WebprotegeGwtApiGatewayApplication.class) +@Import({MockJwtDecoderConfiguration.class}) +@ExtendWith(IntegrationTestsExtension.class) +public class MinioPropertiesTest { + + @Autowired + private MinioProperties minioProperties; + + @Test + void shouldHaveBucketName() { + assertThat(minioProperties.getBucketName()).isEqualTo("foobucket"); + } +} diff --git a/src/test/java/edu/stanford/protege/webprotege/gateway/MinioTestExtension.java b/src/test/java/edu/stanford/protege/webprotege/gateway/MinioTestExtension.java new file mode 100644 index 0000000..1692ddd --- /dev/null +++ b/src/test/java/edu/stanford/protege/webprotege/gateway/MinioTestExtension.java @@ -0,0 +1,34 @@ +package edu.stanford.protege.webprotege.gateway; + +import org.junit.jupiter.api.extension.*; +import org.slf4j.*; +import org.testcontainers.containers.MinIOContainer; + +/** + * Matthew Horridge + * Stanford Center for Biomedical Informatics Research + * 2024-06-12 + */ +public class MinioTestExtension implements BeforeAllCallback, AfterAllCallback { + + private static Logger logger = LoggerFactory.getLogger(MinioTestExtension.class); + + private MinIOContainer container; + + @Override + public void beforeAll(ExtensionContext extensionContext) throws Exception { + container = new MinIOContainer("minio/minio:RELEASE.2024-04-06T05-26-02Z"); + container.start(); + + var mappedHttpPort = container.getMappedPort(9000); + logger.info("MinIO port 9000 is mapped to {}", mappedHttpPort); + System.setProperty("webprotege.minio.endPoint", "http://localhost:" + mappedHttpPort); + } + + @Override + public void afterAll(ExtensionContext extensionContext) throws Exception { + if(container != null) { + container.stop(); + } + } +} diff --git a/src/test/resources/application.yaml b/src/test/resources/application.yaml index aafaa3e..c4d04d2 100644 --- a/src/test/resources/application.yaml +++ b/src/test/resources/application.yaml @@ -26,5 +26,10 @@ webprotege: reply-channel: ${spring.application.name}-replies timeout: 600000 # Ten minutes allowedOrigin: webprotege-local.edu + minio: + bucket-name: foobucket + end-point: https://localhost:9000 + access-key: webprotege + secret-key: webprotege