From 055b7057c0bed58e5543a72c864bf80696a6a671 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Thu, 28 Nov 2024 11:31:34 -0500 Subject: [PATCH] [ALS-7539] Test HPDS connection --- .../dataupload/aws/S3StateVerifier.java | 16 ++- .../avillach/dataupload/hpds/HPDSClient.java | 4 + .../hpds/HPDSConnectionVerifier.java | 56 ++++++++++ .../dataupload/hpds/UUIDGenerator.java | 13 +++ .../hpds/hpdsartifactsdonotchange/Query.java | 11 ++ .../dataupload/hpds/HPDSClientTest.java | 17 +++ .../hpds/HPDSConnectionVerifierTest.java | 100 ++++++++++++++++++ 7 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifier.java create mode 100644 uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/UUIDGenerator.java create mode 100644 uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifierTest.java diff --git a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/S3StateVerifier.java b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/S3StateVerifier.java index e116a86..40085eb 100644 --- a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/S3StateVerifier.java +++ b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/S3StateVerifier.java @@ -1,5 +1,6 @@ package edu.harvard.dbmi.avillach.dataupload.aws; +import edu.harvard.dbmi.avillach.dataupload.hpds.HPDSConnectionVerifier; import jakarta.annotation.PostConstruct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,16 +25,23 @@ public class S3StateVerifier { private static final String testTempPrefix = "S3_DAEMON_INIT_TEST"; private static final Logger LOG = LoggerFactory.getLogger(S3StateVerifier.class); - @Autowired - private Map sites; + private final Map sites; + private final AWSClientBuilder clientBuilder; + private final HPDSConnectionVerifier hpdsVerifier; @Autowired - private AWSClientBuilder clientBuilder; + public S3StateVerifier(Map sites, AWSClientBuilder clientBuilder, HPDSConnectionVerifier hpdsVerifier) { + this.sites = sites; + this.clientBuilder = clientBuilder; + this.hpdsVerifier = hpdsVerifier; + } @PostConstruct private void verifyS3Status() { sites.values().forEach(inst -> Thread.ofVirtual().start(() -> asyncVerify(inst))); - + if (!hpdsVerifier.verifyConnection()) { + throw new RuntimeException("Not correctly connected to HPDS"); + } } private void asyncVerify(SiteAWSInfo institution) { diff --git a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java index e31b01c..9bd114d 100644 --- a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java +++ b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java @@ -33,6 +33,10 @@ public class HPDSClient { @Autowired private HttpClientContext context; + public boolean writeTestData(Query query) { + return writeData(query, "test_upload"); + } + public boolean writePhenotypicData(Query query) { return writeData(query, "phenotypic"); } diff --git a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifier.java b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifier.java new file mode 100644 index 0000000..5d1edee --- /dev/null +++ b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifier.java @@ -0,0 +1,56 @@ +package edu.harvard.dbmi.avillach.dataupload.hpds; + +import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.Query; +import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.ResultType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.nio.file.Path; + +@Service +public class HPDSConnectionVerifier { + + private static final Logger log = LoggerFactory.getLogger(HPDSConnectionVerifier.class); + + private final HPDSClient client; + private final Path sharingRoot; + private final UUIDGenerator uuidGenerator; + + @Autowired + public HPDSConnectionVerifier(HPDSClient client, Path sharingRoot, UUIDGenerator uuidGenerator) { + this.client = client; + this.sharingRoot = sharingRoot; + this.uuidGenerator = uuidGenerator; + } + + public boolean verifyConnection() { + log.info("Verifying connection to hpds by asking it to create a file and then verifying that the file exists."); + Query testQuery = new Query(); + testQuery.setPicSureId(uuidGenerator.generate().toString()); + testQuery.setId(testQuery.getPicSureId()); + testQuery.setExpectedResultType(ResultType.COUNT); + log.info("Created test query with UUID {}", testQuery.getPicSureId()); + + log.info("Sending test query to HPDS"); + boolean hpdsResponse = client.writeTestData(testQuery); + if (!hpdsResponse) { + log.info("HPDS returned non 200 exit code. Assuming failure."); + return false; + } + log.info("HPDS reported successfully writing the data. Verifying that the file exists"); + File testData = Path.of(sharingRoot.toString(), testQuery.getPicSureId(), "test_data.txt").toFile(); + + if (testData.exists() && testData.isFile()) { + log.info("File found! Connection to HPDS verified!"); + return testData.delete(); + } + log.info( + "File {} not found. HPDS is running, but the shared directory is probably misconfigured", + testData.getPath() + ); + return false; + } +} diff --git a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/UUIDGenerator.java b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/UUIDGenerator.java new file mode 100644 index 0000000..7a8fe0a --- /dev/null +++ b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/UUIDGenerator.java @@ -0,0 +1,13 @@ +package edu.harvard.dbmi.avillach.dataupload.hpds; + +import org.springframework.stereotype.Service; + +import java.util.UUID; + +@Service +public class UUIDGenerator { + + public UUID generate() { + return UUID.randomUUID(); + } +} diff --git a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Query.java b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Query.java index bd9709b..bb84106 100644 --- a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Query.java +++ b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Query.java @@ -150,5 +150,16 @@ public VariantInfoFilter(VariantInfoFilter filter) { public Map categoryVariantInfoFilters; } + @Override + public boolean equals(Object object) { + if (this == object) return true; + if (object == null || getClass() != object.getClass()) return false; + Query query = (Query) object; + return getExpectedResultType() == query.getExpectedResultType() && Objects.equals(getId(), query.getId()) && Objects.equals(getPicSureId(), query.getPicSureId()); + } + @Override + public int hashCode() { + return Objects.hash(getExpectedResultType(), getId(), getPicSureId()); + } } diff --git a/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClientTest.java b/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClientTest.java index 1a58620..2043931 100644 --- a/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClientTest.java +++ b/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClientTest.java @@ -117,4 +117,21 @@ void shouldNotWriteGenomicData() throws IOException, InterruptedException { Assertions.assertFalse(actual); } + + @Test + void shouldWriteTestData() throws IOException { + Query query = new Query(); + query.setPicSureId("my id"); + + Mockito.when(response.getStatusLine()) + .thenReturn(line); + Mockito.when(line.getStatusCode()) + .thenReturn(200); + Mockito.when(client.execute(Mockito.any(), Mockito.eq(context))) + .thenReturn(response); + + boolean actual = subject.writeTestData(query); + + Assertions.assertTrue(actual); + } } \ No newline at end of file diff --git a/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifierTest.java b/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifierTest.java new file mode 100644 index 0000000..3ec4e4c --- /dev/null +++ b/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifierTest.java @@ -0,0 +1,100 @@ +package edu.harvard.dbmi.avillach.dataupload.hpds; + +import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.Query; +import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.ResultType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentMatcher; +import org.mockito.Mockito; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.*; + +@SpringBootTest +class HPDSConnectionVerifierTest { + + @MockBean + private HPDSClient client; + + @MockBean + private UUIDGenerator generator; + + private final Query query = new Query(); + { + query.setPicSureId("9f1fc383-611b-4c6a-af37-a33c07feea5e"); + query.setId("9f1fc383-611b-4c6a-af37-a33c07feea5e"); + query.setExpectedResultType(ResultType.COUNT); + } + + @Test + void shouldFailWhenHPDS400s(@TempDir Path sharingRoot) { + Mockito.when(client.writeTestData(query)) + .thenReturn(false); + Mockito.when(generator.generate()) + .thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e")); + HPDSConnectionVerifier subject = new HPDSConnectionVerifier(client, sharingRoot, generator); + + boolean result = subject.verifyConnection(); + + Assertions.assertFalse(result); + Mockito.verify(client, Mockito.times(1)) + .writeTestData(query); + } + + @Test + void shouldFailWhenHPDSDoesNotWrite(@TempDir Path sharingRoot) { + Mockito.when(client.writeTestData(query)) + .thenReturn(true); + Mockito.when(generator.generate()) + .thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e")); + HPDSConnectionVerifier subject = new HPDSConnectionVerifier(client, sharingRoot, generator); + + boolean result = subject.verifyConnection(); + + Assertions.assertFalse(result); + Mockito.verify(client, Mockito.times(1)) + .writeTestData(query); + } + + @Test + void shouldFailWhenHPDSMakesADirectory(@TempDir Path sharingRoot) throws IOException { + Files.createDirectory(Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e")); + Files.createDirectory(Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e", "test_data.txt")); + + Mockito.when(client.writeTestData(query)) + .thenReturn(true); + Mockito.when(generator.generate()) + .thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e")); + HPDSConnectionVerifier subject = new HPDSConnectionVerifier(client, sharingRoot, generator); + + Assertions.assertFalse(subject.verifyConnection()); + Mockito.verify(client, Mockito.times(1)) + .writeTestData(query); + } + + @Test + void shouldPass(@TempDir Path sharingRoot) throws IOException { + Files.createDirectory(Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e")); + Files.writeString( + Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e", "test_data.txt"), + "Howdy :)" + ); + + Mockito.when(client.writeTestData(query)) + .thenReturn(true); + Mockito.when(generator.generate()) + .thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e")); + HPDSConnectionVerifier subject = new HPDSConnectionVerifier(client, sharingRoot, generator); + + Assertions.assertTrue(subject.verifyConnection()); + Mockito.verify(client, Mockito.times(1)) + .writeTestData(query); + } +} \ No newline at end of file