Start adding ES integration test framework. (linkedin#25)

This adds the initial module, as well as some basic annotations and interfaces.

build.gradle

@@ -1,5 +1,6 @@
 buildscript {
   ext.pegasusVersion = '28.3.7'
+  ext.junitJupiterVersion = '5.6.1'
 
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
@@ -30,6 +31,7 @@ project.ext.spec = [
 ]
 
 project.ext.externalDependency = [
+    'assertJ': 'org.assertj:assertj-core:3.11.1',
     'commonsIo': 'commons-io:commons-io:2.4',
     'commonsLang': 'commons-lang:commons-lang:2.6',
     'ebean': 'io.ebean:ebean:11.33.3',
@@ -42,13 +44,18 @@ project.ext.externalDependency = [
     'jacksonDataBind': 'com.fasterxml.jackson.core:jackson-databind:2.9.7',
     'javatuples': 'org.javatuples:javatuples:1.2',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
+    'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
+    'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",
+    'junitJupiterEngine': "org.junit.jupiter:junit-jupiter-engine:$junitJupiterVersion",
     'lombok': 'org.projectlombok:lombok:1.18.12',
     'mockito': 'org.mockito:mockito-core:3.0.0',
     'neo4jHarness': 'org.neo4j.test:neo4j-harness:3.4.11',
     'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:4.0.0',
     'parseqTest': 'com.linkedin.parseq:parseq:3.0.7:test',
     'postgresql': 'org.postgresql:postgresql:42.2.14',
     'reflections': 'org.reflections:reflections:0.9.11',
+    'testContainers': 'org.testcontainers:testcontainers:1.14.3',
+    'testContainersJunit': 'org.testcontainers:junit-jupiter:1.14.3',
     'testng': 'org.testng:testng:6.9.9'
 ]
 

settings.gradle

@@ -6,5 +6,6 @@ include 'dao-impl:elasticsearch-dao'
 include 'dao-impl:neo4j-dao'
 include 'restli-resources'
 include 'testing:core-models-testing'
+include 'testing:elasticsearch-dao-integ-testing'
 include 'testing:test-models'
 include 'validators'

testing/elasticsearch-dao-integ-testing/README.md

@@ -0,0 +1,54 @@
+# Elasticsearch Integration Testing
+
+This module includes a framework to write integration tests against Elasticsearch using GMA, Junit 5, and assertj.
+
+Status: **In Development**.
+
+## Creating a new integration test
+
+1. Add this module as a dependency, as well as an [implementation](#implementations).
+2. Next, create a new Junit 5 test class and add the `@ElasticsearchIntegrationTest` annotation. Adding this annotation
+   should start and stop an Elasticsearch instance for you during your test.
+3. Add fields to your test of type `SearchIndex`, and annotate them with `@SearchIndexType`. Static `SearchIndex` fields
+   will reuse the index across the entire class; instance fields will create a new index per test.
+4. Set the settings / mappings of your index, either with the `@SearchIndexSettings` and `@SearchIndexMappings`
+   annotations, or via methods on `SearchIndex`.
+5. Begin testing by data via `SearchIndex#getWriteDao` and asserting various queries.
+
+```java
+import com.linkedin.metadata.testing.ElasticsearchIntegrationTest;
+import com.linkedin.metadata.testing.SearchIndex;
+import com.linkedin.metadata.testing.annotations.SearchIndexMappings;
+import com.linkedin.metadata.testing.annotations.SearchIndexSettings;
+import com.linkedin.metadata.testing.annotations.SearchIndexType;
+import org.junit.jupiter.api.Test;
+
+@ElasticsearchIntegrationTest // 2
+public class ExampleTest {
+  @SearchIndexType(MySearchDocument.class) // 3
+  @SearchIndexSettings("/settings.json") // 4
+  @SearchIndexMappings("/mappings.json") // 4
+  SearchIndex<MySearchDocument> index; // 3
+
+  @Test
+  public void example() {
+    // 5
+    // given
+    final MySearchDocument mySearchDocument = new MySearchDocument();
+    index.getWriteDao().upsertDocument(mySearchDocument, "myId");
+    index.getRequestContainer().flushAndSettle();
+
+    // TODO finish example once we've decided on how asserts look.
+  }
+}
+```
+
+## Implementations
+
+This module does not ship with code to actually start and stop Elasticsearch. It looks for any class in the
+`com.linkedin.metadata.testing` package annotated with `ElasticsearchContainerFactory.@Implementation`, and implements
+`ElasticsearchContainerFactory`, to use as the implementation to start / stop Elasticsearch.
+
+GMA ships with a default implementation in the `elasticsearch-dao-integ-testing-docker` module, which uses the
+[Testcontainers](http://testcontainers.org) to start / stop Elasticsearch using docker. You are also free to write your
+own implementation.

testing/elasticsearch-dao-integ-testing/build.gradle

@@ -0,0 +1,12 @@
+apply plugin: 'java'
+
+dependencies {
+  compile project(':dao-impl:elasticsearch-dao')
+
+  compile externalDependency.assertJ
+  compile externalDependency.junitJupiterApi
+  compile externalDependency.junitJupiterParams
+
+  testRuntimeOnly externalDependency.junitJupiterEngine
+  testCompile project(':testing:test-models')
+}

testing/elasticsearch-dao-integ-testing/src/main/java/com/linkedin/metadata/testing/ElasticsearchConnection.java

@@ -0,0 +1,30 @@
+package com.linkedin.metadata.testing;
+
+import javax.annotation.Nonnull;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.transport.TransportClient;
+
+
+/**
+ * POJO to hold Elasticsearch client objects.
+ */
+public final class ElasticsearchConnection {
+  private final RestHighLevelClient _restHighLevelClient;
+  private final TransportClient _transportClient;
+
+  public ElasticsearchConnection(@Nonnull RestHighLevelClient restHighLevelClient,
+      @Nonnull TransportClient transportClient) {
+    _restHighLevelClient = restHighLevelClient;
+    _transportClient = transportClient;
+  }
+
+  @Nonnull
+  public RestHighLevelClient getRestHighLevelClient() {
+    return _restHighLevelClient;
+  }
+
+  @Nonnull
+  public TransportClient getTransportClient() {
+    return _transportClient;
+  }
+}

testing/elasticsearch-dao-integ-testing/src/main/java/com/linkedin/metadata/testing/ElasticsearchContainerFactory.java

@@ -0,0 +1,25 @@
+package com.linkedin.metadata.testing;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import javax.annotation.Nonnull;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+
+/**
+ * Factory which can start and stop an Elasticsearch instance.
+ */
+public interface ElasticsearchContainerFactory extends ExtensionContext.Store.CloseableResource {
+  @Target(ElementType.TYPE)
+  @Retention(RetentionPolicy.RUNTIME)
+  @interface Implementation {
+  }
+
+  /**
+   * Starts an Elasticsearch instance for testing and returns clients connected to it.
+   */
+  @Nonnull
+  ElasticsearchConnection start() throws Exception;
+}

testing/elasticsearch-dao-integ-testing/src/main/java/com/linkedin/metadata/testing/ElasticsearchIntegrationTest.java

@@ -0,0 +1,63 @@
+package com.linkedin.metadata.testing;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+
+/**
+ * Junit 5 annotation to indicate that this test requires an instance of Elasticsearch to run against.
+ *
+ * <p>Test classes that have this annotation should also contain one or more public {@link SearchIndex} fields. These
+ * can be static or instance variables to control the test life cycle of the index. The extension will populate these
+ * fields for you.
+ *
+ * <p>The {@link ElasticsearchContainerFactory} implementation, which starts and stops the Elasticsearch instance, is
+ * loaded via reflection. A class marked with {@link
+ * com.linkedin.metadata.testing.ElasticsearchContainerFactory.Implementation} within the {@code
+ * com.linkedin.metadata.testing} namespace will be used. See the {@code elasticsearch-dao-integ-testing-docker} module
+ * for a good default implementation that uses the <a href="https://www.testcontainers.org/">Testcontainers</a>
+ * framework.
+ *
+ * <pre>
+ *   {@code
+ * @ElasticsearchIntegrationTest
+ * public class ExampleTest {
+ *   // Index which is created before any test are run, and is cleaned up after all tests are done.
+ *   @SearchIndexType(MySearchDocument.class)
+ *   public static SearchIndex<MySearchDocument> perClassIndex;
+ *
+ *   // Index which is created before each test method and cleaned up after each test method.
+ *   @SearchIndexType(MySearchDocument.class)
+ *   public SearchIndex<MySearchDocument> perMethodIndex;
+ *
+ *   @BeforeEach
+ *   public void setUpIndex() {
+ *     perMethodIndex.setSettingsAndMappings(/* load json file * /);
+ *   }
+ *
+ *   @Test
+ *   public void example() {
+ *     // given
+ *     final BarSearchDocument searchDocument = new BarSearchDocument().setUrn(new BarUrn(42));
+ *
+ *     // when
+ *     _searchIndex.getWriteDao().upsertDocument(searchDocument, "mydoc");
+ *     _searchIndex.getRequestContainer().flushAndSettle();
+ *
+ *     // then
+ *     assertThat(_searchIndex.getRequestContainer()).wroteOnlyDocuments("mydoc");
+ *   }
+ * }
+ *   }
+ * </pre>
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+@ExtendWith(ElasticsearchIntegrationTestExtension.class)
+@Inherited
+public @interface ElasticsearchIntegrationTest {
+}

testing/elasticsearch-dao-integ-testing/src/main/java/com/linkedin/metadata/testing/ElasticsearchIntegrationTestExtension.java

@@ -0,0 +1,37 @@
+package com.linkedin.metadata.testing;
+
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+
+/**
+ * JUnit 5 extension to start an Elasticsearch instance and create indexes for testing with GMA.
+ *
+ * <p>See {@link ElasticsearchIntegrationTest}.
+ */
+final class ElasticsearchIntegrationTestExtension
+    implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback, AfterEachCallback {
+
+  @Override
+  public void afterAll(ExtensionContext context) throws Exception {
+    // TODO
+  }
+
+  @Override
+  public void afterEach(ExtensionContext context) throws Exception {
+    // TODO
+  }
+
+  @Override
+  public void beforeAll(ExtensionContext context) throws Exception {
+    // TODO
+  }
+
+  @Override
+  public void beforeEach(ExtensionContext context) throws Exception {
+    // TODO
+  }
+}

testing/elasticsearch-dao-integ-testing/src/main/java/com/linkedin/metadata/testing/annotations/SearchIndexMappings.java

@@ -0,0 +1,22 @@
+package com.linkedin.metadata.testing.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+
+/**
+ * Indicates the mappings this index should be created with.
+ *
+ * <p>Optional parameter for {@link com.linkedin.metadata.testing.SearchIndex}es in tests. Can be set directly on the
+ * index after creation with {@link com.linkedin.metadata.testing.SearchIndex#setMappings(String)}.
+ */
+@Target(ElementType.FIELD)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SearchIndexMappings {
+  /**
+   * The JSON resource file to load Elasticsearch mappings from.
+   */
+  String value();
+}

testing/elasticsearch-dao-integ-testing/src/main/java/com/linkedin/metadata/testing/annotations/SearchIndexSettings.java

@@ -0,0 +1,22 @@
+package com.linkedin.metadata.testing.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+
+/**
+ * Indicates the settings this index should be created with.
+ *
+ * <p>Optional parameter for {@link com.linkedin.metadata.testing.SearchIndex}es in tests. Can be set directly on the
+ * index after creation with {@link com.linkedin.metadata.testing.SearchIndex#setSettings(String)} (String)}.
+ */
+@Target(ElementType.FIELD)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SearchIndexSettings {
+  /**
+   * The JSON resource file to load Elasticsearch settings from.
+   */
+  String value();
+}

testing/elasticsearch-dao-integ-testing/src/main/java/com/linkedin/metadata/testing/annotations/SearchIndexType.java

@@ -0,0 +1,26 @@
+package com.linkedin.metadata.testing.annotations;
+
+import com.linkedin.data.template.RecordTemplate;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import javax.annotation.Nonnull;
+
+
+/**
+ * Annotates the given {@link com.linkedin.metadata.testing.SearchIndex} field with the document type.
+ *
+ * <p>Required annotation for {@link com.linkedin.metadata.testing.SearchIndex} instances in tests.</p>
+ */
+@Target(ElementType.FIELD)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SearchIndexType {
+  /**
+   * The search document class for this index.
+   *
+   * <p>Used to create an instance of the {@link com.linkedin.metadata.testing.SearchIndex} during testing.
+   */
+  @Nonnull
+  Class<? extends RecordTemplate> value();
+}