feat: add a docker implementation for the ES testing framework. (#48)

Also start publishing these jars, which required some minor javadoc cleanup.

settings.gradle

@@ -8,5 +8,6 @@ include 'dao-impl:neo4j-dao'
 include 'restli-resources'
 include 'testing:core-models-testing'
 include 'testing:elasticsearch-dao-integ-testing'
+include 'testing:elasticsearch-dao-integ-testing-docker'
 include 'testing:test-models'
 include 'validators'

testing/elasticsearch-dao-integ-testing-docker/build.gradle

@@ -0,0 +1,17 @@
+apply plugin: 'java'
+
+apply from: "$rootDir/gradle/java-publishing.gradle"
+
+dependencies {
+  compile project(':testing:elasticsearch-dao-integ-testing')
+
+  compile externalDependency.assertJ
+  compile externalDependency.junitJupiterApi
+  compile externalDependency.junitJupiterParams
+  compile externalDependency.testContainers
+  compile externalDependency.testContainersJunit
+
+  testRuntimeOnly externalDependency.junitJupiterEngine
+
+  testCompile project(':testing:test-models')
+}

testing/elasticsearch-dao-integ-testing-docker/src/main/java/com/linkedin/metadata/testing/ElasticsearchContainerFactoryDockerImpl.java

@@ -0,0 +1,81 @@
+package com.linkedin.metadata.testing;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import javax.annotation.Nonnull;
+import org.apache.http.HttpHost;
+import org.apache.http.impl.nio.reactor.IOReactorConfig;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
+import org.testcontainers.containers.GenericContainer;
+
+
+/**
+ * Uses the TestContainers framework to launch an Elasticsearch instance using docker.
+ */
+@ElasticsearchContainerFactory.Implementation
+public final class ElasticsearchContainerFactoryDockerImpl implements ElasticsearchContainerFactory {
+  private static final String IMAGE_NAME = "docker.elastic.co/elasticsearch/elasticsearch:5.6.8";
+  private static final int HTTP_PORT = 9200;
+  private static final int TRANSPORT_PORT = 9300;
+
+  /**
+   * Simple implementation that has no extra behavior and is just used to help with the generic typing.
+   */
+  private static final class GenericContainerImpl extends GenericContainer<GenericContainerImpl> {
+    public GenericContainerImpl(@Nonnull String dockerImageName) {
+      super(dockerImageName);
+    }
+  }
+
+  private GenericContainerImpl _container;
+
+  @Nonnull
+  private static RestHighLevelClient buildRestClient(@Nonnull GenericContainerImpl gc) {
+    final RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", gc.getMappedPort(HTTP_PORT), "http"))
+        .setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setDefaultIOReactorConfig(
+            IOReactorConfig.custom().setIoThreadCount(1).build()));
+
+    builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.
+        setConnectionRequestTimeout(3000));
+
+    return new RestHighLevelClient(builder.build());
+  }
+
+  @Nonnull
+  private static TransportClient buildTransportClient(@Nonnull GenericContainerImpl gc) throws UnknownHostException {
+    return new PreBuiltTransportClient(
+        Settings.builder().put("cluster.name", "docker-cluster").build()).addTransportAddress(
+        new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), gc.getMappedPort(TRANSPORT_PORT)));
+  }
+
+  @Nonnull
+  @Override
+  public ElasticsearchConnection start() throws Exception {
+    if (_container == null) {
+      _container = new GenericContainerImpl(IMAGE_NAME).withExposedPorts(HTTP_PORT, TRANSPORT_PORT)
+          .withEnv("xpack.security.enabled", "false");
+      _container.start();
+    }
+
+    return new ElasticsearchConnection(buildRestClient(_container), buildTransportClient(_container));
+  }
+
+  @Override
+  public void close() throws Throwable {
+    if (_container == null) {
+      return;
+    }
+
+    try {
+      _container.close();
+    } finally {
+      _container = null;
+    }
+  }
+}

testing/elasticsearch-dao-integ-testing-docker/src/test/java/com/linkedin/metadata/testing/ElasticsearchIntegrationTestTest.java

@@ -0,0 +1,132 @@
+package com.linkedin.metadata.testing;
+
+import com.linkedin.metadata.dao.SearchResult;
+import com.linkedin.metadata.testing.annotations.SearchIndexMappings;
+import com.linkedin.metadata.testing.annotations.SearchIndexSettings;
+import com.linkedin.metadata.testing.annotations.SearchIndexType;
+import com.linkedin.testing.BarSearchDocument;
+import com.linkedin.testing.urn.BarUrn;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.common.settings.Settings;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+
+import static com.linkedin.metadata.testing.asserts.SearchIndexAssert.assertThat;
+import static com.linkedin.metadata.testing.asserts.SearchResultAssert.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+@ElasticsearchIntegrationTest
+public class ElasticsearchIntegrationTestTest {
+  @SearchIndexType(BarSearchDocument.class)
+  public static SearchIndex<BarSearchDocument> classIndex;
+
+  @SearchIndexType(BarSearchDocument.class)
+  @SearchIndexSettings("/settings.json")
+  @SearchIndexMappings("/mappings.json")
+  public SearchIndex<BarSearchDocument> searchIndex;
+
+  @SearchIndexType(BarSearchDocument.class)
+  public SearchIndex<BarSearchDocument> secondSearchIndex;
+
+  private static String _classIndexName;
+  private static String _methodIndexName;
+
+  @Test
+  public void staticIndexInjected() {
+    assertThat(classIndex).isNotNull();
+  }
+
+  @Test
+  public void instanceIndexesInjected() {
+    assertThat(searchIndex).isNotNull();
+    assertThat(secondSearchIndex).isNotNull();
+  }
+
+  @Test
+  public void uniqueIndexesAreMadeForEachVariable() {
+    assertThat(classIndex.getName()).isNotEqualTo(searchIndex.getName());
+    assertThat(searchIndex.getName()).isNotEqualTo(secondSearchIndex.getName());
+  }
+
+  @Test
+  @Order(1)
+  public void saveIndexNames() {
+    // not a real test, values used to test the life cycle later
+    _classIndexName = classIndex.getName();
+    _methodIndexName = searchIndex.getName();
+  }
+
+  @Test
+  @Order(2)
+  public void staticIndexIsSame() {
+    assertThat(_classIndexName).isEqualTo(classIndex.getName());
+  }
+
+  @Test
+  @Order(2)
+  public void instanceIndexIsDifferent() {
+    assertThat(_methodIndexName).isNotEqualTo(searchIndex.getName());
+  }
+
+  @Test
+  @Order(2)
+  public void instanceIsCleanedUpBetweenMethods() {
+    // given
+    final IndicesAdminClient indicesAdminClient = searchIndex.getConnection().getTransportClient().admin().indices();
+
+    // when
+    final boolean exists = indicesAdminClient.prepareExists(_methodIndexName).get().isExists();
+
+    // then
+    assertThat(exists).isFalse();
+  }
+
+  @Test
+  public void canWriteToIndex() throws Exception {
+    // given
+    final BarSearchDocument searchDocument = new BarSearchDocument().setUrn(new BarUrn(42));
+
+    // when
+    searchIndex.getWriteDao().upsertDocument(searchDocument, "mydoc");
+    searchIndex.getRequestContainer().flushAndSettle();
+
+    // then
+    assertThat(searchIndex).bulkRequests().documentIds().containsExactly("mydoc");
+  }
+
+  @Test
+  public void canReadAllFromIndex() throws Exception {
+    // given
+    final BarUrn urn = new BarUrn(42);
+    final BarSearchDocument searchDocument = new BarSearchDocument().setUrn(urn);
+    searchIndex.getWriteDao().upsertDocument(searchDocument, "mydoc");
+    searchIndex.getRequestContainer().flushAndSettle();
+
+    // when
+    final SearchResult<BarSearchDocument> result = searchIndex.createReadAllDocumentsDao().search("", null, null, 0, 1);
+
+    // then
+    assertThat(result).hasNoMoreResults();
+    assertThat(result).hasTotalCount(1);
+    assertThat(result).documents().containsExactly(searchDocument);
+    assertThat(result).urns().containsExactly(urn);
+  }
+
+  @Test
+  public void settingsAndMappingsAnnotation() throws Exception {
+    // when
+    final GetSettingsResponse response = searchIndex.getConnection()
+        .getTransportClient()
+        .admin()
+        .indices()
+        .prepareGetSettings(searchIndex.getName())
+        .get();
+    final Settings settings = response.getIndexToSettings().get(searchIndex.getName());
+    final String actual = settings.get("index.analysis.filter.autocomplete_filter.type");
+
+    // then
+    assertThat(actual).isEqualTo("edge_ngram");
+  }
+}

testing/elasticsearch-dao-integ-testing-docker/src/test/java/com/linkedin/metadata/testing/ExampleTest.java

@@ -0,0 +1,56 @@
+package com.linkedin.metadata.testing;
+
+import com.linkedin.metadata.dao.SearchResult;
+import com.linkedin.metadata.testing.annotations.SearchIndexMappings;
+import com.linkedin.metadata.testing.annotations.SearchIndexSettings;
+import com.linkedin.metadata.testing.annotations.SearchIndexType;
+import com.linkedin.metadata.testing.asserts.SearchResultAssert;
+import com.linkedin.testing.BarSearchDocument;
+import com.linkedin.testing.urn.BarUrn;
+import org.junit.jupiter.api.Test;
+
+import static com.linkedin.metadata.testing.asserts.SearchIndexAssert.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+@ElasticsearchIntegrationTest
+public class ExampleTest {
+  @SearchIndexType(BarSearchDocument.class)
+  @SearchIndexSettings("/settings.json")
+  @SearchIndexMappings("/mappings.json")
+  public SearchIndex<BarSearchDocument> _searchIndex;
+
+  @Test
+  public void canWriteToIndex() throws Exception {
+    // given
+    final BarSearchDocument searchDocument = new BarSearchDocument().setUrn(new BarUrn(42));
+
+    // when
+    _searchIndex.getWriteDao().upsertDocument(searchDocument, "mydoc");
+    _searchIndex.getRequestContainer().flushAndSettle();
+
+    // then
+    assertThat(_searchIndex).bulkRequests().allRequestsSettled();
+    assertThat(_searchIndex).bulkRequests().hadNoErrors();
+    assertThat(_searchIndex).bulkRequests().documentIds().containsExactly("mydoc");
+  }
+
+  @Test
+  public void canReadAllFromIndex() throws Exception {
+    // given
+    final BarUrn urn = new BarUrn(42);
+    final BarSearchDocument searchDocument = new BarSearchDocument().setUrn(urn);
+    _searchIndex.getWriteDao().upsertDocument(searchDocument, "mydoc");
+    _searchIndex.getRequestContainer().flushAndSettle();
+
+    // when
+    final SearchResult<BarSearchDocument> result =
+        _searchIndex.createReadAllDocumentsDao().search("", null, null, 0, 1);
+
+    // then
+    SearchResultAssert.assertThat(result).hasNoMoreResults();
+    SearchResultAssert.assertThat(result).hasTotalCount(1);
+    SearchResultAssert.assertThat(result).documents().containsExactly(searchDocument);
+    SearchResultAssert.assertThat(result).urns().containsExactly(urn);
+  }
+}

testing/elasticsearch-dao-integ-testing-docker/src/test/resources/mappings.json

@@ -0,0 +1,8 @@
+{
+  "properties": {
+    "urn": {
+      "type": "keyword",
+      "normalizer": "custom_normalizer"
+    }
+  }
+}

testing/elasticsearch-dao-integ-testing-docker/src/test/resources/settings.json

@@ -0,0 +1,53 @@
+{
+  "index": {
+    "analysis": {
+      "filter": {
+        "autocomplete_filter": {
+          "type": "edge_ngram",
+          "min_gram": "3",
+          "max_gram": "20"
+        },
+        "custom_delimiter": {
+          "split_on_numerics": "false",
+          "split_on_case_change": "false",
+          "type": "word_delimiter",
+          "preserve_original": "true",
+          "catenate_words": "false"
+        }
+      },
+      "normalizer": {
+        "custom_normalizer": {
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ],
+          "type": "custom"
+        }
+      },
+      "analyzer": {
+        "delimit_edgengram": {
+          "filter": [
+            "lowercase",
+            "custom_delimiter",
+            "autocomplete_filter"
+          ],
+          "tokenizer": "whitespace"
+        },
+        "delimit": {
+          "filter": [
+            "lowercase",
+            "custom_delimiter"
+          ],
+          "tokenizer": "whitespace"
+        },
+        "lowercase_keyword": {
+          "filter": [
+            "lowercase"
+          ],
+          "type": "custom",
+          "tokenizer": "keyword"
+        }
+      }
+    }
+  }
+}

testing/elasticsearch-dao-integ-testing/build.gradle

@@ -1,5 +1,7 @@
 apply plugin: 'java'
 
+apply from: "$rootDir/gradle/java-publishing.gradle"
+
 dependencies {
   compile project(':dao-impl:elasticsearch-dao')