diff --git a/settings.gradle b/settings.gradle index ab4429e01..623db8cd6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,5 +8,6 @@ include 'dao-impl:neo4j-dao' include 'restli-resources' include 'testing:core-models-testing' include 'testing:elasticsearch-dao-integ-testing' +include 'testing:elasticsearch-dao-integ-testing-docker' include 'testing:test-models' include 'validators' diff --git a/testing/elasticsearch-dao-integ-testing-docker/build.gradle b/testing/elasticsearch-dao-integ-testing-docker/build.gradle new file mode 100644 index 000000000..b53aa6aaf --- /dev/null +++ b/testing/elasticsearch-dao-integ-testing-docker/build.gradle @@ -0,0 +1,17 @@ +apply plugin: 'java' + +apply from: "$rootDir/gradle/java-publishing.gradle" + +dependencies { + compile project(':testing:elasticsearch-dao-integ-testing') + + compile externalDependency.assertJ + compile externalDependency.junitJupiterApi + compile externalDependency.junitJupiterParams + compile externalDependency.testContainers + compile externalDependency.testContainersJunit + + testRuntimeOnly externalDependency.junitJupiterEngine + + testCompile project(':testing:test-models') +} \ No newline at end of file diff --git a/testing/elasticsearch-dao-integ-testing-docker/src/main/java/com/linkedin/metadata/testing/ElasticsearchContainerFactoryDockerImpl.java b/testing/elasticsearch-dao-integ-testing-docker/src/main/java/com/linkedin/metadata/testing/ElasticsearchContainerFactoryDockerImpl.java new file mode 100644 index 000000000..97eb72f5d --- /dev/null +++ b/testing/elasticsearch-dao-integ-testing-docker/src/main/java/com/linkedin/metadata/testing/ElasticsearchContainerFactoryDockerImpl.java @@ -0,0 +1,81 @@ +package com.linkedin.metadata.testing; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import javax.annotation.Nonnull; +import org.apache.http.HttpHost; +import org.apache.http.impl.nio.reactor.IOReactorConfig; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.transport.client.PreBuiltTransportClient; +import org.testcontainers.containers.GenericContainer; + + +/** + * Uses the TestContainers framework to launch an Elasticsearch instance using docker. + */ +@ElasticsearchContainerFactory.Implementation +public final class ElasticsearchContainerFactoryDockerImpl implements ElasticsearchContainerFactory { + private static final String IMAGE_NAME = "docker.elastic.co/elasticsearch/elasticsearch:5.6.8"; + private static final int HTTP_PORT = 9200; + private static final int TRANSPORT_PORT = 9300; + + /** + * Simple implementation that has no extra behavior and is just used to help with the generic typing. + */ + private static final class GenericContainerImpl extends GenericContainer { + public GenericContainerImpl(@Nonnull String dockerImageName) { + super(dockerImageName); + } + } + + private GenericContainerImpl _container; + + @Nonnull + private static RestHighLevelClient buildRestClient(@Nonnull GenericContainerImpl gc) { + final RestClientBuilder builder = RestClient.builder(new HttpHost("localhost", gc.getMappedPort(HTTP_PORT), "http")) + .setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setDefaultIOReactorConfig( + IOReactorConfig.custom().setIoThreadCount(1).build())); + + builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder. + setConnectionRequestTimeout(3000)); + + return new RestHighLevelClient(builder.build()); + } + + @Nonnull + private static TransportClient buildTransportClient(@Nonnull GenericContainerImpl gc) throws UnknownHostException { + return new PreBuiltTransportClient( + Settings.builder().put("cluster.name", "docker-cluster").build()).addTransportAddress( + new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), gc.getMappedPort(TRANSPORT_PORT))); + } + + @Nonnull + @Override + public ElasticsearchConnection start() throws Exception { + if (_container == null) { + _container = new GenericContainerImpl(IMAGE_NAME).withExposedPorts(HTTP_PORT, TRANSPORT_PORT) + .withEnv("xpack.security.enabled", "false"); + _container.start(); + } + + return new ElasticsearchConnection(buildRestClient(_container), buildTransportClient(_container)); + } + + @Override + public void close() throws Throwable { + if (_container == null) { + return; + } + + try { + _container.close(); + } finally { + _container = null; + } + } +} diff --git a/testing/elasticsearch-dao-integ-testing-docker/src/test/java/com/linkedin/metadata/testing/ElasticsearchIntegrationTestTest.java b/testing/elasticsearch-dao-integ-testing-docker/src/test/java/com/linkedin/metadata/testing/ElasticsearchIntegrationTestTest.java new file mode 100644 index 000000000..277f97b18 --- /dev/null +++ b/testing/elasticsearch-dao-integ-testing-docker/src/test/java/com/linkedin/metadata/testing/ElasticsearchIntegrationTestTest.java @@ -0,0 +1,132 @@ +package com.linkedin.metadata.testing; + +import com.linkedin.metadata.dao.SearchResult; +import com.linkedin.metadata.testing.annotations.SearchIndexMappings; +import com.linkedin.metadata.testing.annotations.SearchIndexSettings; +import com.linkedin.metadata.testing.annotations.SearchIndexType; +import com.linkedin.testing.BarSearchDocument; +import com.linkedin.testing.urn.BarUrn; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.common.settings.Settings; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + +import static com.linkedin.metadata.testing.asserts.SearchIndexAssert.assertThat; +import static com.linkedin.metadata.testing.asserts.SearchResultAssert.assertThat; +import static org.assertj.core.api.Assertions.assertThat; + + +@ElasticsearchIntegrationTest +public class ElasticsearchIntegrationTestTest { + @SearchIndexType(BarSearchDocument.class) + public static SearchIndex classIndex; + + @SearchIndexType(BarSearchDocument.class) + @SearchIndexSettings("/settings.json") + @SearchIndexMappings("/mappings.json") + public SearchIndex searchIndex; + + @SearchIndexType(BarSearchDocument.class) + public SearchIndex secondSearchIndex; + + private static String _classIndexName; + private static String _methodIndexName; + + @Test + public void staticIndexInjected() { + assertThat(classIndex).isNotNull(); + } + + @Test + public void instanceIndexesInjected() { + assertThat(searchIndex).isNotNull(); + assertThat(secondSearchIndex).isNotNull(); + } + + @Test + public void uniqueIndexesAreMadeForEachVariable() { + assertThat(classIndex.getName()).isNotEqualTo(searchIndex.getName()); + assertThat(searchIndex.getName()).isNotEqualTo(secondSearchIndex.getName()); + } + + @Test + @Order(1) + public void saveIndexNames() { + // not a real test, values used to test the life cycle later + _classIndexName = classIndex.getName(); + _methodIndexName = searchIndex.getName(); + } + + @Test + @Order(2) + public void staticIndexIsSame() { + assertThat(_classIndexName).isEqualTo(classIndex.getName()); + } + + @Test + @Order(2) + public void instanceIndexIsDifferent() { + assertThat(_methodIndexName).isNotEqualTo(searchIndex.getName()); + } + + @Test + @Order(2) + public void instanceIsCleanedUpBetweenMethods() { + // given + final IndicesAdminClient indicesAdminClient = searchIndex.getConnection().getTransportClient().admin().indices(); + + // when + final boolean exists = indicesAdminClient.prepareExists(_methodIndexName).get().isExists(); + + // then + assertThat(exists).isFalse(); + } + + @Test + public void canWriteToIndex() throws Exception { + // given + final BarSearchDocument searchDocument = new BarSearchDocument().setUrn(new BarUrn(42)); + + // when + searchIndex.getWriteDao().upsertDocument(searchDocument, "mydoc"); + searchIndex.getRequestContainer().flushAndSettle(); + + // then + assertThat(searchIndex).bulkRequests().documentIds().containsExactly("mydoc"); + } + + @Test + public void canReadAllFromIndex() throws Exception { + // given + final BarUrn urn = new BarUrn(42); + final BarSearchDocument searchDocument = new BarSearchDocument().setUrn(urn); + searchIndex.getWriteDao().upsertDocument(searchDocument, "mydoc"); + searchIndex.getRequestContainer().flushAndSettle(); + + // when + final SearchResult result = searchIndex.createReadAllDocumentsDao().search("", null, null, 0, 1); + + // then + assertThat(result).hasNoMoreResults(); + assertThat(result).hasTotalCount(1); + assertThat(result).documents().containsExactly(searchDocument); + assertThat(result).urns().containsExactly(urn); + } + + @Test + public void settingsAndMappingsAnnotation() throws Exception { + // when + final GetSettingsResponse response = searchIndex.getConnection() + .getTransportClient() + .admin() + .indices() + .prepareGetSettings(searchIndex.getName()) + .get(); + final Settings settings = response.getIndexToSettings().get(searchIndex.getName()); + final String actual = settings.get("index.analysis.filter.autocomplete_filter.type"); + + // then + assertThat(actual).isEqualTo("edge_ngram"); + } +} diff --git a/testing/elasticsearch-dao-integ-testing-docker/src/test/java/com/linkedin/metadata/testing/ExampleTest.java b/testing/elasticsearch-dao-integ-testing-docker/src/test/java/com/linkedin/metadata/testing/ExampleTest.java new file mode 100644 index 000000000..a13021873 --- /dev/null +++ b/testing/elasticsearch-dao-integ-testing-docker/src/test/java/com/linkedin/metadata/testing/ExampleTest.java @@ -0,0 +1,56 @@ +package com.linkedin.metadata.testing; + +import com.linkedin.metadata.dao.SearchResult; +import com.linkedin.metadata.testing.annotations.SearchIndexMappings; +import com.linkedin.metadata.testing.annotations.SearchIndexSettings; +import com.linkedin.metadata.testing.annotations.SearchIndexType; +import com.linkedin.metadata.testing.asserts.SearchResultAssert; +import com.linkedin.testing.BarSearchDocument; +import com.linkedin.testing.urn.BarUrn; +import org.junit.jupiter.api.Test; + +import static com.linkedin.metadata.testing.asserts.SearchIndexAssert.assertThat; +import static org.assertj.core.api.Assertions.assertThat; + + +@ElasticsearchIntegrationTest +public class ExampleTest { + @SearchIndexType(BarSearchDocument.class) + @SearchIndexSettings("/settings.json") + @SearchIndexMappings("/mappings.json") + public SearchIndex _searchIndex; + + @Test + public void canWriteToIndex() throws Exception { + // given + final BarSearchDocument searchDocument = new BarSearchDocument().setUrn(new BarUrn(42)); + + // when + _searchIndex.getWriteDao().upsertDocument(searchDocument, "mydoc"); + _searchIndex.getRequestContainer().flushAndSettle(); + + // then + assertThat(_searchIndex).bulkRequests().allRequestsSettled(); + assertThat(_searchIndex).bulkRequests().hadNoErrors(); + assertThat(_searchIndex).bulkRequests().documentIds().containsExactly("mydoc"); + } + + @Test + public void canReadAllFromIndex() throws Exception { + // given + final BarUrn urn = new BarUrn(42); + final BarSearchDocument searchDocument = new BarSearchDocument().setUrn(urn); + _searchIndex.getWriteDao().upsertDocument(searchDocument, "mydoc"); + _searchIndex.getRequestContainer().flushAndSettle(); + + // when + final SearchResult result = + _searchIndex.createReadAllDocumentsDao().search("", null, null, 0, 1); + + // then + SearchResultAssert.assertThat(result).hasNoMoreResults(); + SearchResultAssert.assertThat(result).hasTotalCount(1); + SearchResultAssert.assertThat(result).documents().containsExactly(searchDocument); + SearchResultAssert.assertThat(result).urns().containsExactly(urn); + } +} diff --git a/testing/elasticsearch-dao-integ-testing-docker/src/test/resources/mappings.json b/testing/elasticsearch-dao-integ-testing-docker/src/test/resources/mappings.json new file mode 100644 index 000000000..92f0710f9 --- /dev/null +++ b/testing/elasticsearch-dao-integ-testing-docker/src/test/resources/mappings.json @@ -0,0 +1,8 @@ +{ + "properties": { + "urn": { + "type": "keyword", + "normalizer": "custom_normalizer" + } + } +} \ No newline at end of file diff --git a/testing/elasticsearch-dao-integ-testing-docker/src/test/resources/settings.json b/testing/elasticsearch-dao-integ-testing-docker/src/test/resources/settings.json new file mode 100644 index 000000000..b4698fbe8 --- /dev/null +++ b/testing/elasticsearch-dao-integ-testing-docker/src/test/resources/settings.json @@ -0,0 +1,53 @@ +{ + "index": { + "analysis": { + "filter": { + "autocomplete_filter": { + "type": "edge_ngram", + "min_gram": "3", + "max_gram": "20" + }, + "custom_delimiter": { + "split_on_numerics": "false", + "split_on_case_change": "false", + "type": "word_delimiter", + "preserve_original": "true", + "catenate_words": "false" + } + }, + "normalizer": { + "custom_normalizer": { + "filter": [ + "lowercase", + "asciifolding" + ], + "type": "custom" + } + }, + "analyzer": { + "delimit_edgengram": { + "filter": [ + "lowercase", + "custom_delimiter", + "autocomplete_filter" + ], + "tokenizer": "whitespace" + }, + "delimit": { + "filter": [ + "lowercase", + "custom_delimiter" + ], + "tokenizer": "whitespace" + }, + "lowercase_keyword": { + "filter": [ + "lowercase" + ], + "type": "custom", + "tokenizer": "keyword" + } + } + } + } +} \ No newline at end of file diff --git a/testing/elasticsearch-dao-integ-testing/build.gradle b/testing/elasticsearch-dao-integ-testing/build.gradle index 5ebbf1f94..f189f549f 100644 --- a/testing/elasticsearch-dao-integ-testing/build.gradle +++ b/testing/elasticsearch-dao-integ-testing/build.gradle @@ -1,5 +1,7 @@ apply plugin: 'java' +apply from: "$rootDir/gradle/java-publishing.gradle" + dependencies { compile project(':dao-impl:elasticsearch-dao') diff --git a/testing/elasticsearch-dao-integ-testing/src/main/java/com/linkedin/metadata/testing/ElasticsearchIntegrationTest.java b/testing/elasticsearch-dao-integ-testing/src/main/java/com/linkedin/metadata/testing/ElasticsearchIntegrationTest.java index 3a4ef8776..e91175061 100644 --- a/testing/elasticsearch-dao-integ-testing/src/main/java/com/linkedin/metadata/testing/ElasticsearchIntegrationTest.java +++ b/testing/elasticsearch-dao-integ-testing/src/main/java/com/linkedin/metadata/testing/ElasticsearchIntegrationTest.java @@ -22,38 +22,7 @@ * for a good default implementation that uses the Testcontainers * framework. * - *
- *   {@code
- * @ElasticsearchIntegrationTest
- * public class ExampleTest {
- *   // Index which is created before any test are run, and is cleaned up after all tests are done.
- *   @SearchIndexType(MySearchDocument.class)
- *   public static SearchIndex perClassIndex;
- *
- *   // Index which is created before each test method and cleaned up after each test method.
- *   @SearchIndexType(MySearchDocument.class)
- *   public SearchIndex perMethodIndex;
- *
- *   @BeforeEach
- *   public void setUpIndex() {
- *     perMethodIndex.setSettingsAndMappings(/* load json file * /);
- *   }
- *
- *   @Test
- *   public void example() {
- *     // given
- *     final BarSearchDocument searchDocument = new BarSearchDocument().setUrn(new BarUrn(42));
- *
- *     // when
- *     _searchIndex.getWriteDao().upsertDocument(searchDocument, "mydoc");
- *     _searchIndex.getRequestContainer().flushAndSettle();
- *
- *     // then
- *     assertThat(_searchIndex.getRequestContainer()).wroteOnlyDocuments("mydoc");
- *   }
- * }
- *   }
- * 
+ *

See the README file in this module for more information. */ @Target(ElementType.TYPE) @Retention(RetentionPolicy.RUNTIME)