Library changes for arrow integration

Signed-off-by: Rishabh Maurya <[email protected]>

gradle/libs.versions.toml

@@ -20,7 +20,7 @@ jettison          = "1.5.4"
 woodstox          = "6.4.0"
 kotlin            = "1.7.10"
 antlr4            = "4.13.1"
-guava             = "32.1.1-jre"
+guava             = "33.2.1-jre"
 protobuf          = "3.25.5"
 jakarta_annotation = "1.3.5"
 google_http_client = "1.44.1"
@@ -79,3 +79,7 @@ resteasy          = "6.2.4.Final"
 # opentelemetry dependencies
 opentelemetry         = "1.41.0"
 opentelemetrysemconv  = "1.27.0-alpha"
+
+# arrow dependencies
+arrow                 = "17.0.0"
+flatbuffers           = "2.0.0"

libs/arrow/build.gradle

@@ -0,0 +1,189 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+dependencies {
+  implementation project(':libs:opensearch-common')
+  implementation project(':libs:opensearch-core')
+  api "org.apache.arrow:arrow-vector:${versions.arrow}"
+  api "org.apache.arrow:arrow-format:${versions.arrow}"
+  api "org.apache.arrow:arrow-memory-core:${versions.arrow}"
+  runtimeOnly "org.apache.arrow:arrow-memory-netty-buffer-patch:${versions.arrow}"
+  runtimeOnly "org.apache.arrow:arrow-memory-netty:${versions.arrow}"
+  runtimeOnly "io.netty:netty-buffer:${versions.netty}"
+  runtimeOnly "io.netty:netty-common:${versions.netty}"
+
+  runtimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}"
+  implementation "org.slf4j:slf4j-api:${versions.slf4j}"
+  implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+  implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
+  implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}"
+  implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}"
+  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+
+  implementation "commons-codec:commons-codec:${versions.commonscodec}"
+
+  // third party audit missing classes and violations
+  implementation "org.slf4j:slf4j-api:${versions.slf4j}"
+  implementation "org.apache.logging.log4j:log4j-api:${versions.log4j}"
+  implementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
+  implementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
+  implementation "org.yaml:snakeyaml:${versions.snakeyaml}"
+  implementation "io.projectreactor.tools:blockhound:1.0.9.RELEASE"
+
+  testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  testImplementation "junit:junit:${versions.junit}"
+  testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
+  testImplementation(project(":test:framework")) {
+    exclude group: 'org.opensearch', module: 'opensearch-arrow'
+  }
+}
+
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}
+
+tasks.named('thirdPartyAudit').configure {
+  ignoreMissingClasses(
+    "com.conversantmedia.util.concurrent.SpinPolicy",
+    "com.fasterxml.jackson.dataformat.xml.JacksonXmlModule",
+    "com.fasterxml.jackson.dataformat.xml.XmlMapper",
+    "com.fasterxml.jackson.dataformat.xml.util.DefaultXmlPrettyPrinter",
+    "com.lmax.disruptor.EventFactory",
+    "com.lmax.disruptor.EventTranslator",
+    "com.lmax.disruptor.EventTranslatorTwoArg",
+    "com.lmax.disruptor.EventTranslatorVararg",
+    "com.lmax.disruptor.ExceptionHandler",
+    "com.lmax.disruptor.LifecycleAware",
+    "com.lmax.disruptor.RingBuffer",
+    "com.lmax.disruptor.Sequence",
+    "com.lmax.disruptor.SequenceBarrier",
+    "com.lmax.disruptor.SequenceReportingEventHandler",
+    "com.lmax.disruptor.TimeoutException",
+    "com.lmax.disruptor.WaitStrategy",
+    "com.lmax.disruptor.dsl.Disruptor",
+    "com.lmax.disruptor.dsl.ProducerType",
+    "com.sun.jna.FunctionMapper",
+    "com.sun.jna.JNIEnv",
+    "com.sun.jna.Library",
+    "com.sun.jna.Memory",
+    "com.sun.jna.Native",
+    "com.sun.jna.NativeLibrary",
+    "com.sun.jna.Platform",
+    "com.sun.jna.Pointer",
+    "com.sun.jna.Structure",
+    "com.sun.jna.platform.win32.Advapi32",
+    "com.sun.jna.platform.win32.Kernel32",
+    "com.sun.jna.platform.win32.WinBase\$SECURITY_ATTRIBUTES",
+    "com.sun.jna.platform.win32.WinDef\$DWORD",
+    "com.sun.jna.platform.win32.WinDef\$LPVOID",
+    "com.sun.jna.platform.win32.WinNT\$HANDLE",
+    "com.sun.jna.platform.win32.WinNT\$SECURITY_DESCRIPTOR",
+    "com.sun.jna.ptr.IntByReference",
+    "com.sun.jna.win32.StdCallLibrary",
+    "com.sun.jna.win32.W32APIOptions",
+    "io.reactivex.internal.schedulers.NonBlockingThread",
+    "javax.jms.Connection",
+    "javax.jms.ConnectionFactory",
+    "javax.jms.Destination",
+    "javax.jms.JMSException",
+    "javax.jms.MapMessage",
+    "javax.jms.Message",
+    "javax.jms.MessageConsumer",
+    "javax.jms.MessageProducer",
+    "javax.jms.Session",
+    "javax.mail.Authenticator",
+    "javax.mail.Message\$RecipientType",
+    "javax.mail.PasswordAuthentication",
+    "javax.mail.Session",
+    "javax.mail.Transport",
+    "javax.mail.internet.InternetAddress",
+    "javax.mail.internet.InternetHeaders",
+    "javax.mail.internet.MimeMessage",
+    "javax.mail.internet.MimeMultipart",
+    "javax.mail.internet.MimeUtility",
+    "org.apache.commons.compress.compressors.CompressorStreamFactory",
+    "org.apache.commons.compress.utils.IOUtils",
+    "org.apache.commons.csv.CSVFormat",
+    "org.apache.commons.csv.QuoteMode",
+    "org.apache.commons.logging.Log",
+    "org.apache.commons.logging.LogFactory",
+    "org.apache.kafka.clients.producer.Producer",
+    "org.apache.kafka.clients.producer.RecordMetadata",
+    "org.apache.kafka.common.serialization.ByteArraySerializer",
+    "org.apache.log4j.Level",
+    "org.apache.log4j.Logger",
+    "org.codehaus.stax2.XMLStreamWriter2",
+    "org.fusesource.jansi.Ansi",
+    "org.fusesource.jansi.AnsiRenderer\$Code",
+    "org.jctools.queues.MpscArrayQueue",
+    "org.osgi.framework.Bundle",
+    "org.osgi.framework.BundleActivator",
+    "org.osgi.framework.BundleContext",
+    "org.osgi.framework.BundleEvent",
+    "org.osgi.framework.BundleReference",
+    "org.osgi.framework.FrameworkUtil",
+    "org.osgi.framework.ServiceReference",
+    "org.osgi.framework.ServiceRegistration",
+    "org.osgi.framework.SynchronousBundleListener",
+    "org.osgi.framework.wiring.BundleWire",
+    "org.osgi.framework.wiring.BundleWiring",
+    "org.slf4j.ext.EventData",
+    "org.zeromq.SocketType",
+    "org.zeromq.ZContext",
+    "org.zeromq.ZMQ",
+    "org.zeromq.ZMQ\$Context",
+    "org.zeromq.ZMQ\$Socket",
+    "org.zeromq.ZMonitor",
+    "org.zeromq.ZMonitor\$Event",
+    "org.zeromq.ZMonitor\$ZEvent",
+    "reactor.core.scheduler.NonBlocking"
+  )
+
+  ignoreViolations(
+    "io.netty.util.internal.PlatformDependent0",
+    "io.netty.util.internal.PlatformDependent0\$1",
+    "io.netty.util.internal.PlatformDependent0\$2",
+    "io.netty.util.internal.PlatformDependent0\$3",
+    "io.netty.util.internal.PlatformDependent0\$4",
+    "io.netty.util.internal.PlatformDependent0\$6",
+    "io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef",
+    "io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef",
+    "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields",
+    "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields",
+    "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields",
+    "io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode",
+    "io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField",
+    "io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField",
+    "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField",
+    "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField",
+    "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField",
+    "io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess",
+    "io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess",
+    "io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess",
+    "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField",
+    "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField",
+    "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField",
+    "org.apache.arrow.memory.ArrowBuf",
+    "org.apache.arrow.memory.util.ByteFunctionHelpers",
+    "org.apache.arrow.memory.util.MemoryUtil",
+    "org.apache.arrow.memory.util.MemoryUtil\$1",
+    "org.apache.arrow.memory.util.hash.MurmurHasher",
+    "org.apache.arrow.memory.util.hash.SimpleHasher",
+    "org.apache.arrow.vector.BaseFixedWidthVector",
+    "org.apache.arrow.vector.BitVectorHelper",
+    "org.apache.arrow.vector.Decimal256Vector",
+    "org.apache.arrow.vector.DecimalVector",
+    "org.apache.arrow.vector.util.DecimalUtility",
+    "org.apache.arrow.vector.util.VectorAppender",
+    "org.apache.logging.log4j.core.util.internal.UnsafeUtil",
+    "org.apache.logging.log4j.core.util.internal.UnsafeUtil\$1"
+  )
+}

libs/arrow/licenses/arrow-format-17.0.0.jar.sha1

@@ -0,0 +1 @@
+5d052f20fd1193840eb59818515e710156c364b2