-download task support for geoip (#3373) (#3428)

* -download task support for geoip
Signed-off-by: rajeshLovesToCode <[email protected]>

* -download task support for geoip
Signed-off-by: rajeshLovesToCode <[email protected]>

* -download task support for geoip
Signed-off-by: rajeshLovesToCode <[email protected]>

* -fix for geoip IP constant
Signed-off-by: rajeshLovesToCode <[email protected]>
(cherry picked from commit 56b74be)

Co-authored-by: rajeshLovesToCode <[email protected]>

data-prepper-plugins/geoip-processor/build.gradle

@@ -1,3 +1,10 @@
+plugins{
+    id 'de.undercouch.download' version '4.1.2'
+}
+apply plugin: 'de.undercouch.download'
+
+import de.undercouch.gradle.tasks.download.Download
+
 /*
  * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
@@ -23,8 +30,75 @@ dependencies {
     testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
     testImplementation project(':data-prepper-test-common')
 }
+def geoIP2='GeoIP2'
+def geoLite2= 'GeoLite2'
+task downloadFile(type: Download) {
+
+    def urls = [
+            'https://raw.githubusercontent.com/maxmind/MaxMind-DB/main/test-data/GeoIP2-City-Test.mmdb',
+            'https://raw.githubusercontent.com/maxmind/MaxMind-DB/main/test-data/GeoIP2-Country-Test.mmdb',
+            'https://raw.githubusercontent.com/maxmind/MaxMind-DB/main/test-data/GeoLite2-ASN-Test.mmdb'
+    ]
+    def mmdbFileExtension = '.mmdb'
+    def baseDirPath =  'src/test/resources/mmdb-file/geo-lite2/'
+
+    urls.each { url ->
+        src(url)
+        dest(baseDirPath)
+        doLast {
+
+            def testFileName = url.substring(url.lastIndexOf('/') + 1)
+            def testMmdbSubString = testFileName.substring(testFileName.lastIndexOf('-'))
+            def fileName = testFileName.substring(0, testFileName.length() - testMmdbSubString.length())
+
+            if(fileName.contains(geoIP2)) {
+                fileName = fileName.replace(geoIP2, geoLite2)
+            }
+            File sourceFile = file(baseDirPath+testFileName)
+            File destinationFile = file( baseDirPath+fileName+mmdbFileExtension)
+            sourceFile.renameTo(destinationFile)
+
+        }
+
+    }
+
+
+}
+task downloadEnterpriseFile(type: Download) {
+    dependsOn downloadFile
+    def urls = [
+            'https://raw.githubusercontent.com/maxmind/MaxMind-DB/main/test-data/GeoIP2-Enterprise-Test.mmdb'
+    ]
+    def mmdbFileExtension = '.mmdb'
+    def baseDirPath =  'src/test/resources/mmdb-file/geo-enterprise/'
+
+    urls.each { url ->
+        src(url)
+        def testFileName = url.substring(url.lastIndexOf('/') + 1)
+        def testMmdbSubString = testFileName.substring(testFileName.lastIndexOf('-'))
+        def fileName = testFileName.substring(0, testFileName.length() - testMmdbSubString.length())
 
+        dest(baseDirPath+testFileName)
+        doLast {
+            if(fileName.contains(geoIP2)) {
+                fileName = fileName.replace(geoIP2, geoLite2)
+            }
+            File sourceFile = file(baseDirPath+testFileName)
+            File destinationFile = file( baseDirPath+fileName+mmdbFileExtension)
+            sourceFile.renameTo(destinationFile)
+        }
+
+    }
+
+}
 
+/*task processTestResources(type: Copy) {
+    dependsOn  downloadEnterpriseFile
+    from 'src/test/resources' // Source directory containing test resources
+    into 'build/resources/test' // Destination directory for processed test resources
+}*/
+tasks.test.dependsOn 'processTestResources'
+tasks.processTestResources.dependsOn 'downloadEnterpriseFile'
 test {
     useJUnitPlatform()
 }

data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2Data.java

@@ -182,7 +182,7 @@ public Map<String, Object> getGeoData(InetAddress inetAddress, List<String> attr
                             enrichData(geoData, ORGANIZATION_NAME, organizationName);
                             break;
                         case NETWORK:
-                            enrichData(geoData, NETWORK, network.toString());
+                            enrichData(geoData, NETWORK,network!=null? network.toString():null);
                             break;
                     }
                 }
@@ -204,7 +204,7 @@ public Map<String, Object> getGeoData(InetAddress inetAddress, List<String> attr
                 }
 
                 enrichData(geoData, ORGANIZATION_NAME, organizationName);
-                enrichData(geoData, NETWORK, network.toString());
+                enrichData(geoData, NETWORK,network!=null? network.toString():null);
             }
         } catch (Exception ex) {
             throw new EnrichFailedException("Enrichment failed exception" + ex);

data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databaseenrich/GetGeoLite2DataTest.java

@@ -36,7 +36,7 @@
 class GetGeoLite2DataTest {
 
     private static final String PATH = "./src/test/resources/mmdb-file/geo-lite2";
-    public static final String IP = "2001:4860:4860::8888";
+    public static final String IP = "2a02:ec00:0:0:0:0:0:0";
     private static final String PREFIX_DIR = "first_database";
     private String tempFolderPath = System.getProperty("java.io.tmpdir") + File.separator + "GeoIP";
     @Mock
@@ -64,8 +64,8 @@ void getGeoDataTest_without_attributes() throws UnknownHostException {
         GeoIPProcessorService.downloadReady = false;
         Map<String, Object> geoData = getGeoLite2Data.getGeoData(inetAddress, attributes, tempFolderPath);
         Assertions.assertNotNull(geoData);
-        assertThat(geoData.get("country_iso_code"), equalTo("US"));
-        assertThat(geoData.get("ip"), equalTo("2001:4860:4860:0:0:0:0:8888"));
+        assertThat(geoData.get("country_iso_code"), equalTo("FR"));
+        assertThat(geoData.get("ip"), equalTo(IP));
         assertDoesNotThrow(() -> {
             getGeoLite2Data.closeReader();
         });
@@ -88,8 +88,8 @@ void getGeoDataTest_with_attributes() throws UnknownHostException {
         GeoIPProcessorService.downloadReady = false;
         Map<String, Object> geoData = getGeoLite2Data.getGeoData(inetAddress, attributes, tempFolderPath);
         Assertions.assertNotNull(geoData);
-        assertThat(geoData.get("country_name"), equalTo("United States"));
-        assertThat(geoData.get("ip"), equalTo("2001:4860:4860:0:0:0:0:8888"));
+        assertThat(geoData.get("country_name"), equalTo("France"));
+        assertThat(geoData.get("ip"), equalTo(IP));
         assertDoesNotThrow(() -> {
             getGeoLite2Data.closeReader();
         });