Merge pull request data-integrations#64 from cloudsufi/story/PLUGIN-1689

ServiceNow Plugin : Make PAGE_SIZE configurable
cloudsufi · Oct 31, 2023 · 7848498 · 7848498
2 parents 554fa90 + 6fab0c2
commit 7848498
Show file tree

Hide file tree

Showing 16 changed files with 95 additions and 29 deletions.
diff --git a/docs/ServiceNow-batchsource.md b/docs/ServiceNow-batchsource.md
@@ -58,6 +58,8 @@ ignored if the Mode is set to `Reporting`.
 
 **End Date**: The End date to be used to filter the data. The format must be `yyyy-MM-dd`.
 
+**Page Size**: The number of records to fetch from ServiceNow. Default is 5000.
+
 **Type of values**: The type of values to be returned. The type can be one of two values: 
 
 `Actual` -  will fetch the actual values from the ServiceNow tables,  

diff --git a/docs/ServiceNowMultiSource-batchsource.md b/docs/ServiceNowMultiSource-batchsource.md
@@ -31,6 +31,8 @@ Properties
 
 **End Date**: The End date to be used to filter the data. The format must be `yyyy-MM-dd`.
 
+**Page Size**: The number of records to fetch from ServiceNow. Default is 5000.
+
 **Type of values**: The type of values to be returned. The type can be one of two values: 
 
 `Actual` -  will fetch the actual values from the ServiceNow tables,  

diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseSourceConfig.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseSourceConfig.java
@@ -69,6 +69,12 @@ public class ServiceNowBaseSourceConfig extends ServiceNowBaseConfig {
     "is set to `Table`.")
   protected String tableNameField;
 
+  @Name(ServiceNowConstants.PROPERTY_PAGE_SIZE)
+  @Macro
+  @Nullable
+  @Description("The number of records to fetch from ServiceNow. Default is 5000.")
+  private Integer pageSize;
+
   /**
    * Constructor for ServiceNowSourceConfig object.
    *
@@ -82,16 +88,18 @@ public class ServiceNowBaseSourceConfig extends ServiceNowBaseConfig {
    * @param valueType The value type
    * @param startDate The start date
    * @param endDate The end date
+   * @param pageSize The page size
    */
   public ServiceNowBaseSourceConfig(String referenceName, String clientId, String clientSecret, String restApiEndpoint,
                                     String user, String password, String tableNameField, String valueType,
-                                    @Nullable String startDate, @Nullable String endDate) {
+                                    @Nullable String startDate, @Nullable String endDate, Integer pageSize) {
     super(clientId, clientSecret, restApiEndpoint, user, password);
     this.referenceName = referenceName;
     this.tableNameField = tableNameField;
     this.valueType = valueType;
     this.startDate = startDate;
     this.endDate = endDate;
+    this.pageSize = pageSize;
   }
 
   public String getReferenceName() {
@@ -112,6 +120,10 @@ public String getTableNameField() {
     return Strings.isNullOrEmpty(tableNameField) ? ServiceNowConstants.TABLE_NAME_FIELD_DEFAULT : tableNameField;
   }
 
+  public Integer getPageSize() {
+    return pageSize == null ? ServiceNowConstants.PAGE_SIZE : pageSize;
+  }
+
   /**
    * Validates {@link ServiceNowBaseSourceConfig} instance.
    */
@@ -120,6 +132,7 @@ public void validate(FailureCollector collector) {
     IdUtils.validateReferenceName(referenceName, collector);
     validateValueType(collector);
     validateDateRange(collector);
+    validatePageSize(collector);
   }
 
   /**
@@ -208,6 +221,18 @@ private void validateDateRange(FailureCollector collector) {
     }
   }
 
+  private void validatePageSize(FailureCollector collector) {
+    if (containsMacro(ServiceNowConstants.PROPERTY_PAGE_SIZE)) {
+      return;
+    }
+
+    if (getPageSize() <= 0 || getPageSize() > 10000) {
+      collector.addFailure("Invalid Page Size.", "Page Size must be greater than 0 and " +
+          "less than or equal to 10000.")
+        .withConfigProperty(ServiceNowConstants.PROPERTY_PAGE_SIZE);
+    }
+  }
+
   public boolean shouldGetSchema() {
     return !containsMacro(ServiceNowConstants.PROPERTY_QUERY_MODE)
       && !containsMacro(ServiceNowConstants.PROPERTY_APPLICATION_NAME)

diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java
@@ -114,28 +114,28 @@ private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNow
   @Override
   public List<InputSplit> getSplits(JobContext jobContext) {
     ServiceNowJobConfiguration jobConfig = new ServiceNowJobConfiguration(jobContext.getConfiguration());
-
+    int pageSize = jobConfig.getPluginConf().getPageSize().intValue();
     List<ServiceNowTableInfo> tableInfos = jobConfig.getTableInfos();
     List<InputSplit> resultSplits = new ArrayList<>();
 
     for (ServiceNowTableInfo tableInfo : tableInfos) {
       String tableName = tableInfo.getTableName();
       int totalRecords = tableInfo.getRecordCount();
-      if (totalRecords <= ServiceNowConstants.PAGE_SIZE) {
+      if (totalRecords <= pageSize) {
         // add single split for table and continue
         resultSplits.add(new ServiceNowInputSplit(tableName, 0));
         continue;
       }
 
-      int pages = (tableInfo.getRecordCount() / ServiceNowConstants.PAGE_SIZE);
-      if (tableInfo.getRecordCount() % ServiceNowConstants.PAGE_SIZE > 0) {
+      int pages = (tableInfo.getRecordCount() / pageSize);
+      if (tableInfo.getRecordCount() % pageSize > 0) {
         pages++;
       }
       int offset = 0;
 
       for (int page = 1; page <= pages; page++) {
         resultSplits.add(new ServiceNowInputSplit(tableName, offset));
-        offset += ServiceNowConstants.PAGE_SIZE;
+        offset += pageSize;
       }
     }
 

diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java
@@ -116,23 +116,23 @@ public static Set<String> getList(String value) {
   @Override
   public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException {
     ServiceNowJobConfiguration jobConfig = new ServiceNowJobConfiguration(jobContext.getConfiguration());
-
+    int pageSize = jobConfig.getPluginConf().getPageSize().intValue();
     List<ServiceNowTableInfo> tableInfos = jobConfig.getTableInfos();
     List<InputSplit> resultSplits = new ArrayList<>();
 
     for (ServiceNowTableInfo tableInfo : tableInfos) {
       String tableName = tableInfo.getTableName();
       int totalRecords = tableInfo.getRecordCount();
 
-      int pages = (totalRecords / ServiceNowConstants.PAGE_SIZE);
-      if (totalRecords % ServiceNowConstants.PAGE_SIZE > 0) {
+      int pages = (totalRecords / pageSize);
+      if (totalRecords % pageSize > 0) {
         pages++;
       }
       int offset = 0;
 
       for (int page = 1; page <= pages; page++) {
         resultSplits.add(new ServiceNowInputSplit(tableName, offset));
-        offset += ServiceNowConstants.PAGE_SIZE;
+        offset += pageSize;
       }
     }
 

diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java
@@ -99,7 +99,7 @@ void fetchData() throws IOException {
     results = restApi.fetchTableRecordsRetryableMode(tableName, multiSourcePluginConf.getValueType(),
                                                      multiSourcePluginConf.getStartDate(),
                                                      multiSourcePluginConf.getEndDate(), split.getOffset(),
-                                                     ServiceNowConstants.PAGE_SIZE);
+                                                     multiSourcePluginConf.getPageSize());
 
     iterator = results.iterator();
   }

diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfig.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfig.java
@@ -50,12 +50,14 @@ public class ServiceNowMultiSourceConfig extends ServiceNowBaseSourceConfig {
    * @param valueType       The value type
    * @param startDate       The start date
    * @param endDate         The end date
+   * @param pageSize        The page size
    */
   public ServiceNowMultiSourceConfig(String referenceName, String clientId, String clientSecret, String restApiEndpoint,
                                      String user, String password, String tableNameField, String valueType,
-                                     @Nullable String startDate, @Nullable String endDate, String tableNames) {
+                                     @Nullable String startDate, @Nullable String endDate, Integer pageSize,
+                                     String tableNames) {
     super(referenceName, clientId, clientSecret, restApiEndpoint, user, password, tableNameField, valueType, startDate,
-          endDate);
+          endDate, pageSize);
     this.tableNames = tableNames;
   }
 

diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java
@@ -101,7 +101,7 @@ private void fetchData() throws IOException {
     // Get the table data
     results = restApi.fetchTableRecordsRetryableMode(tableName, pluginConf.getValueType(), pluginConf.getStartDate(),
                                                      pluginConf.getEndDate(), split.getOffset(),
-                                                     ServiceNowConstants.PAGE_SIZE);
+                                                     pluginConf.getPageSize());
     LOG.debug("Results size={}", results.size());
 
     iterator = results.iterator();

diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfig.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfig.java
@@ -57,7 +57,6 @@ public class ServiceNowSourceConfig extends ServiceNowBaseSourceConfig {
     "will be ignored if the Mode is set to `Reporting`. The ServiceNow table can be browsed using browse button.")
   private String tableName;
 
-
   /**
    * Constructor for ServiceNowSourceConfig object.
    *
@@ -74,13 +73,15 @@ public class ServiceNowSourceConfig extends ServiceNowBaseSourceConfig {
    * @param valueType       The value type
    * @param startDate       The start date
    * @param endDate         The end date
+   * @param pageSize        The page size
    */
   public ServiceNowSourceConfig(String referenceName, String queryMode, @Nullable String applicationName,
                                 @Nullable String tableNameField, @Nullable String tableName, String clientId,
                                 String clientSecret, String restApiEndpoint, String user, String password,
-                                String valueType, @Nullable String startDate, @Nullable String endDate) {
+                                String valueType, @Nullable String startDate, @Nullable String endDate,
+                                Integer pageSize) {
     super(referenceName, clientId, clientSecret, restApiEndpoint, user, password, tableNameField, valueType, startDate,
-          endDate);
+          endDate, pageSize);
     this.referenceName = referenceName;
     this.queryMode = queryMode;
     this.applicationName = applicationName;
@@ -152,7 +153,7 @@ public SourceApplication getApplicationName() {
   public String getTableName() {
     return tableName;
   }
-
+  
   /**
    * Validates {@link ServiceNowSourceConfig} instance.
    */

diff --git a/src/main/java/io/cdap/plugin/servicenow/util/ServiceNowConstants.java b/src/main/java/io/cdap/plugin/servicenow/util/ServiceNowConstants.java
@@ -86,6 +86,11 @@ public interface ServiceNowConstants {
    */
   String PROPERTY_OPERATION = "operation";
 
+  /**
+   * Configuration property name used to specify the page size.
+   */
+  String PROPERTY_PAGE_SIZE = "pageSize";
+
   /**
    * Configuration property name used to get the schema.
    */

diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java
@@ -166,6 +166,7 @@ public void testFetchDataOnInvalidTable() throws IOException, OAuthProblemExcept
       .setValueType("Actual")
       .setStartDate("2021-01-01")
       .setEndDate("2022-02-18")
+      .setPageSize(10)
       .setTableNameField("tablename")
       .buildMultiSource();
 
@@ -186,7 +187,7 @@ public void testFetchDataOnInvalidTable() throws IOException, OAuthProblemExcept
     restApi.fetchTableRecords(tableName, serviceNowMultiSourceConfig.getValueType(),
                               serviceNowMultiSourceConfig.getStartDate(), serviceNowMultiSourceConfig.getEndDate(),
                               split.getOffset(),
-                              ServiceNowConstants.PAGE_SIZE);
+                              serviceNowMultiSourceConfig.getPageSize());
 
     ServiceNowTableDataResponse response = new ServiceNowTableDataResponse();
     response.setResult(results);

diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java
@@ -53,7 +53,7 @@ public class ServiceNowMultiSourceConfigTest {
   public void testConstructor() {
     serviceNowMultiSourceConfig = new ServiceNowMultiSourceConfig("referenceName", "client_id",
       "client_secret", "https://example.com", "user", "password",
-      "tablename", "Actual", "2021-12-30", "2021-12-31",
+      "tablename", "Actual", "2021-12-30", "2021-12-31", 10,
       "sys_user");
     Assert.assertEquals("sys_user", serviceNowMultiSourceConfig.getTableNames());
     Assert.assertEquals("Actual", serviceNowMultiSourceConfig.getValueType().getValueType());
@@ -426,7 +426,8 @@ public void testValidateTableNames() {
     ServiceNowMultiSourceConfig serviceNowMultiSourceConfig = new ServiceNowMultiSourceConfig(
       "Reference Name", "42", "client_secret",
       "https://config.us-east-2.amazonaws.com", "User", "password",
-      "tablename", "Actual", "2020-03-01", "2020-03-01", ",");
+      "tablename", "Actual", "2020-03-01", "2020-03-01", 10,
+      ",");
     serviceNowMultiSourceConfig.validateTableNames(new MockFailureCollector("Stage Name"));
     Assert.assertEquals("42", serviceNowMultiSourceConfig.getConnection().getClientId());
     Assert.assertEquals("tablename", serviceNowMultiSourceConfig.tableNameField);
@@ -446,7 +447,7 @@ public void testValidateTableNames2() {
     ServiceNowMultiSourceConfig serviceNowMultiSourceConfig = new ServiceNowMultiSourceConfig(
       "Reference Name", "Table Name Field", "42", "Client Secret",
       "https://config.us-east-2.amazonaws.com", "User", "password", "42",
-      "2020-03-01", "2020-03-01", "");
+      "2020-03-01", "2020-03-01", 10, "");
     MockFailureCollector mockFailureCollector = new MockFailureCollector("Stage Name");
     serviceNowMultiSourceConfig.validateTableNames(mockFailureCollector);
     List<ValidationFailure> validationFailures = mockFailureCollector.getValidationFailures();

diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java
@@ -74,6 +74,7 @@ public void initializeTests() {
       .setValueType("Actual")
       .setStartDate("2021-12-30")
       .setEndDate("2021-12-31")
+      .setPageSize(10)
       .setTableNameField("tablename")
       .build();
 
@@ -98,8 +99,9 @@ public void testConstructor2() throws IOException {
                                                                                "https://ven05127." +
                                                                                  "service-now.com/", "User",
                                                                                "password",
-                                                                               "Actual", "2021-12-30",
-                                                                               "2021-12-31");
+                                                                               "Actual",
+                                                                               "2021-12-30",
+                                                                               "2021-12-31", 10);
 
     serviceNowRecordReader.close();
     Assert.assertEquals(0, serviceNowRecordReader.pos);
@@ -182,7 +184,7 @@ public void testFetchData() throws Exception {
     Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(),
                                                         serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.
                                                           getEndDate(), split.getOffset(),
-                                                        ServiceNowConstants.PAGE_SIZE)).thenReturn(results);
+                                                        serviceNowSourceConfig.getPageSize())).thenReturn(results);
     Mockito.when(restApi.fetchTableSchema(tableName))
       .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING))));
     serviceNowRecordReader.initialize(split, null);
@@ -236,7 +238,7 @@ public void testFetchDataReportingMode() throws Exception {
     Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(),
                                                         serviceNowSourceConfig.getStartDate(),
                                                         serviceNowSourceConfig.getEndDate(), split.getOffset(),
-                                                        ServiceNowConstants.PAGE_SIZE)).thenReturn(results);
+                                                        serviceNowSourceConfig.getPageSize())).thenReturn(results);
     Mockito.when(restApi.fetchTableSchema(tableName))
       .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING))));
     serviceNowRecordReader.initialize(split, null);
@@ -269,7 +271,7 @@ public void testFetchDataOnInvalidTable() throws Exception {
     Mockito.when(restApi.fetchTableRecords(tableName, serviceNowSourceConfig.getValueType(),
                                            serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(),
                                            split.getOffset(),
-                                           ServiceNowConstants.PAGE_SIZE)).thenReturn(results);
+                                           serviceNowSourceConfig.getPageSize())).thenReturn(results);
     ServiceNowTableDataResponse response = new ServiceNowTableDataResponse();
     response.setResult(results);
     Mockito.when(restApi.fetchTableSchema(tableName))

diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigHelper.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigHelper.java
@@ -58,6 +58,8 @@ The name of the ServiceNow table(s) from which data to be fetched.
     private String valueType = "Actual";
     private String startDate = "";
     private String endDate = "";
+    private Integer pageSize = 5000;
+
 
     public ConfigBuilder setReferenceName(String referenceName) {
       this.referenceName = referenceName;
@@ -129,14 +131,19 @@ public ConfigBuilder setEndDate(String endDate) {
       return this;
     }
 
+    public ConfigBuilder setPageSize(Integer pageSize) {
+      this.pageSize = pageSize;
+      return this;
+    }
+
     public ServiceNowSourceConfig build() {
       return new ServiceNowSourceConfig(referenceName, queryMode, applicationName, tableNameField, tableName,
-        clientId, clientSecret, restApiEndpoint, user, password, valueType, startDate, endDate);
+        clientId, clientSecret, restApiEndpoint, user, password, valueType, startDate, endDate, pageSize);
     }
 
     public ServiceNowMultiSourceConfig buildMultiSource() {
       return new ServiceNowMultiSourceConfig(referenceName, clientId, clientSecret, restApiEndpoint, user, password,
-        tableNameField, valueType, startDate, endDate, tableNames);
+        tableNameField, valueType, startDate, endDate, pageSize, tableNames);
     }
 
 

diff --git a/widgets/ServiceNow-batchsource.json b/widgets/ServiceNow-batchsource.json
@@ -177,6 +177,15 @@
           "widget-attributes" : {
             "placeholder": "End date to be used to filter the data"
           }
+        },
+        {
+          "widget-type": "number",
+          "label": "Page Size",
+          "name": "pageSize",
+          "widget-attributes" : {
+            "placeholder": "Number of records to fetch from ServiceNow",
+            "default": 5000
+          }
         }
       ]
     }

diff --git a/widgets/ServiceNowMultiSource-batchsource.json b/widgets/ServiceNowMultiSource-batchsource.json
@@ -131,6 +131,15 @@
             "placeholder": "End date to be used to filter the data"
           }
         },
+        {
+          "widget-type": "number",
+          "label": "Page Size",
+          "name": "pageSize",
+          "widget-attributes" : {
+            "placeholder": "Number of records to fetch from ServiceNow",
+            "default": 5000
+          }
+        },
         {
           "widget-type": "textbox",
           "label": "Table Name Field",