diff --git a/docs/ServiceNow-batchsource.md b/docs/ServiceNow-batchsource.md index 116a78b..4a2351a 100644 --- a/docs/ServiceNow-batchsource.md +++ b/docs/ServiceNow-batchsource.md @@ -58,6 +58,8 @@ ignored if the Mode is set to `Reporting`. **End Date**: The End date to be used to filter the data. The format must be `yyyy-MM-dd`. +**Page Size**: The number of records to fetch from ServiceNow. Default is 5000. + **Type of values**: The type of values to be returned. The type can be one of two values: `Actual` - will fetch the actual values from the ServiceNow tables, diff --git a/docs/ServiceNowMultiSource-batchsource.md b/docs/ServiceNowMultiSource-batchsource.md index 41f1b73..2f7952d 100644 --- a/docs/ServiceNowMultiSource-batchsource.md +++ b/docs/ServiceNowMultiSource-batchsource.md @@ -31,6 +31,8 @@ Properties **End Date**: The End date to be used to filter the data. The format must be `yyyy-MM-dd`. +**Page Size**: The number of records to fetch from ServiceNow. Default is 5000. + **Type of values**: The type of values to be returned. The type can be one of two values: `Actual` - will fetch the actual values from the ServiceNow tables, diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseSourceConfig.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseSourceConfig.java index bad1302..9310942 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseSourceConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseSourceConfig.java @@ -69,6 +69,12 @@ public class ServiceNowBaseSourceConfig extends ServiceNowBaseConfig { "is set to `Table`.") protected String tableNameField; + @Name(ServiceNowConstants.PROPERTY_PAGE_SIZE) + @Macro + @Nullable + @Description("The number of records to fetch from ServiceNow. Default is 5000.") + private Integer pageSize; + /** * Constructor for ServiceNowSourceConfig object. * @@ -82,16 +88,18 @@ public class ServiceNowBaseSourceConfig extends ServiceNowBaseConfig { * @param valueType The value type * @param startDate The start date * @param endDate The end date + * @param pageSize The page size */ public ServiceNowBaseSourceConfig(String referenceName, String clientId, String clientSecret, String restApiEndpoint, String user, String password, String tableNameField, String valueType, - @Nullable String startDate, @Nullable String endDate) { + @Nullable String startDate, @Nullable String endDate, Integer pageSize) { super(clientId, clientSecret, restApiEndpoint, user, password); this.referenceName = referenceName; this.tableNameField = tableNameField; this.valueType = valueType; this.startDate = startDate; this.endDate = endDate; + this.pageSize = pageSize; } public String getReferenceName() { @@ -112,6 +120,10 @@ public String getTableNameField() { return Strings.isNullOrEmpty(tableNameField) ? ServiceNowConstants.TABLE_NAME_FIELD_DEFAULT : tableNameField; } + public Integer getPageSize() { + return pageSize == null ? ServiceNowConstants.PAGE_SIZE : pageSize; + } + /** * Validates {@link ServiceNowBaseSourceConfig} instance. */ @@ -120,6 +132,7 @@ public void validate(FailureCollector collector) { IdUtils.validateReferenceName(referenceName, collector); validateValueType(collector); validateDateRange(collector); + validatePageSize(collector); } /** @@ -208,6 +221,18 @@ private void validateDateRange(FailureCollector collector) { } } + private void validatePageSize(FailureCollector collector) { + if (containsMacro(ServiceNowConstants.PROPERTY_PAGE_SIZE)) { + return; + } + + if (getPageSize() <= 0 || getPageSize() > 10000) { + collector.addFailure("Invalid Page Size.", "Page Size must be greater than 0 and " + + "less than or equal to 10000.") + .withConfigProperty(ServiceNowConstants.PROPERTY_PAGE_SIZE); + } + } + public boolean shouldGetSchema() { return !containsMacro(ServiceNowConstants.PROPERTY_QUERY_MODE) && !containsMacro(ServiceNowConstants.PROPERTY_APPLICATION_NAME) diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java index 371a895..57f0da7 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java @@ -114,28 +114,28 @@ private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNow @Override public List getSplits(JobContext jobContext) { ServiceNowJobConfiguration jobConfig = new ServiceNowJobConfiguration(jobContext.getConfiguration()); - + int pageSize = jobConfig.getPluginConf().getPageSize().intValue(); List tableInfos = jobConfig.getTableInfos(); List resultSplits = new ArrayList<>(); for (ServiceNowTableInfo tableInfo : tableInfos) { String tableName = tableInfo.getTableName(); int totalRecords = tableInfo.getRecordCount(); - if (totalRecords <= ServiceNowConstants.PAGE_SIZE) { + if (totalRecords <= pageSize) { // add single split for table and continue resultSplits.add(new ServiceNowInputSplit(tableName, 0)); continue; } - int pages = (tableInfo.getRecordCount() / ServiceNowConstants.PAGE_SIZE); - if (tableInfo.getRecordCount() % ServiceNowConstants.PAGE_SIZE > 0) { + int pages = (tableInfo.getRecordCount() / pageSize); + if (tableInfo.getRecordCount() % pageSize > 0) { pages++; } int offset = 0; for (int page = 1; page <= pages; page++) { resultSplits.add(new ServiceNowInputSplit(tableName, offset)); - offset += ServiceNowConstants.PAGE_SIZE; + offset += pageSize; } } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java index f4e5410..0acaf6d 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java @@ -116,7 +116,7 @@ public static Set getList(String value) { @Override public List getSplits(JobContext jobContext) throws IOException, InterruptedException { ServiceNowJobConfiguration jobConfig = new ServiceNowJobConfiguration(jobContext.getConfiguration()); - + int pageSize = jobConfig.getPluginConf().getPageSize().intValue(); List tableInfos = jobConfig.getTableInfos(); List resultSplits = new ArrayList<>(); @@ -124,15 +124,15 @@ public List getSplits(JobContext jobContext) throws IOException, Int String tableName = tableInfo.getTableName(); int totalRecords = tableInfo.getRecordCount(); - int pages = (totalRecords / ServiceNowConstants.PAGE_SIZE); - if (totalRecords % ServiceNowConstants.PAGE_SIZE > 0) { + int pages = (totalRecords / pageSize); + if (totalRecords % pageSize > 0) { pages++; } int offset = 0; for (int page = 1; page <= pages; page++) { resultSplits.add(new ServiceNowInputSplit(tableName, offset)); - offset += ServiceNowConstants.PAGE_SIZE; + offset += pageSize; } } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java index 00f1d44..3d25c9c 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java @@ -99,7 +99,7 @@ void fetchData() throws IOException { results = restApi.fetchTableRecordsRetryableMode(tableName, multiSourcePluginConf.getValueType(), multiSourcePluginConf.getStartDate(), multiSourcePluginConf.getEndDate(), split.getOffset(), - ServiceNowConstants.PAGE_SIZE); + multiSourcePluginConf.getPageSize()); iterator = results.iterator(); } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfig.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfig.java index 1257249..a5fa7ea 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfig.java @@ -50,12 +50,14 @@ public class ServiceNowMultiSourceConfig extends ServiceNowBaseSourceConfig { * @param valueType The value type * @param startDate The start date * @param endDate The end date + * @param pageSize The page size */ public ServiceNowMultiSourceConfig(String referenceName, String clientId, String clientSecret, String restApiEndpoint, String user, String password, String tableNameField, String valueType, - @Nullable String startDate, @Nullable String endDate, String tableNames) { + @Nullable String startDate, @Nullable String endDate, Integer pageSize, + String tableNames) { super(referenceName, clientId, clientSecret, restApiEndpoint, user, password, tableNameField, valueType, startDate, - endDate); + endDate, pageSize); this.tableNames = tableNames; } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java index 8e0fe13..26df0c8 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java @@ -101,7 +101,7 @@ private void fetchData() throws IOException { // Get the table data results = restApi.fetchTableRecordsRetryableMode(tableName, pluginConf.getValueType(), pluginConf.getStartDate(), pluginConf.getEndDate(), split.getOffset(), - ServiceNowConstants.PAGE_SIZE); + pluginConf.getPageSize()); LOG.debug("Results size={}", results.size()); iterator = results.iterator(); diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfig.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfig.java index b7a80ee..6c81133 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfig.java @@ -57,7 +57,6 @@ public class ServiceNowSourceConfig extends ServiceNowBaseSourceConfig { "will be ignored if the Mode is set to `Reporting`. The ServiceNow table can be browsed using browse button.") private String tableName; - /** * Constructor for ServiceNowSourceConfig object. * @@ -74,13 +73,15 @@ public class ServiceNowSourceConfig extends ServiceNowBaseSourceConfig { * @param valueType The value type * @param startDate The start date * @param endDate The end date + * @param pageSize The page size */ public ServiceNowSourceConfig(String referenceName, String queryMode, @Nullable String applicationName, @Nullable String tableNameField, @Nullable String tableName, String clientId, String clientSecret, String restApiEndpoint, String user, String password, - String valueType, @Nullable String startDate, @Nullable String endDate) { + String valueType, @Nullable String startDate, @Nullable String endDate, + Integer pageSize) { super(referenceName, clientId, clientSecret, restApiEndpoint, user, password, tableNameField, valueType, startDate, - endDate); + endDate, pageSize); this.referenceName = referenceName; this.queryMode = queryMode; this.applicationName = applicationName; @@ -152,7 +153,7 @@ public SourceApplication getApplicationName() { public String getTableName() { return tableName; } - + /** * Validates {@link ServiceNowSourceConfig} instance. */ diff --git a/src/main/java/io/cdap/plugin/servicenow/util/ServiceNowConstants.java b/src/main/java/io/cdap/plugin/servicenow/util/ServiceNowConstants.java index 18e3024..2a126a8 100644 --- a/src/main/java/io/cdap/plugin/servicenow/util/ServiceNowConstants.java +++ b/src/main/java/io/cdap/plugin/servicenow/util/ServiceNowConstants.java @@ -86,6 +86,11 @@ public interface ServiceNowConstants { */ String PROPERTY_OPERATION = "operation"; + /** + * Configuration property name used to specify the page size. + */ + String PROPERTY_PAGE_SIZE = "pageSize"; + /** * Configuration property name used to get the schema. */ diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java index ff3602d..c94ae81 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java @@ -166,6 +166,7 @@ public void testFetchDataOnInvalidTable() throws IOException, OAuthProblemExcept .setValueType("Actual") .setStartDate("2021-01-01") .setEndDate("2022-02-18") + .setPageSize(10) .setTableNameField("tablename") .buildMultiSource(); @@ -186,7 +187,7 @@ public void testFetchDataOnInvalidTable() throws IOException, OAuthProblemExcept restApi.fetchTableRecords(tableName, serviceNowMultiSourceConfig.getValueType(), serviceNowMultiSourceConfig.getStartDate(), serviceNowMultiSourceConfig.getEndDate(), split.getOffset(), - ServiceNowConstants.PAGE_SIZE); + serviceNowMultiSourceConfig.getPageSize()); ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); response.setResult(results); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java index d07bb9e..67fb025 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java @@ -53,7 +53,7 @@ public class ServiceNowMultiSourceConfigTest { public void testConstructor() { serviceNowMultiSourceConfig = new ServiceNowMultiSourceConfig("referenceName", "client_id", "client_secret", "https://example.com", "user", "password", - "tablename", "Actual", "2021-12-30", "2021-12-31", + "tablename", "Actual", "2021-12-30", "2021-12-31", 10, "sys_user"); Assert.assertEquals("sys_user", serviceNowMultiSourceConfig.getTableNames()); Assert.assertEquals("Actual", serviceNowMultiSourceConfig.getValueType().getValueType()); @@ -426,7 +426,8 @@ public void testValidateTableNames() { ServiceNowMultiSourceConfig serviceNowMultiSourceConfig = new ServiceNowMultiSourceConfig( "Reference Name", "42", "client_secret", "https://config.us-east-2.amazonaws.com", "User", "password", - "tablename", "Actual", "2020-03-01", "2020-03-01", ","); + "tablename", "Actual", "2020-03-01", "2020-03-01", 10, + ","); serviceNowMultiSourceConfig.validateTableNames(new MockFailureCollector("Stage Name")); Assert.assertEquals("42", serviceNowMultiSourceConfig.getConnection().getClientId()); Assert.assertEquals("tablename", serviceNowMultiSourceConfig.tableNameField); @@ -446,7 +447,7 @@ public void testValidateTableNames2() { ServiceNowMultiSourceConfig serviceNowMultiSourceConfig = new ServiceNowMultiSourceConfig( "Reference Name", "Table Name Field", "42", "Client Secret", "https://config.us-east-2.amazonaws.com", "User", "password", "42", - "2020-03-01", "2020-03-01", ""); + "2020-03-01", "2020-03-01", 10, ""); MockFailureCollector mockFailureCollector = new MockFailureCollector("Stage Name"); serviceNowMultiSourceConfig.validateTableNames(mockFailureCollector); List validationFailures = mockFailureCollector.getValidationFailures(); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java index c5383e5..1527d59 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java @@ -74,6 +74,7 @@ public void initializeTests() { .setValueType("Actual") .setStartDate("2021-12-30") .setEndDate("2021-12-31") + .setPageSize(10) .setTableNameField("tablename") .build(); @@ -98,8 +99,9 @@ public void testConstructor2() throws IOException { "https://ven05127." + "service-now.com/", "User", "password", - "Actual", "2021-12-30", - "2021-12-31"); + "Actual", + "2021-12-30", + "2021-12-31", 10); serviceNowRecordReader.close(); Assert.assertEquals(0, serviceNowRecordReader.pos); @@ -182,7 +184,7 @@ public void testFetchData() throws Exception { Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig. getEndDate(), split.getOffset(), - ServiceNowConstants.PAGE_SIZE)).thenReturn(results); + serviceNowSourceConfig.getPageSize())).thenReturn(results); Mockito.when(restApi.fetchTableSchema(tableName)) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split, null); @@ -236,7 +238,7 @@ public void testFetchDataReportingMode() throws Exception { Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), - ServiceNowConstants.PAGE_SIZE)).thenReturn(results); + serviceNowSourceConfig.getPageSize())).thenReturn(results); Mockito.when(restApi.fetchTableSchema(tableName)) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split, null); @@ -269,7 +271,7 @@ public void testFetchDataOnInvalidTable() throws Exception { Mockito.when(restApi.fetchTableRecords(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), - ServiceNowConstants.PAGE_SIZE)).thenReturn(results); + serviceNowSourceConfig.getPageSize())).thenReturn(results); ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); response.setResult(results); Mockito.when(restApi.fetchTableSchema(tableName)) diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigHelper.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigHelper.java index 819adef..c9f091b 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigHelper.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigHelper.java @@ -58,6 +58,8 @@ The name of the ServiceNow table(s) from which data to be fetched. private String valueType = "Actual"; private String startDate = ""; private String endDate = ""; + private Integer pageSize = 5000; + public ConfigBuilder setReferenceName(String referenceName) { this.referenceName = referenceName; @@ -129,14 +131,19 @@ public ConfigBuilder setEndDate(String endDate) { return this; } + public ConfigBuilder setPageSize(Integer pageSize) { + this.pageSize = pageSize; + return this; + } + public ServiceNowSourceConfig build() { return new ServiceNowSourceConfig(referenceName, queryMode, applicationName, tableNameField, tableName, - clientId, clientSecret, restApiEndpoint, user, password, valueType, startDate, endDate); + clientId, clientSecret, restApiEndpoint, user, password, valueType, startDate, endDate, pageSize); } public ServiceNowMultiSourceConfig buildMultiSource() { return new ServiceNowMultiSourceConfig(referenceName, clientId, clientSecret, restApiEndpoint, user, password, - tableNameField, valueType, startDate, endDate, tableNames); + tableNameField, valueType, startDate, endDate, pageSize, tableNames); } diff --git a/widgets/ServiceNow-batchsource.json b/widgets/ServiceNow-batchsource.json index 1534c2a..121933b 100644 --- a/widgets/ServiceNow-batchsource.json +++ b/widgets/ServiceNow-batchsource.json @@ -177,6 +177,15 @@ "widget-attributes" : { "placeholder": "End date to be used to filter the data" } + }, + { + "widget-type": "number", + "label": "Page Size", + "name": "pageSize", + "widget-attributes" : { + "placeholder": "Number of records to fetch from ServiceNow", + "default": 5000 + } } ] } diff --git a/widgets/ServiceNowMultiSource-batchsource.json b/widgets/ServiceNowMultiSource-batchsource.json index e0e0806..0823f5c 100644 --- a/widgets/ServiceNowMultiSource-batchsource.json +++ b/widgets/ServiceNowMultiSource-batchsource.json @@ -131,6 +131,15 @@ "placeholder": "End date to be used to filter the data" } }, + { + "widget-type": "number", + "label": "Page Size", + "name": "pageSize", + "widget-attributes" : { + "placeholder": "Number of records to fetch from ServiceNow", + "default": 5000 + } + }, { "widget-type": "textbox", "label": "Table Name Field",