Merge pull request #505 from medizininformatik-initiative/500-make-de…

…faultpagesize-in-everythingdataselector-configurable

Add `pageSize` to EverythingDataSelectorConfig

.github/test/cd-agent/projects/example.yaml

@@ -29,10 +29,12 @@ dataSelector:
     fhirServer:
       # Base URL of the FHIR server endpoint
       baseUrl: http://cd-hds:8080/fhir
-    # Patient identifier resolution configuration
+    # (optional) Patient identifier resolution configuration
     resolve:
       # System URL for patient identifier lookup
       patientIdentifierSystem: http://fts.smith.care
+    # (default: 500) Maximum size of bundles requested from cd-hds
+    pageSize: 500
 
 ### Deidentificator Configuration
 ##! https://medizininformatik-initiative.github.io/fts-next/configuration/cd-agent/deidentificator
@@ -49,9 +51,9 @@ deidentificator:
       domains:
         # TCA domain to store pseudonyms in
         pseudonym: MII
-        # TCA domain to store peudonym salts in
+        # TCA domain to store pseudonym salts in
         salt: MII
-        # TCA domain to store dateshift salts in
+        # TCA domain to store dateShift salts in
         dateShift: MII
     # Dates should be shifted a maximum of this amount
     # specified as an ISO-8601 duration (e.g., P1D means 1 day)

clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/EverythingDataSelector.java

@@ -32,17 +32,19 @@ public class EverythingDataSelector implements DataSelector {
   private final WebClient client;
   private final PatientIdResolver pidResolver;
   private final MeterRegistry meterRegistry;
-  private final int defaultPageSize = 500;
+  private final int pageSize;
 
   public EverythingDataSelector(
       Config common,
       WebClient client,
       PatientIdResolver patientIdResolver,
-      MeterRegistry meterRegistry) {
+      MeterRegistry meterRegistry,
+      int pageSize) {
     this.common = common;
     this.client = client;
     this.pidResolver = patientIdResolver;
     this.meterRegistry = meterRegistry;
+    this.pageSize = pageSize;
   }
 
   @Override
@@ -81,8 +83,7 @@ private Mono<Bundle> fetchNextPage(Bundle bundle) {
   }
 
   private Function<UriBuilder, URI> withoutConsent(IIdType fhirId) {
-    return (uriBuilder) ->
-        uriBuilder.queryParam("_count", defaultPageSize).build(fhirId.getIdPart());
+    return (uriBuilder) -> uriBuilder.queryParam("_count", pageSize).build(fhirId.getIdPart());
   }
 
   private Function<UriBuilder, URI> withConsent(ConsentedPatient patient, IIdType fhirId) {
@@ -93,7 +94,7 @@ private Function<UriBuilder, URI> withConsent(ConsentedPatient patient, IIdType
     }
     return (uriBuilder) ->
         uriBuilder
-            .queryParam("_count", defaultPageSize)
+            .queryParam("_count", pageSize)
             .queryParam("start", formatWithSystemTZ(period.get().start()))
             .queryParam("end", formatWithSystemTZ(period.get().end()))
             .build(Map.of("id", fhirId.getIdPart()));

clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/EverythingDataSelectorConfig.java

@@ -1,17 +1,45 @@
 package care.smith.fts.cda.impl;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static lombok.AccessLevel.PRIVATE;
+
 import care.smith.fts.cda.services.FhirResolveConfig;
 import care.smith.fts.util.HttpClientConfig;
 import jakarta.validation.constraints.NotNull;
+import lombok.EqualsAndHashCode;
+import lombok.Setter;
+import lombok.ToString;
+
+@Setter(PRIVATE)
+@EqualsAndHashCode
+@ToString
+public final class EverythingDataSelectorConfig {
+
+  public static int DEFAULT_PAGE_SIZE = 500;
 
-public record EverythingDataSelectorConfig(
-    /* Server to fetch data from */
-    @NotNull HttpClientConfig fhirServer,
+  private @NotNull HttpClientConfig fhirServer;
+  private FhirResolveConfig resolve;
+  private int pageSize = DEFAULT_PAGE_SIZE;
 
-    /* */
-    FhirResolveConfig resolve) {
+  private EverythingDataSelectorConfig() {}
+
+  public EverythingDataSelectorConfig(
+      HttpClientConfig fhirServer, FhirResolveConfig resolve, int pageSize) {
+    this.fhirServer = fhirServer;
+    this.resolve = resolve;
+    checkArgument(pageSize > 0, "pageSize must be greater than 0");
+    this.pageSize = pageSize;
+  }
+
+  public @NotNull HttpClientConfig fhirServer() {
+    return fhirServer;
+  }
+
+  public FhirResolveConfig resolve() {
+    return resolve;
+  }
 
-  public EverythingDataSelectorConfig(HttpClientConfig fhirServer) {
-    this(fhirServer, null);
+  public int pageSize() {
+    return pageSize;
   }
 }

clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/EverythingDataSelectorFactory.java

@@ -31,7 +31,7 @@ public Class<EverythingDataSelectorConfig> getConfigType() {
   public DataSelector create(DataSelector.Config common, EverythingDataSelectorConfig config) {
     var client = clientFactory.create(config.fhirServer());
     PatientIdResolver resolver = createResolver(config, client);
-    return new EverythingDataSelector(common, client, resolver, meterRegistry);
+    return new EverythingDataSelector(common, client, resolver, meterRegistry, config.pageSize());
   }
 
   private PatientIdResolver createResolver(EverythingDataSelectorConfig config, WebClient client) {

clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/EverythingDataSelectorConfigTest.java

@@ -0,0 +1,23 @@
+package care.smith.fts.cda.impl;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import care.smith.fts.util.HttpClientConfig;
+import org.junit.jupiter.api.Test;
+
+class EverythingDataSelectorConfigTest {
+
+  @Test
+  void useDefaultIfPageSizeIsInvalid() {
+    assertThrows(
+        IllegalArgumentException.class,
+        () -> {
+          new EverythingDataSelectorConfig(new HttpClientConfig("http://localhost"), null, 0);
+        });
+    assertThrows(
+        IllegalArgumentException.class,
+        () -> {
+          new EverythingDataSelectorConfig(new HttpClientConfig("http://localhost"), null, -1);
+        });
+  }
+}

clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/EverythingDataSelectorFactoryTest.java

@@ -2,6 +2,7 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
+import care.smith.fts.api.cda.DataSelector;
 import care.smith.fts.cda.services.FhirResolveConfig;
 import care.smith.fts.util.HttpClientConfig;
 import care.smith.fts.util.WebClientFactory;
@@ -31,10 +32,11 @@ void testConfigType() {
 
   @Test
   void testCreateWithoutResolver() {
-    assertThat(
-            factory.create(
-                null, new EverythingDataSelectorConfig(new HttpClientConfig("http://localhost"))))
-        .isNotNull();
+    var dataSelector =
+        factory.create(
+            null,
+            new EverythingDataSelectorConfig(new HttpClientConfig("http://localhost"), null, 500));
+    assertThat(dataSelector).isNotNull();
   }
 
   @Test
@@ -44,7 +46,8 @@ void testCreateWithResolver() {
                 null,
                 new EverythingDataSelectorConfig(
                     new HttpClientConfig("http://localhost"),
-                    new FhirResolveConfig("https://patient-identifier.example.com"))))
+                    new FhirResolveConfig("https://patient-identifier.example.com"),
+                    500)))
         .isNotNull();
   }
 }

clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/EverythingDataSelectorTest.java

@@ -46,7 +46,7 @@ void noConsentErrors() {
     var client = builder();
     var dataSelector =
         new EverythingDataSelector(
-            common, clientFactory.create(client, server), patient, meterRegistry);
+            common, clientFactory.create(client, server), patient, meterRegistry, 500);
 
     create(dataSelector.select(new ConsentedPatient(PATIENT_ID))).expectError().verify();
   }
@@ -57,7 +57,7 @@ void noConsentSucceedsIfConsentIgnored() {
     DataSelector.Config common = new DataSelector.Config(true, null);
     var dataSelector =
         new EverythingDataSelector(
-            common, clientFactory.create(client, server), patient, meterRegistry);
+            common, clientFactory.create(client, server), patient, meterRegistry, 500);
 
     create(dataSelector.select(new ConsentedPatient(PATIENT_ID))).verifyComplete();
   }
@@ -73,7 +73,7 @@ void selectionSucceeds() throws Exception {
     }
     var dataSelector =
         new EverythingDataSelector(
-            common, clientFactory.create(client, server), patient, meterRegistry);
+            common, clientFactory.create(client, server), patient, meterRegistry, 500);
 
     var consentedPolicies = new ConsentedPolicies();
     consentedPolicies.put("pol", new Period(ZonedDateTime.now(), ZonedDateTime.now().plusYears(5)));

docs/cd-agent/data-selector.md

@@ -16,6 +16,7 @@ dataSelector:
       baseUrl: http://cd-hds:8080/fhir
     resolve:
       patientIdentifierSystem: http://fts.smith.care
+    pageSize: 500
 ```
 
 ## Fields
@@ -46,6 +47,17 @@ The "everything" data selector uses the FHIR servers `patient/$everything` opera
       patientIdentifierSystem: http://custom.identifier.system
   ```
 
+#### `pageSize`
+
+* **Description**: Specifies the maximum number of FHIR resources to be included in a single bundle
+  when requesting data from the clinical domain health data storage (cd-hds).
+  This parameter helps control memory usage and network load by limiting the size of data transfers.
+* **Type**: Integer
+* **Default**: 500
+* **Example**:
+  ```yaml
+    pageSize: 1000
+
 ## Notes
 
   the `$everything` operation.