test(e2e): added e2e tests

.github/workflows/ci.yaml

@@ -26,6 +26,48 @@ jobs:
     secrets:
       github-token: ${{ secrets.GITHUB_TOKEN }}
 
+  test:
+    needs:
+      - build
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Download container image
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
+        with:
+          name: ${{ needs.build.outputs.image-slug }}
+          path: /tmp
+
+      - name: Load image
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          ls -lar /tmp
+          docker load --input /tmp/image.tar
+          docker image ls -a
+
+      - name: Start compose fixtures
+        run: |
+          docker compose up wait-for-pathling
+
+      - name: Install .NET
+        uses: actions/setup-dotnet@4d6c8fcf3c8f7a60068d26b594648e99df24cee3 # v4.0.0
+        with:
+          dotnet-version: "8.0.x"
+
+      - name: Run tests
+        env:
+          PATHLING_S3_IMPORT_IMAGE_TAG: ${{ needs.build.outputs.image-version }}
+        run: dotnet test src/PathlingS3Import.Tests.E2E --configuration=Release -l "console;verbosity=detailed"
+
+      - name: Print compose logs
+        if: always()
+        run: |
+          docker compose logs
+          docker compose down --volumes --remove-orphans
+
   lint:
     uses: miracum/.github/.github/workflows/standard-lint.yaml@d09a237ae62959d3cf89d526a035fbd9d9d816ee # v1.5.8
     permissions:

renovate.json → .renovaterc.json

@@ -1,6 +1,4 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "config:recommended"
-  ]
+  "extends": ["config:recommended"]
 }

PathlingS3Import.sln

@@ -7,6 +7,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{7C72D930-9EE
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PathlingS3Import", "src\PathlingS3Import\PathlingS3Import.csproj", "{1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PathlingS3Import.Tests.E2E", "src\PathlingS3Import.Tests.E2E\PathlingS3Import.Tests.E2E.csproj", "{91793B35-10E1-49D7-9D97-D69B6C083BEA}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -20,8 +22,13 @@ Global
 		{1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{91793B35-10E1-49D7-9D97-D69B6C083BEA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{91793B35-10E1-49D7-9D97-D69B6C083BEA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{91793B35-10E1-49D7-9D97-D69B6C083BEA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{91793B35-10E1-49D7-9D97-D69B6C083BEA}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB} = {7C72D930-9EE6-4CA3-8F01-40429262EE29}
+		{91793B35-10E1-49D7-9D97-D69B6C083BEA} = {7C72D930-9EE6-4CA3-8F01-40429262EE29}
 	EndGlobalSection
 EndGlobal

README.md

@@ -1,5 +1,8 @@
 # pathling-s3-import
 
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/miracum/pathling-s3-import/badge)](https://api.securityscorecards.dev/projects/github.com/miracum/pathling-s3-import)
+[![SLSA 3](https://slsa.dev/images/gh-badge-level3.svg)](https://slsa.dev)
+
 Tool for automatically [$import'ing](https://pathling.csiro.au/docs/server/operations/import) lists of ndjson FHIR resources from an S3-compatible bucket into a Pathling server.
 
 ## Usage
@@ -9,3 +12,31 @@ See the help text of the command by simply running:
 ```sh
 docker run --rm -it ghcr.io/miracum/pathling-s3-import:v1.1.1
 ```
+
+## Development
+
+Launch development fixtures:
+
+```sh
+docker compose up
+```
+
+Install dependencies
+
+```sh
+dotnet restore
+dotnet tool restore
+```
+
+Start the tool
+
+```sh
+dotnet run --project src/PathlingS3Import/ -- \
+    --s3-endpoint=http://localhost:9000 \
+    --pathling-server-base-url=http://localhost:8082/fhir \
+    --s3-access-key=admin \
+    --s3-secret-key=miniopass \
+    --s3-bucket-name=fhir \
+    --s3-object-name-prefix=staging/ \
+    --dry-run=false
+```

compose.yaml

@@ -7,9 +7,43 @@ services:
       MINIO_ROOT_PASSWORD: "miniopass" # gitleaks:allow
       MINIO_DEFAULT_BUCKETS: "fhir"
     ports:
-      - "127.0.0.1:9000:9000"
+      - "9000:9000"
       - "127.0.0.1:9001:9001"
 
+  wait-for-minio:
+    image: docker.io/curlimages/curl:8.6.0@sha256:c3b8bee303c6c6beed656cfc921218c529d65aa61114eb9e27c62047a1271b9b
+    restart: "no"
+    environment:
+      MINIO_ENDPOINT_URL: http://minio:9000
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        until [ "$(curl -s -o /dev/null -L -w "%{http_code}" "$$MINIO_ENDPOINT_URL/minio/health/live")" == "200" ]; do
+            echo "$(date): Waiting for minio server @ $$MINIO_ENDPOINT_URL to be up";
+            sleep 5;
+        done;
+    depends_on:
+      minio:
+        condition: service_started
+
+  minio-client:
+    image: docker.io/bitnami/minio-client:2024.2.16-debian-12-r2@sha256:ccef919b89fcf8f429a2e61c30c68ce1f091184e6d43545164667d340dd3a6fb
+    environment:
+      MINIO_SERVER_ACCESS_KEY: admin
+      # kics-scan ignore-line
+      MINIO_SERVER_SECRET_KEY: miniopass # gitleaks:allow
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        mc alias set minio http://minio:9000 $${MINIO_SERVER_ACCESS_KEY} $${MINIO_SERVER_SECRET_KEY}
+        mc mb minio/fhir
+        mc cp /tmp/data/bundle-0.ndjson /tmp/data/bundle-1.ndjson minio/fhir/staging/Patient/
+    depends_on:
+      wait-for-minio:
+        condition: service_completed_successfully
+    volumes:
+      - $PWD/hack/data/:/tmp/data/:ro
+
   pathling:
     image: docker.io/aehrc/pathling:6.4.2@sha256:9b8ee32d4b8bb40192d6bf25814492a616153a0df15d178c286db9ec80c1c85e
     environment:
@@ -24,4 +58,23 @@ services:
       fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem"
       fs.s3a.path.style.access: "true"
     ports:
-      - "127.0.0.1:8082:8080"
+      - "8082:8080"
+    depends_on:
+      minio-client:
+        condition: service_completed_successfully
+
+  wait-for-pathling:
+    image: docker.io/curlimages/curl:8.6.0@sha256:c3b8bee303c6c6beed656cfc921218c529d65aa61114eb9e27c62047a1271b9b
+    restart: "no"
+    environment:
+      PATHLING_URL: http://pathling:8080
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        until [ "$(curl -s -o /dev/null -L -w "%{http_code}" "$$PATHLING_URL/fhir/metadata")" == "200" ]; do
+            echo "$(date): Waiting for pathling server @ $$PATHLING_URL to be up";
+            sleep 5;
+        done;
+    depends_on:
+      pathling:
+        condition: service_started

hack/data/bundle-0.ndjson

@@ -0,0 +1,2 @@
+{"resourceType":"Patient","id":"pid.999","meta":{"source":"#p21"},"identifier":[{"use":"usual","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"MR"}]},"system":"https://miracum.org/fhir/NamingSystem/identifier/PatientId","value":"pid.999"},{"use":"official","type":{"coding":[{"system":"http://fhir.de/CodeSystem/identifier-type-de-basis","code":"GKV"}]},"system":"http://fhir.de/NamingSystem/gkv/kvid-10","value":"5678","assigner":{"identifier":{"use":"official","system":"http://fhir.de/NamingSystem/arge-ik/iknr","value":"109905113"}}}],"name":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/data-absent-reason","valueCode":"unsupported"}]}],"gender":"unknown","birthDate":"1941-01-01","deceasedDateTime":"2018-08-05T21:28:00+02:00","address":[{"type":"physical","city":"Buschau","postalCode":"12365"}]}
+{"resourceType":"Patient","id":"pid-02cb8631-8342-4343-84d7-cd364e0ab101","identifier":[{"system":"http://example.com/fhir/id","value":"4ca676c2-e2d0-4726-b6dd-83eae57dcf50"}],"name":[{"family":"Wisozk","given":["Mariana"]}],"gender":"male","birthDate":"2005-12-08"}

hack/data/bundle-1.ndjson

@@ -0,0 +1,2 @@
+{"resourceType":"Patient","id":"pid.999","meta":{"source":"#p21"},"identifier":[{"use":"usual","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"MR"}]},"system":"https://miracum.org/fhir/NamingSystem/identifier/PatientId","value":"pid.999"},{"use":"official","type":{"coding":[{"system":"http://fhir.de/CodeSystem/identifier-type-de-basis","code":"GKV"}]},"system":"http://fhir.de/NamingSystem/gkv/kvid-10","value":"5678","assigner":{"identifier":{"use":"official","system":"http://fhir.de/NamingSystem/arge-ik/iknr","value":"109905113"}}}],"name":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/data-absent-reason","valueCode":"unsupported"}]}],"gender":"unknown","birthDate":"1941-01-01","deceasedDateTime":"2018-08-05T21:28:00+02:00","address":[{"type":"physical","city":"Buschau","postalCode":"12365"}]}
+{"resourceType":"Patient","id":"new-id-123","identifier":[{"system":"http://example.com/fhir/id","value":"4ca676c2-e2d0-4726-b6dd-83eae57dcf50"}],"name":[{"family":"Wisozk","given":["Mariana"]}],"gender":"male","birthDate":"2005-12-08"}

src/PathlingS3Import.Tests.E2E/PathlingS3Import.Tests.E2E.csproj

@@ -0,0 +1,26 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
+    <PackageReference Include="FluentAssertions" Version="6.12.0" />
+    <PackageReference Include="Hl7.Fhir.R4" Version="5.6.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+    <PackageReference Include="Testcontainers" Version="3.7.0" />
+    <PackageReference Include="xunit" Version="2.5.3" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Using Include="Xunit" />
+  </ItemGroup>
+
+</Project>

src/PathlingS3Import.Tests.E2E/Test.cs

@@ -0,0 +1,81 @@
+using DotNet.Testcontainers.Builders;
+using FluentAssertions;
+using Hl7.Fhir.Model;
+using Hl7.Fhir.Rest;
+using Xunit.Abstractions;
+using Task = System.Threading.Tasks.Task;
+
+namespace PathlingS3Import.Tests.E2E;
+
+public class Tests(ITestOutputHelper output)
+{
+    private readonly ITestOutputHelper output = output;
+
+    [Fact]
+    public async Task StartImportTool_WithRunningPathlingServerAndMinio_ShouldCreateExpectedNumberOfResources()
+    {
+        // this test requires the dev fixtures to be running on their default ports as well as
+        // a PathlingS3Import image to exist.
+
+        using var stdoutStream = new MemoryStream();
+        using var stderrStream = new MemoryStream();
+        using var consumer = Consume.RedirectStdoutAndStderrToStream(stdoutStream, stderrStream);
+
+        var pathlingServerBaseUrl = "http://host.docker.internal:8082/fhir";
+        var resourceType = ResourceType.Patient;
+
+        string[] args =
+        [
+            "--s3-endpoint=http://host.docker.internal:9000",
+            $"--pathling-server-base-url={pathlingServerBaseUrl}",
+            "--s3-access-key=admin",
+            "--s3-secret-key=miniopass",
+            "--s3-bucket-name=fhir",
+            "--s3-object-name-prefix=staging/",
+            $"--import-resource-type={resourceType}",
+            "--dry-run=false"
+        ];
+
+        var testImageTag =
+            Environment.GetEnvironmentVariable("PATHLING_S3_IMPORT_IMAGE_TAG") ?? "test";
+
+        var testContainer = new ContainerBuilder()
+            .WithImage($"ghcr.io/miracum/pathling-s3-import:{testImageTag}")
+            .WithCommand(args)
+            .WithOutputConsumer(consumer)
+            .WithExtraHost("host.docker.internal", "host-gateway")
+            .Build();
+
+        await testContainer.StartAsync();
+
+        var exitCode = await testContainer.GetExitCodeAsync();
+
+        output.WriteLine("Test container exited");
+
+        consumer.Stdout.Seek(0, SeekOrigin.Begin);
+        using var stdoutReader = new StreamReader(consumer.Stdout);
+        var stdout = stdoutReader.ReadToEnd();
+        output.WriteLine(stdout);
+
+        exitCode.Should().Be(0);
+
+        // use a different base URL since this test isn't run inside
+        // a container. Slightly ugly.
+        using var fhirClient = new FhirClient(
+            "http://localhost:8082/fhir",
+            settings: new()
+            {
+                PreferredFormat = ResourceFormat.Json,
+                Timeout = (int)TimeSpan.FromSeconds(60).TotalMilliseconds
+            }
+        );
+
+        var response = await fhirClient.SearchAsync(
+            resourceType.ToString(),
+            summary: SummaryType.Count
+        );
+
+        response.Should().NotBeNull();
+        response!.Total.Should().Be(3);
+    }
+}