diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8b50d4b..4b8908f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,6 +26,48 @@ jobs: secrets: github-token: ${{ secrets.GITHUB_TOKEN }} + test: + needs: + - build + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Download container image + if: ${{ github.event_name == 'pull_request' }} + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2 + with: + name: ${{ needs.build.outputs.image-slug }} + path: /tmp + + - name: Load image + if: ${{ github.event_name == 'pull_request' }} + run: | + ls -lar /tmp + docker load --input /tmp/image.tar + docker image ls -a + + - name: Start compose fixtures + run: | + docker compose up wait-for-pathling + + - name: Install .NET + uses: actions/setup-dotnet@4d6c8fcf3c8f7a60068d26b594648e99df24cee3 # v4.0.0 + with: + dotnet-version: "8.0.x" + + - name: Run tests + env: + PATHLING_S3_IMPORT_IMAGE_TAG: ${{ needs.build.outputs.image-version }} + run: dotnet test src/PathlingS3Import.Tests.E2E --configuration=Release -l "console;verbosity=detailed" + + - name: Print compose logs + if: always() + run: | + docker compose logs + docker compose down --volumes --remove-orphans + lint: uses: miracum/.github/.github/workflows/standard-lint.yaml@d09a237ae62959d3cf89d526a035fbd9d9d816ee # v1.5.8 permissions: diff --git a/renovate.json b/.renovaterc.json similarity index 61% rename from renovate.json rename to .renovaterc.json index 5db72dd..22a9943 100644 --- a/renovate.json +++ b/.renovaterc.json @@ -1,6 +1,4 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "config:recommended" - ] + "extends": ["config:recommended"] } diff --git a/PathlingS3Import.sln b/PathlingS3Import.sln index 32a88c4..37bc33f 100644 --- a/PathlingS3Import.sln +++ b/PathlingS3Import.sln @@ -7,6 +7,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{7C72D930-9EE EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PathlingS3Import", "src\PathlingS3Import\PathlingS3Import.csproj", "{1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PathlingS3Import.Tests.E2E", "src\PathlingS3Import.Tests.E2E\PathlingS3Import.Tests.E2E.csproj", "{91793B35-10E1-49D7-9D97-D69B6C083BEA}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -20,8 +22,13 @@ Global {1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}.Debug|Any CPU.Build.0 = Debug|Any CPU {1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}.Release|Any CPU.ActiveCfg = Release|Any CPU {1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}.Release|Any CPU.Build.0 = Release|Any CPU + {91793B35-10E1-49D7-9D97-D69B6C083BEA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {91793B35-10E1-49D7-9D97-D69B6C083BEA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {91793B35-10E1-49D7-9D97-D69B6C083BEA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {91793B35-10E1-49D7-9D97-D69B6C083BEA}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB} = {7C72D930-9EE6-4CA3-8F01-40429262EE29} + {91793B35-10E1-49D7-9D97-D69B6C083BEA} = {7C72D930-9EE6-4CA3-8F01-40429262EE29} EndGlobalSection EndGlobal diff --git a/README.md b/README.md index dc205a6..b4b08e5 100644 --- a/README.md +++ b/README.md @@ -9,3 +9,31 @@ See the help text of the command by simply running: ```sh docker run --rm -it ghcr.io/miracum/pathling-s3-import:v1.1.1 ``` + +## Development + +Launch development fixtures: + +```sh +docker compose up +``` + +Install dependencies + +```sh +dotnet restore +dotnet tool restore +``` + +Start the tool + +```sh +dotnet run --project src/PathlingS3Import/ -- \ + --s3-endpoint=http://localhost:9000 \ + --pathling-server-base-url=http://localhost:8082/fhir \ + --s3-access-key=admin \ + --s3-secret-key=miniopass \ + --s3-bucket-name=fhir \ + --s3-object-name-prefix=staging/ \ + --dry-run=false +``` diff --git a/compose.yaml b/compose.yaml index d9826a2..13ebb07 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,9 +7,41 @@ services: MINIO_ROOT_PASSWORD: "miniopass" # gitleaks:allow MINIO_DEFAULT_BUCKETS: "fhir" ports: - - "127.0.0.1:9000:9000" + - "9000:9000" - "127.0.0.1:9001:9001" + wait-for-minio: + image: docker.io/curlimages/curl:8.6.0@sha256:c3b8bee303c6c6beed656cfc921218c529d65aa61114eb9e27c62047a1271b9b + restart: "no" + environment: + MINIO_ENDPOINT_URL: http://minio:9000 + entrypoint: ["/bin/sh", "-c"] + command: + - | + until [ "$(curl -s -o /dev/null -L -w "%{http_code}" "$$MINIO_ENDPOINT_URL/minio/health/live")" == "200" ]; do + echo "$(date): Waiting for minio server @ $$MINIO_ENDPOINT_URL to be up"; + sleep 5; + done; + + minio-client: + image: docker.io/bitnami/minio-client:2024.2.16-debian-12-r2@sha256:ccef919b89fcf8f429a2e61c30c68ce1f091184e6d43545164667d340dd3a6fb + environment: + MINIO_SERVER_ACCESS_KEY: admin + # kics-scan ignore-line + MINIO_SERVER_SECRET_KEY: miniopass # gitleaks:allow + entrypoint: ["/bin/sh", "-c"] + command: + - | + mc alias set minio http://minio:9000 $${MINIO_SERVER_ACCESS_KEY} $${MINIO_SERVER_SECRET_KEY} + mc cp /tmp/data/bundle-0.ndjson /tmp/data/bundle-1.ndjson minio/fhir/staging/Patient/ + depends_on: + wait-for-minio: + condition: service_completed_successfully + minio: + condition: service_started + volumes: + - $PWD/hack/data/:/tmp/data/:ro + pathling: image: docker.io/aehrc/pathling:6.4.2@sha256:9b8ee32d4b8bb40192d6bf25814492a616153a0df15d178c286db9ec80c1c85e environment: @@ -24,4 +56,23 @@ services: fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" fs.s3a.path.style.access: "true" ports: - - "127.0.0.1:8082:8080" + - "8082:8080" + depends_on: + minio-client: + condition: service_completed_successfully + + wait-for-pathling: + image: docker.io/curlimages/curl:8.6.0@sha256:c3b8bee303c6c6beed656cfc921218c529d65aa61114eb9e27c62047a1271b9b + restart: "no" + environment: + PATHLING_URL: http://pathling:8080 + entrypoint: ["/bin/sh", "-c"] + command: + - | + until [ "$(curl -s -o /dev/null -L -w "%{http_code}" "$$PATHLING_URL/fhir/metadata")" == "200" ]; do + echo "$(date): Waiting for pathling server @ $$PATHLING_URL to be up"; + sleep 5; + done; + depends_on: + pathling: + condition: service_started diff --git a/hack/data/bundle-0.ndjson b/hack/data/bundle-0.ndjson new file mode 100644 index 0000000..04508bf --- /dev/null +++ b/hack/data/bundle-0.ndjson @@ -0,0 +1,2 @@ +{"resourceType":"Patient","id":"pid.999","meta":{"source":"#p21"},"identifier":[{"use":"usual","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"MR"}]},"system":"https://miracum.org/fhir/NamingSystem/identifier/PatientId","value":"pid.999"},{"use":"official","type":{"coding":[{"system":"http://fhir.de/CodeSystem/identifier-type-de-basis","code":"GKV"}]},"system":"http://fhir.de/NamingSystem/gkv/kvid-10","value":"5678","assigner":{"identifier":{"use":"official","system":"http://fhir.de/NamingSystem/arge-ik/iknr","value":"109905113"}}}],"name":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/data-absent-reason","valueCode":"unsupported"}]}],"gender":"unknown","birthDate":"1941-01-01","deceasedDateTime":"2018-08-05T21:28:00+02:00","address":[{"type":"physical","city":"Buschau","postalCode":"12365"}]} +{"resourceType":"Patient","id":"pid-02cb8631-8342-4343-84d7-cd364e0ab101","identifier":[{"system":"http://example.com/fhir/id","value":"4ca676c2-e2d0-4726-b6dd-83eae57dcf50"}],"name":[{"family":"Wisozk","given":["Mariana"]}],"gender":"male","birthDate":"2005-12-08"} diff --git a/hack/data/bundle-1.ndjson b/hack/data/bundle-1.ndjson new file mode 100644 index 0000000..563a4c1 --- /dev/null +++ b/hack/data/bundle-1.ndjson @@ -0,0 +1,2 @@ +{"resourceType":"Patient","id":"pid.999","meta":{"source":"#p21"},"identifier":[{"use":"usual","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"MR"}]},"system":"https://miracum.org/fhir/NamingSystem/identifier/PatientId","value":"pid.999"},{"use":"official","type":{"coding":[{"system":"http://fhir.de/CodeSystem/identifier-type-de-basis","code":"GKV"}]},"system":"http://fhir.de/NamingSystem/gkv/kvid-10","value":"5678","assigner":{"identifier":{"use":"official","system":"http://fhir.de/NamingSystem/arge-ik/iknr","value":"109905113"}}}],"name":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/data-absent-reason","valueCode":"unsupported"}]}],"gender":"unknown","birthDate":"1941-01-01","deceasedDateTime":"2018-08-05T21:28:00+02:00","address":[{"type":"physical","city":"Buschau","postalCode":"12365"}]} +{"resourceType":"Patient","id":"new-id-123","identifier":[{"system":"http://example.com/fhir/id","value":"4ca676c2-e2d0-4726-b6dd-83eae57dcf50"}],"name":[{"family":"Wisozk","given":["Mariana"]}],"gender":"male","birthDate":"2005-12-08"} diff --git a/src/PathlingS3Import.Tests.E2E/PathlingS3Import.Tests.E2E.csproj b/src/PathlingS3Import.Tests.E2E/PathlingS3Import.Tests.E2E.csproj new file mode 100644 index 0000000..fb8fc5e --- /dev/null +++ b/src/PathlingS3Import.Tests.E2E/PathlingS3Import.Tests.E2E.csproj @@ -0,0 +1,26 @@ + + + + net8.0 + enable + enable + + false + true + + + + + + + + + + + + + + + + + diff --git a/src/PathlingS3Import.Tests.E2E/Test.cs b/src/PathlingS3Import.Tests.E2E/Test.cs new file mode 100644 index 0000000..47e96e9 --- /dev/null +++ b/src/PathlingS3Import.Tests.E2E/Test.cs @@ -0,0 +1,90 @@ +using DotNet.Testcontainers; +using DotNet.Testcontainers.Builders; +using FluentAssertions; +using Hl7.Fhir.Model; +using Hl7.Fhir.Rest; +using Xunit.Abstractions; +using Task = System.Threading.Tasks.Task; + +namespace PathlingS3Import.Tests.E2E; + +public class Tests +{ + private readonly ITestOutputHelper output; + + public Tests(ITestOutputHelper output) + { + this.output = output; + } + + [Fact] + public async Task StartImportTool_WithRunningPathlingServerAndMinio_ShouldCreateExpectedNumberOfResources() + { + // this test requires the dev fixtures to be running on their default ports as well as + // a PathlingS3Import image to exist. + + using var stdoutStream = new MemoryStream(); + using var stderrStream = new MemoryStream(); + using var consumer = Consume.RedirectStdoutAndStderrToStream( + stdoutStream, + stderrStream + ); + + var pathlingServerBaseUrl = "http://host.docker.internal:8082/fhir"; + var resourceType = ResourceType.Patient; + + string[] args = + [ + "--s3-endpoint=http://host.docker.internal:9000", + $"--pathling-server-base-url={pathlingServerBaseUrl}", + "--s3-access-key=admin", + "--s3-secret-key=miniopass", + "--s3-bucket-name=fhir", + "--s3-object-name-prefix=staging/", + $"--import-resource-type={resourceType}", + "--dry-run=false" + ]; + + var testImageTag = + Environment.GetEnvironmentVariable("PATHLING_S3_IMPORT_IMAGE_TAG") ?? "test"; + + var testContainer = new ContainerBuilder() + .WithImage($"ghcr.io/miracum/pathling-s3-import:{testImageTag}") + .WithCommand(args) + .WithOutputConsumer(consumer) + .WithExtraHost("host.docker.internal", "host-gateway") + .Build(); + + await testContainer.StartAsync(); + + var exitCode = await testContainer.GetExitCodeAsync(); + + output.WriteLine("Test container exited"); + + consumer.Stdout.Seek(0, SeekOrigin.Begin); + using var stdoutReader = new StreamReader(consumer.Stdout); + var stdout = stdoutReader.ReadToEnd(); + output.WriteLine(stdout); + + exitCode.Should().Be(0); + + // use a different base URL since this test isn't run inside + // a container. Slightly ugly. + using var fhirClient = new FhirClient( + "http://localhost:8082/fhir", + settings: new() + { + PreferredFormat = ResourceFormat.Json, + Timeout = (int)TimeSpan.FromSeconds(60).TotalMilliseconds + } + ); + + var response = await fhirClient.SearchAsync( + resourceType.ToString(), + summary: SummaryType.Count + ); + + response.Should().NotBeNull(); + response!.Total.Should().Be(3); + } +}