Skip to content

Commit

Permalink
fix: improved logging and added e2e tests (#7)
Browse files Browse the repository at this point in the history
* feat: improved logging

* test(e2e): added e2e tests

* chore: formatted
  • Loading branch information
chgl authored Feb 24, 2024
1 parent 422d260 commit d48cb91
Show file tree
Hide file tree
Showing 11 changed files with 507 additions and 239 deletions.
42 changes: 42 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,48 @@ jobs:
secrets:
github-token: ${{ secrets.GITHUB_TOKEN }}

test:
needs:
- build
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

- name: Download container image
if: ${{ github.event_name == 'pull_request' }}
uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: ${{ needs.build.outputs.image-slug }}
path: /tmp

- name: Load image
if: ${{ github.event_name == 'pull_request' }}
run: |
ls -lar /tmp
docker load --input /tmp/image.tar
docker image ls -a
- name: Start compose fixtures
run: |
docker compose up wait-for-pathling
- name: Install .NET
uses: actions/setup-dotnet@4d6c8fcf3c8f7a60068d26b594648e99df24cee3 # v4.0.0
with:
dotnet-version: "8.0.x"

- name: Run tests
env:
PATHLING_S3_IMPORT_IMAGE_TAG: ${{ needs.build.outputs.image-version }}
run: dotnet test src/PathlingS3Import.Tests.E2E --configuration=Release -l "console;verbosity=detailed"

- name: Print compose logs
if: always()
run: |
docker compose logs
docker compose down --volumes --remove-orphans
lint:
uses: miracum/.github/.github/workflows/standard-lint.yaml@d09a237ae62959d3cf89d526a035fbd9d9d816ee # v1.5.8
permissions:
Expand Down
4 changes: 1 addition & 3 deletions renovate.json → .renovaterc.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": [
"config:recommended"
]
"extends": ["config:recommended"]
}
366 changes: 183 additions & 183 deletions LICENSE

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions PathlingS3Import.sln
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{7C72D930-9EE
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PathlingS3Import", "src\PathlingS3Import\PathlingS3Import.csproj", "{1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PathlingS3Import.Tests.E2E", "src\PathlingS3Import.Tests.E2E\PathlingS3Import.Tests.E2E.csproj", "{91793B35-10E1-49D7-9D97-D69B6C083BEA}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand All @@ -20,8 +22,13 @@ Global
{1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}.Release|Any CPU.Build.0 = Release|Any CPU
{91793B35-10E1-49D7-9D97-D69B6C083BEA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{91793B35-10E1-49D7-9D97-D69B6C083BEA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{91793B35-10E1-49D7-9D97-D69B6C083BEA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{91793B35-10E1-49D7-9D97-D69B6C083BEA}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB} = {7C72D930-9EE6-4CA3-8F01-40429262EE29}
{91793B35-10E1-49D7-9D97-D69B6C083BEA} = {7C72D930-9EE6-4CA3-8F01-40429262EE29}
EndGlobalSection
EndGlobal
31 changes: 31 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# pathling-s3-import

[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/miracum/pathling-s3-import/badge)](https://api.securityscorecards.dev/projects/github.com/miracum/pathling-s3-import)
[![SLSA 3](https://slsa.dev/images/gh-badge-level3.svg)](https://slsa.dev)

Tool for automatically [$import'ing](https://pathling.csiro.au/docs/server/operations/import) lists of ndjson FHIR resources from an S3-compatible bucket into a Pathling server.

## Usage
Expand All @@ -9,3 +12,31 @@ See the help text of the command by simply running:
```sh
docker run --rm -it ghcr.io/miracum/pathling-s3-import:v1.1.1
```

## Development

Launch development fixtures:

```sh
docker compose up
```

Install dependencies

```sh
dotnet restore
dotnet tool restore
```

Start the tool

```sh
dotnet run --project src/PathlingS3Import/ -- \
--s3-endpoint=http://localhost:9000 \
--pathling-server-base-url=http://localhost:8082/fhir \
--s3-access-key=admin \
--s3-secret-key=miniopass \
--s3-bucket-name=fhir \
--s3-object-name-prefix=staging/ \
--dry-run=false
```
57 changes: 55 additions & 2 deletions compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,43 @@ services:
MINIO_ROOT_PASSWORD: "miniopass" # gitleaks:allow
MINIO_DEFAULT_BUCKETS: "fhir"
ports:
- "127.0.0.1:9000:9000"
- "9000:9000"
- "127.0.0.1:9001:9001"

wait-for-minio:
image: docker.io/curlimages/curl:8.6.0@sha256:c3b8bee303c6c6beed656cfc921218c529d65aa61114eb9e27c62047a1271b9b
restart: "no"
environment:
MINIO_ENDPOINT_URL: http://minio:9000
entrypoint: ["/bin/sh", "-c"]
command:
- |
until [ "$(curl -s -o /dev/null -L -w "%{http_code}" "$$MINIO_ENDPOINT_URL/minio/health/live")" == "200" ]; do
echo "$(date): Waiting for minio server @ $$MINIO_ENDPOINT_URL to be up";
sleep 5;
done;
depends_on:
minio:
condition: service_started

minio-client:
image: docker.io/bitnami/minio-client:2024.2.16-debian-12-r2@sha256:ccef919b89fcf8f429a2e61c30c68ce1f091184e6d43545164667d340dd3a6fb
environment:
MINIO_SERVER_ACCESS_KEY: admin
# kics-scan ignore-line
MINIO_SERVER_SECRET_KEY: miniopass # gitleaks:allow
entrypoint: ["/bin/sh", "-c"]
command:
- |
mc alias set minio http://minio:9000 $${MINIO_SERVER_ACCESS_KEY} $${MINIO_SERVER_SECRET_KEY}
mc mb minio/fhir
mc cp /tmp/data/bundle-0.ndjson /tmp/data/bundle-1.ndjson minio/fhir/staging/Patient/
depends_on:
wait-for-minio:
condition: service_completed_successfully
volumes:
- $PWD/hack/data/:/tmp/data/:ro

pathling:
image: docker.io/aehrc/pathling:6.4.2@sha256:9b8ee32d4b8bb40192d6bf25814492a616153a0df15d178c286db9ec80c1c85e
environment:
Expand All @@ -24,4 +58,23 @@ services:
fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem"
fs.s3a.path.style.access: "true"
ports:
- "127.0.0.1:8082:8080"
- "8082:8080"
depends_on:
minio-client:
condition: service_completed_successfully

wait-for-pathling:
image: docker.io/curlimages/curl:8.6.0@sha256:c3b8bee303c6c6beed656cfc921218c529d65aa61114eb9e27c62047a1271b9b
restart: "no"
environment:
PATHLING_URL: http://pathling:8080
entrypoint: ["/bin/sh", "-c"]
command:
- |
until [ "$(curl -s -o /dev/null -L -w "%{http_code}" "$$PATHLING_URL/fhir/metadata")" == "200" ]; do
echo "$(date): Waiting for pathling server @ $$PATHLING_URL to be up";
sleep 5;
done;
depends_on:
pathling:
condition: service_started
2 changes: 2 additions & 0 deletions hack/data/bundle-0.ndjson
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{"resourceType":"Patient","id":"pid.999","meta":{"source":"#p21"},"identifier":[{"use":"usual","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"MR"}]},"system":"https://miracum.org/fhir/NamingSystem/identifier/PatientId","value":"pid.999"},{"use":"official","type":{"coding":[{"system":"http://fhir.de/CodeSystem/identifier-type-de-basis","code":"GKV"}]},"system":"http://fhir.de/NamingSystem/gkv/kvid-10","value":"5678","assigner":{"identifier":{"use":"official","system":"http://fhir.de/NamingSystem/arge-ik/iknr","value":"109905113"}}}],"name":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/data-absent-reason","valueCode":"unsupported"}]}],"gender":"unknown","birthDate":"1941-01-01","deceasedDateTime":"2018-08-05T21:28:00+02:00","address":[{"type":"physical","city":"Buschau","postalCode":"12365"}]}
{"resourceType":"Patient","id":"pid-02cb8631-8342-4343-84d7-cd364e0ab101","identifier":[{"system":"http://example.com/fhir/id","value":"4ca676c2-e2d0-4726-b6dd-83eae57dcf50"}],"name":[{"family":"Wisozk","given":["Mariana"]}],"gender":"male","birthDate":"2005-12-08"}
2 changes: 2 additions & 0 deletions hack/data/bundle-1.ndjson
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{"resourceType":"Patient","id":"pid.999","meta":{"source":"#p21"},"identifier":[{"use":"usual","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"MR"}]},"system":"https://miracum.org/fhir/NamingSystem/identifier/PatientId","value":"pid.999"},{"use":"official","type":{"coding":[{"system":"http://fhir.de/CodeSystem/identifier-type-de-basis","code":"GKV"}]},"system":"http://fhir.de/NamingSystem/gkv/kvid-10","value":"5678","assigner":{"identifier":{"use":"official","system":"http://fhir.de/NamingSystem/arge-ik/iknr","value":"109905113"}}}],"name":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/data-absent-reason","valueCode":"unsupported"}]}],"gender":"unknown","birthDate":"1941-01-01","deceasedDateTime":"2018-08-05T21:28:00+02:00","address":[{"type":"physical","city":"Buschau","postalCode":"12365"}]}
{"resourceType":"Patient","id":"new-id-123","identifier":[{"system":"http://example.com/fhir/id","value":"4ca676c2-e2d0-4726-b6dd-83eae57dcf50"}],"name":[{"family":"Wisozk","given":["Mariana"]}],"gender":"male","birthDate":"2005-12-08"}
26 changes: 26 additions & 0 deletions src/PathlingS3Import.Tests.E2E/PathlingS3Import.Tests.E2E.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.0" />
<PackageReference Include="FluentAssertions" Version="6.12.0" />
<PackageReference Include="Hl7.Fhir.R4" Version="5.6.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
<PackageReference Include="Testcontainers" Version="3.7.0" />
<PackageReference Include="xunit" Version="2.5.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" />
</ItemGroup>

<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>

</Project>
81 changes: 81 additions & 0 deletions src/PathlingS3Import.Tests.E2E/Test.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
using DotNet.Testcontainers.Builders;
using FluentAssertions;
using Hl7.Fhir.Model;
using Hl7.Fhir.Rest;
using Xunit.Abstractions;
using Task = System.Threading.Tasks.Task;

namespace PathlingS3Import.Tests.E2E;

public class Tests(ITestOutputHelper output)
{
private readonly ITestOutputHelper output = output;

[Fact]
public async Task StartImportTool_WithRunningPathlingServerAndMinio_ShouldCreateExpectedNumberOfResources()
{
// this test requires the dev fixtures to be running on their default ports as well as
// a PathlingS3Import image to exist.

using var stdoutStream = new MemoryStream();
using var stderrStream = new MemoryStream();
using var consumer = Consume.RedirectStdoutAndStderrToStream(stdoutStream, stderrStream);

var pathlingServerBaseUrl = "http://host.docker.internal:8082/fhir";
var resourceType = ResourceType.Patient;

string[] args =
[
"--s3-endpoint=http://host.docker.internal:9000",
$"--pathling-server-base-url={pathlingServerBaseUrl}",
"--s3-access-key=admin",
"--s3-secret-key=miniopass",
"--s3-bucket-name=fhir",
"--s3-object-name-prefix=staging/",
$"--import-resource-type={resourceType}",
"--dry-run=false"
];

var testImageTag =
Environment.GetEnvironmentVariable("PATHLING_S3_IMPORT_IMAGE_TAG") ?? "test";

var testContainer = new ContainerBuilder()
.WithImage($"ghcr.io/miracum/pathling-s3-import:{testImageTag}")
.WithCommand(args)
.WithOutputConsumer(consumer)
.WithExtraHost("host.docker.internal", "host-gateway")
.Build();

await testContainer.StartAsync();

var exitCode = await testContainer.GetExitCodeAsync();

output.WriteLine("Test container exited");

consumer.Stdout.Seek(0, SeekOrigin.Begin);
using var stdoutReader = new StreamReader(consumer.Stdout);
var stdout = stdoutReader.ReadToEnd();
output.WriteLine(stdout);

exitCode.Should().Be(0);

// use a different base URL since this test isn't run inside
// a container. Slightly ugly.
using var fhirClient = new FhirClient(
"http://localhost:8082/fhir",
settings: new()
{
PreferredFormat = ResourceFormat.Json,
Timeout = (int)TimeSpan.FromSeconds(60).TotalMilliseconds
}
);

var response = await fhirClient.SearchAsync(
resourceType.ToString(),
summary: SummaryType.Count
);

response.Should().NotBeNull();
response!.Total.Should().Be(3);
}
}
Loading

0 comments on commit d48cb91

Please sign in to comment.