diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8b50d4b..4b8908f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,6 +26,48 @@ jobs: secrets: github-token: ${{ secrets.GITHUB_TOKEN }} + test: + needs: + - build + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Download container image + if: ${{ github.event_name == 'pull_request' }} + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2 + with: + name: ${{ needs.build.outputs.image-slug }} + path: /tmp + + - name: Load image + if: ${{ github.event_name == 'pull_request' }} + run: | + ls -lar /tmp + docker load --input /tmp/image.tar + docker image ls -a + + - name: Start compose fixtures + run: | + docker compose up wait-for-pathling + + - name: Install .NET + uses: actions/setup-dotnet@4d6c8fcf3c8f7a60068d26b594648e99df24cee3 # v4.0.0 + with: + dotnet-version: "8.0.x" + + - name: Run tests + env: + PATHLING_S3_IMPORT_IMAGE_TAG: ${{ needs.build.outputs.image-version }} + run: dotnet test src/PathlingS3Import.Tests.E2E --configuration=Release -l "console;verbosity=detailed" + + - name: Print compose logs + if: always() + run: | + docker compose logs + docker compose down --volumes --remove-orphans + lint: uses: miracum/.github/.github/workflows/standard-lint.yaml@d09a237ae62959d3cf89d526a035fbd9d9d816ee # v1.5.8 permissions: diff --git a/renovate.json b/.renovaterc.json similarity index 61% rename from renovate.json rename to .renovaterc.json index 5db72dd..22a9943 100644 --- a/renovate.json +++ b/.renovaterc.json @@ -1,6 +1,4 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "config:recommended" - ] + "extends": ["config:recommended"] } diff --git a/LICENSE b/LICENSE index 261eeb9..c61b663 100644 --- a/LICENSE +++ b/LICENSE @@ -2,180 +2,180 @@ Version 2.0, January 2004 http://www.apache.org/licenses/ - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" @@ -186,16 +186,16 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] +Copyright [yyyy] [name of copyright owner] - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/PathlingS3Import.sln b/PathlingS3Import.sln index 32a88c4..37bc33f 100644 --- a/PathlingS3Import.sln +++ b/PathlingS3Import.sln @@ -7,6 +7,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{7C72D930-9EE EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PathlingS3Import", "src\PathlingS3Import\PathlingS3Import.csproj", "{1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PathlingS3Import.Tests.E2E", "src\PathlingS3Import.Tests.E2E\PathlingS3Import.Tests.E2E.csproj", "{91793B35-10E1-49D7-9D97-D69B6C083BEA}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -20,8 +22,13 @@ Global {1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}.Debug|Any CPU.Build.0 = Debug|Any CPU {1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}.Release|Any CPU.ActiveCfg = Release|Any CPU {1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB}.Release|Any CPU.Build.0 = Release|Any CPU + {91793B35-10E1-49D7-9D97-D69B6C083BEA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {91793B35-10E1-49D7-9D97-D69B6C083BEA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {91793B35-10E1-49D7-9D97-D69B6C083BEA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {91793B35-10E1-49D7-9D97-D69B6C083BEA}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {1B97F255-F56D-4AE5-A25A-2C9C2AFEBFAB} = {7C72D930-9EE6-4CA3-8F01-40429262EE29} + {91793B35-10E1-49D7-9D97-D69B6C083BEA} = {7C72D930-9EE6-4CA3-8F01-40429262EE29} EndGlobalSection EndGlobal diff --git a/README.md b/README.md index dc205a6..a457ccc 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # pathling-s3-import +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/miracum/pathling-s3-import/badge)](https://api.securityscorecards.dev/projects/github.com/miracum/pathling-s3-import) +[![SLSA 3](https://slsa.dev/images/gh-badge-level3.svg)](https://slsa.dev) + Tool for automatically [$import'ing](https://pathling.csiro.au/docs/server/operations/import) lists of ndjson FHIR resources from an S3-compatible bucket into a Pathling server. ## Usage @@ -9,3 +12,31 @@ See the help text of the command by simply running: ```sh docker run --rm -it ghcr.io/miracum/pathling-s3-import:v1.1.1 ``` + +## Development + +Launch development fixtures: + +```sh +docker compose up +``` + +Install dependencies + +```sh +dotnet restore +dotnet tool restore +``` + +Start the tool + +```sh +dotnet run --project src/PathlingS3Import/ -- \ + --s3-endpoint=http://localhost:9000 \ + --pathling-server-base-url=http://localhost:8082/fhir \ + --s3-access-key=admin \ + --s3-secret-key=miniopass \ + --s3-bucket-name=fhir \ + --s3-object-name-prefix=staging/ \ + --dry-run=false +``` diff --git a/compose.yaml b/compose.yaml index d9826a2..6795f48 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,9 +7,43 @@ services: MINIO_ROOT_PASSWORD: "miniopass" # gitleaks:allow MINIO_DEFAULT_BUCKETS: "fhir" ports: - - "127.0.0.1:9000:9000" + - "9000:9000" - "127.0.0.1:9001:9001" + wait-for-minio: + image: docker.io/curlimages/curl:8.6.0@sha256:c3b8bee303c6c6beed656cfc921218c529d65aa61114eb9e27c62047a1271b9b + restart: "no" + environment: + MINIO_ENDPOINT_URL: http://minio:9000 + entrypoint: ["/bin/sh", "-c"] + command: + - | + until [ "$(curl -s -o /dev/null -L -w "%{http_code}" "$$MINIO_ENDPOINT_URL/minio/health/live")" == "200" ]; do + echo "$(date): Waiting for minio server @ $$MINIO_ENDPOINT_URL to be up"; + sleep 5; + done; + depends_on: + minio: + condition: service_started + + minio-client: + image: docker.io/bitnami/minio-client:2024.2.16-debian-12-r2@sha256:ccef919b89fcf8f429a2e61c30c68ce1f091184e6d43545164667d340dd3a6fb + environment: + MINIO_SERVER_ACCESS_KEY: admin + # kics-scan ignore-line + MINIO_SERVER_SECRET_KEY: miniopass # gitleaks:allow + entrypoint: ["/bin/sh", "-c"] + command: + - | + mc alias set minio http://minio:9000 $${MINIO_SERVER_ACCESS_KEY} $${MINIO_SERVER_SECRET_KEY} + mc mb minio/fhir + mc cp /tmp/data/bundle-0.ndjson /tmp/data/bundle-1.ndjson minio/fhir/staging/Patient/ + depends_on: + wait-for-minio: + condition: service_completed_successfully + volumes: + - $PWD/hack/data/:/tmp/data/:ro + pathling: image: docker.io/aehrc/pathling:6.4.2@sha256:9b8ee32d4b8bb40192d6bf25814492a616153a0df15d178c286db9ec80c1c85e environment: @@ -24,4 +58,23 @@ services: fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" fs.s3a.path.style.access: "true" ports: - - "127.0.0.1:8082:8080" + - "8082:8080" + depends_on: + minio-client: + condition: service_completed_successfully + + wait-for-pathling: + image: docker.io/curlimages/curl:8.6.0@sha256:c3b8bee303c6c6beed656cfc921218c529d65aa61114eb9e27c62047a1271b9b + restart: "no" + environment: + PATHLING_URL: http://pathling:8080 + entrypoint: ["/bin/sh", "-c"] + command: + - | + until [ "$(curl -s -o /dev/null -L -w "%{http_code}" "$$PATHLING_URL/fhir/metadata")" == "200" ]; do + echo "$(date): Waiting for pathling server @ $$PATHLING_URL to be up"; + sleep 5; + done; + depends_on: + pathling: + condition: service_started diff --git a/hack/data/bundle-0.ndjson b/hack/data/bundle-0.ndjson new file mode 100644 index 0000000..04508bf --- /dev/null +++ b/hack/data/bundle-0.ndjson @@ -0,0 +1,2 @@ +{"resourceType":"Patient","id":"pid.999","meta":{"source":"#p21"},"identifier":[{"use":"usual","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"MR"}]},"system":"https://miracum.org/fhir/NamingSystem/identifier/PatientId","value":"pid.999"},{"use":"official","type":{"coding":[{"system":"http://fhir.de/CodeSystem/identifier-type-de-basis","code":"GKV"}]},"system":"http://fhir.de/NamingSystem/gkv/kvid-10","value":"5678","assigner":{"identifier":{"use":"official","system":"http://fhir.de/NamingSystem/arge-ik/iknr","value":"109905113"}}}],"name":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/data-absent-reason","valueCode":"unsupported"}]}],"gender":"unknown","birthDate":"1941-01-01","deceasedDateTime":"2018-08-05T21:28:00+02:00","address":[{"type":"physical","city":"Buschau","postalCode":"12365"}]} +{"resourceType":"Patient","id":"pid-02cb8631-8342-4343-84d7-cd364e0ab101","identifier":[{"system":"http://example.com/fhir/id","value":"4ca676c2-e2d0-4726-b6dd-83eae57dcf50"}],"name":[{"family":"Wisozk","given":["Mariana"]}],"gender":"male","birthDate":"2005-12-08"} diff --git a/hack/data/bundle-1.ndjson b/hack/data/bundle-1.ndjson new file mode 100644 index 0000000..563a4c1 --- /dev/null +++ b/hack/data/bundle-1.ndjson @@ -0,0 +1,2 @@ +{"resourceType":"Patient","id":"pid.999","meta":{"source":"#p21"},"identifier":[{"use":"usual","type":{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/v2-0203","code":"MR"}]},"system":"https://miracum.org/fhir/NamingSystem/identifier/PatientId","value":"pid.999"},{"use":"official","type":{"coding":[{"system":"http://fhir.de/CodeSystem/identifier-type-de-basis","code":"GKV"}]},"system":"http://fhir.de/NamingSystem/gkv/kvid-10","value":"5678","assigner":{"identifier":{"use":"official","system":"http://fhir.de/NamingSystem/arge-ik/iknr","value":"109905113"}}}],"name":[{"extension":[{"url":"http://hl7.org/fhir/StructureDefinition/data-absent-reason","valueCode":"unsupported"}]}],"gender":"unknown","birthDate":"1941-01-01","deceasedDateTime":"2018-08-05T21:28:00+02:00","address":[{"type":"physical","city":"Buschau","postalCode":"12365"}]} +{"resourceType":"Patient","id":"new-id-123","identifier":[{"system":"http://example.com/fhir/id","value":"4ca676c2-e2d0-4726-b6dd-83eae57dcf50"}],"name":[{"family":"Wisozk","given":["Mariana"]}],"gender":"male","birthDate":"2005-12-08"} diff --git a/src/PathlingS3Import.Tests.E2E/PathlingS3Import.Tests.E2E.csproj b/src/PathlingS3Import.Tests.E2E/PathlingS3Import.Tests.E2E.csproj new file mode 100644 index 0000000..fb8fc5e --- /dev/null +++ b/src/PathlingS3Import.Tests.E2E/PathlingS3Import.Tests.E2E.csproj @@ -0,0 +1,26 @@ + + + + net8.0 + enable + enable + + false + true + + + + + + + + + + + + + + + + + diff --git a/src/PathlingS3Import.Tests.E2E/Test.cs b/src/PathlingS3Import.Tests.E2E/Test.cs new file mode 100644 index 0000000..afbb0f2 --- /dev/null +++ b/src/PathlingS3Import.Tests.E2E/Test.cs @@ -0,0 +1,81 @@ +using DotNet.Testcontainers.Builders; +using FluentAssertions; +using Hl7.Fhir.Model; +using Hl7.Fhir.Rest; +using Xunit.Abstractions; +using Task = System.Threading.Tasks.Task; + +namespace PathlingS3Import.Tests.E2E; + +public class Tests(ITestOutputHelper output) +{ + private readonly ITestOutputHelper output = output; + + [Fact] + public async Task StartImportTool_WithRunningPathlingServerAndMinio_ShouldCreateExpectedNumberOfResources() + { + // this test requires the dev fixtures to be running on their default ports as well as + // a PathlingS3Import image to exist. + + using var stdoutStream = new MemoryStream(); + using var stderrStream = new MemoryStream(); + using var consumer = Consume.RedirectStdoutAndStderrToStream(stdoutStream, stderrStream); + + var pathlingServerBaseUrl = "http://host.docker.internal:8082/fhir"; + var resourceType = ResourceType.Patient; + + string[] args = + [ + "--s3-endpoint=http://host.docker.internal:9000", + $"--pathling-server-base-url={pathlingServerBaseUrl}", + "--s3-access-key=admin", + "--s3-secret-key=miniopass", + "--s3-bucket-name=fhir", + "--s3-object-name-prefix=staging/", + $"--import-resource-type={resourceType}", + "--dry-run=false" + ]; + + var testImageTag = + Environment.GetEnvironmentVariable("PATHLING_S3_IMPORT_IMAGE_TAG") ?? "test"; + + var testContainer = new ContainerBuilder() + .WithImage($"ghcr.io/miracum/pathling-s3-import:{testImageTag}") + .WithCommand(args) + .WithOutputConsumer(consumer) + .WithExtraHost("host.docker.internal", "host-gateway") + .Build(); + + await testContainer.StartAsync(); + + var exitCode = await testContainer.GetExitCodeAsync(); + + output.WriteLine("Test container exited"); + + consumer.Stdout.Seek(0, SeekOrigin.Begin); + using var stdoutReader = new StreamReader(consumer.Stdout); + var stdout = stdoutReader.ReadToEnd(); + output.WriteLine(stdout); + + exitCode.Should().Be(0); + + // use a different base URL since this test isn't run inside + // a container. Slightly ugly. + using var fhirClient = new FhirClient( + "http://localhost:8082/fhir", + settings: new() + { + PreferredFormat = ResourceFormat.Json, + Timeout = (int)TimeSpan.FromSeconds(60).TotalMilliseconds + } + ); + + var response = await fhirClient.SearchAsync( + resourceType.ToString(), + summary: SummaryType.Count + ); + + response.Should().NotBeNull(); + response!.Total.Should().Be(3); + } +} diff --git a/src/PathlingS3Import/ImportCliCommand.cs b/src/PathlingS3Import/ImportCliCommand.cs index 606aa2e..28f3be7 100644 --- a/src/PathlingS3Import/ImportCliCommand.cs +++ b/src/PathlingS3Import/ImportCliCommand.cs @@ -93,6 +93,10 @@ public void Run() .AddRetry(retryOptions) // Add retry using the default options .Build(); // Builds the resilience pipeline + log.LogInformation( + "Pathling FHIR base URL set to {PathlingBaseUrl}", + PathlingServerBaseUrl + ); using var fhirClient = new FhirClient( PathlingServerBaseUrl, settings: new FhirClientSettings @@ -104,6 +108,7 @@ public void Run() } ); + log.LogInformation("Minio endpoint set to {S3Endpoint}", S3Endpoint); using var minio = new MinioClient() .WithEndpoint(S3Endpoint) .WithCredentials(S3AccessKey, S3SecretKey) @@ -118,6 +123,12 @@ private async Task DoAsync( ResiliencePipeline retryPipeline ) { + log.LogInformation( + "Checking if bucket {S3BucketName} exists in {S3BaseUrl}.", + S3BucketName, + minio.Config.BaseUrl + ); + var bucketExistsArgs = new BucketExistsArgs().WithBucket(S3BucketName); bool found = await minio.BucketExistsAsync(bucketExistsArgs); if (!found) @@ -139,16 +150,22 @@ ResiliencePipeline retryPipeline .WithPrefix(prefix) .WithRecursive(false); - var observable = minio.ListObjectsAsync(listArgs); + var observable = + minio.ListObjectsAsync(listArgs) + ?? throw new InvalidOperationException("observable for listing buckets is null"); var allObjects = new List(); - using (log.BeginScope("[listing objects]")) + using (log.BeginScope("[Listing objects in {ObjectsPath}]", $"{S3BucketName}/{prefix}")) { using var subscription = observable.Subscribe( item => { - log.LogInformation("Listing object: {ItemKey} ({IsDir})", item.Key, item.IsDir); + log.LogInformation( + "Got object: {ItemKey} (IsDir: {IsDir})", + item.Key, + item.IsDir + ); if (!item.IsDir && item.Key.EndsWith(".ndjson")) { allObjects.Add(item); @@ -161,18 +178,26 @@ ResiliencePipeline retryPipeline observable.Wait(); } + log.LogInformation("Found a total of {ObjectCount} matching objects.", allObjects.Count); + var allObjectsSorted = allObjects.OrderBy(o => o.Key); var currentProgressObjectName = $"{prefix}pathling-s3-importer-last-imported.txt"; + log.LogInformation( + "Name of the current progress tracker object set to {CurrentProgressObjectName}.", + currentProgressObjectName + ); + var stopwatch = new Stopwatch(); var importedCount = 0; - using (log.BeginScope("[importing ndjson]")) + + foreach (var item in allObjectsSorted) { - foreach (var item in allObjectsSorted) - { - var objectUrl = $"s3://{S3BucketName}/{item.Key}"; + var objectUrl = $"s3://{S3BucketName}/{item.Key}"; + using (log.BeginScope("[Importing ndjson file {NdjsonObjectUrl}]", objectUrl)) + { var parameter = new Parameters.ParameterComponent() { Name = "source", @@ -201,54 +226,55 @@ ResiliencePipeline retryPipeline // we might want to add multiple ndjson files at once in batches. importParameters.Parameter.Add(parameter); - using (log.BeginScope($"[importing {objectUrl}]")) - { - log.LogInformation("{ImportParameters}", importParameters.ToJson()); + log.LogInformation("{ImportParameters}", importParameters.ToJson()); - log.LogInformation( - "Starting {PathlingServerBaseUrl}/$import for {ObjectUrl}", - PathlingServerBaseUrl, - objectUrl - ); + log.LogInformation( + "Starting {PathlingServerBaseUrl}/$import for {ObjectUrl}", + PathlingServerBaseUrl, + objectUrl + ); - if (!IsDryRun) - { - stopwatch.Restart(); - var response = await retryPipeline.ExecuteAsync(async token => - { - return await fhirClient.WholeSystemOperationAsync( - "import", - importParameters - ); - }); - stopwatch.Stop(); - - log.LogInformation("{ImportResponse}", response.ToJson()); - log.LogInformation("Import took {ImportDuration}", stopwatch.Elapsed); - - var bytes = Encoding.UTF8.GetBytes(objectUrl); - using var memoryStream = new MemoryStream(bytes); - - // persist progress - var putArgs = new PutObjectArgs() - .WithBucket(S3BucketName) - .WithObject(currentProgressObjectName) - .WithContentType("text/plain") - .WithStreamData(memoryStream) - .WithObjectSize(bytes.LongLength); - - await retryPipeline.ExecuteAsync(async token => - { - await minio.PutObjectAsync(putArgs, token); - }); - } - else + if (!IsDryRun) + { + stopwatch.Restart(); + var response = await retryPipeline.ExecuteAsync(async token => { - log.LogInformation( - "Running import in dry run mode. Waiting a few seconds." + return await fhirClient.WholeSystemOperationAsync( + "import", + importParameters ); - await Task.Delay(TimeSpan.FromSeconds(5)); - } + }); + stopwatch.Stop(); + + log.LogInformation("{ImportResponse}", response.ToJson()); + log.LogInformation("Import took {ImportDuration}", stopwatch.Elapsed); + + var bytes = Encoding.UTF8.GetBytes(objectUrl); + using var memoryStream = new MemoryStream(bytes); + + // persist progress + var putArgs = new PutObjectArgs() + .WithBucket(S3BucketName) + .WithObject(currentProgressObjectName) + .WithContentType("text/plain") + .WithStreamData(memoryStream) + .WithObjectSize(bytes.LongLength); + + stopwatch.Restart(); + await retryPipeline.ExecuteAsync(async token => + { + await minio.PutObjectAsync(putArgs, token); + }); + stopwatch.Stop(); + log.LogInformation( + "Persisting progress took {PutProgressDuration}", + stopwatch.Elapsed + ); + } + else + { + log.LogInformation("Running import in dry run mode. Waiting a few seconds."); + await Task.Delay(TimeSpan.FromSeconds(5)); } importedCount++;