From 2013159f06f902b4e9a2060df411c6b5646d49d4 Mon Sep 17 00:00:00 2001 From: Basit <1305718+mabaasit@users.noreply.github.com> Date: Thu, 15 Aug 2024 22:49:08 +0200 Subject: [PATCH] feat: update libmongocrypt and fix tests (#2076) - Migrate mongosh to new mongodb-client-encryption repository - Update mongodb-client-encryption to 6.1.0 - Enable QE Range as GA and disable the RangePreview version of the algorithm - Use the 8.0 server crypt_shared library everywhere and consequentially break out-of-the-box automatic encryption on platforms with lower glibc requirements Co-authored-by: Neal Beeken Co-authored-by: Anna Henningsen --- .evergreen.yml | 160 +++++---- .evergreen/evergreen.yml.in | 22 +- package-lock.json | 74 +++- package.json | 2 +- .../build/src/compile/signable-compiler.ts | 2 +- .../src/packaging/download-crypt-library.ts | 18 +- packages/cli-repl/src/smoke-tests-fle.ts | 3 +- packages/cli-repl/src/smoke-tests.ts | 7 +- packages/e2e-tests/test/e2e-fle.spec.ts | 324 ++++++++++-------- packages/service-provider-core/package.json | 2 +- packages/service-provider-server/package.json | 2 +- .../src/field-level-encryption.spec.ts | 6 +- scripts/docker/centos7-epel-rpm.Dockerfile | 2 +- scripts/docker/centos7-rpm.Dockerfile | 2 +- scripts/docker/suse12-rpm.Dockerfile | 2 +- scripts/prep-fle-addon.sh | 92 +++-- 16 files changed, 413 insertions(+), 307 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index e74a992f2..e8384f5aa 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -7751,6 +7751,8 @@ functions: params: working_dir: src shell: bash + env: + MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT: ${no_automatic_encryption_support|} script: | set -e set -x @@ -7777,6 +7779,8 @@ functions: params: working_dir: src shell: bash + env: + MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT: ${no_automatic_encryption_support|} script: | set -e set -x @@ -7802,6 +7806,8 @@ functions: params: working_dir: src shell: bash + env: + MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT: ${no_automatic_encryption_support|} script: | set -e set -x @@ -8519,7 +8525,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n20_cli_repl" @@ -8536,7 +8542,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n20_cli_repl" @@ -8740,7 +8746,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n16_cli_repl" @@ -8757,7 +8763,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n16_cli_repl" @@ -8995,7 +9001,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n20_e2e_tests" @@ -9012,7 +9018,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n20_e2e_tests" @@ -9216,7 +9222,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n16_e2e_tests" @@ -9233,7 +9239,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n16_e2e_tests" @@ -9573,7 +9579,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n20_java_shell" @@ -9590,7 +9596,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n20_java_shell" @@ -9794,7 +9800,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n16_java_shell" @@ -9811,7 +9817,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n16_java_shell" @@ -10083,7 +10089,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n20_mongosh" @@ -10100,7 +10106,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n20_mongosh" @@ -10304,7 +10310,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n16_mongosh" @@ -10321,7 +10327,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n16_mongosh" @@ -10525,7 +10531,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n20_node_runtime_worker_thread" @@ -10542,7 +10548,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n20_node_runtime_worker_thread" @@ -10746,7 +10752,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n16_node_runtime_worker_thread" @@ -10763,7 +10769,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n16_node_runtime_worker_thread" @@ -11001,7 +11007,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n20_service_provider_server" @@ -11018,7 +11024,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n20_service_provider_server" @@ -11222,7 +11228,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n16_service_provider_server" @@ -11239,7 +11245,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n16_service_provider_server" @@ -11443,7 +11449,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n20_shell_api" @@ -11460,7 +11466,7 @@ tasks: node_js_version: "20.16.0" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "20.16.0" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n20_shell_api" @@ -11664,7 +11670,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15" + mongosh_server_test_version: "8.0.0-rc17" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xc_n16_shell_api" @@ -11681,7 +11687,7 @@ tasks: node_js_version: "16.20.2" - func: test vars: - mongosh_server_test_version: ">= 8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" node_js_version: "16.20.2" mongosh_skip_node_version_check: "" mongosh_test_id: "m80xe_n16_shell_api" @@ -11924,7 +11930,7 @@ tasks: mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_darwin_x64_800rc15_fips + - name: e2e_tests_darwin_x64_800rc17_fips tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -11940,10 +11946,10 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "1" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_darwin_x64_800rc15 + - name: e2e_tests_darwin_x64_800rc17 tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -11959,7 +11965,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - name: e2e_tests_darwin_x64_60x_fips @@ -12052,7 +12058,7 @@ tasks: mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_darwin_arm64_800rc15_fips + - name: e2e_tests_darwin_arm64_800rc17_fips tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12068,10 +12074,10 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "1" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_darwin_arm64_800rc15 + - name: e2e_tests_darwin_arm64_800rc17 tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12087,7 +12093,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - name: e2e_tests_darwin_arm64_60x_fips @@ -12180,7 +12186,7 @@ tasks: mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_x64_800rc15_fips + - name: e2e_tests_linux_x64_800rc17_fips tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12196,10 +12202,10 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "1" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_x64_800rc15 + - name: e2e_tests_linux_x64_800rc17 tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12215,7 +12221,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - name: e2e_tests_linux_x64_60x_fips @@ -12308,7 +12314,7 @@ tasks: mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_x64_openssl11_800rc15_fips + - name: e2e_tests_linux_x64_openssl11_800rc17_fips tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12324,10 +12330,10 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "1" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_x64_openssl11_800rc15 + - name: e2e_tests_linux_x64_openssl11_800rc17 tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12343,7 +12349,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - name: e2e_tests_linux_x64_openssl11_60x_fips @@ -12436,7 +12442,7 @@ tasks: mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_x64_openssl3_800rc15_fips + - name: e2e_tests_linux_x64_openssl3_800rc17_fips tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12452,10 +12458,10 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "1" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_x64_openssl3_800rc15 + - name: e2e_tests_linux_x64_openssl3_800rc17 tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12471,7 +12477,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - name: e2e_tests_linux_x64_openssl3_60x_fips @@ -12564,7 +12570,7 @@ tasks: mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_arm64_800rc15_fips + - name: e2e_tests_linux_arm64_800rc17_fips tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12580,10 +12586,10 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "1" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_arm64_800rc15 + - name: e2e_tests_linux_arm64_800rc17 tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12599,7 +12605,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - name: e2e_tests_linux_arm64_60x_fips @@ -12692,7 +12698,7 @@ tasks: mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_arm64_openssl11_800rc15_fips + - name: e2e_tests_linux_arm64_openssl11_800rc17_fips tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12708,10 +12714,10 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "1" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_arm64_openssl11_800rc15 + - name: e2e_tests_linux_arm64_openssl11_800rc17 tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12727,7 +12733,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - name: e2e_tests_linux_arm64_openssl11_60x_fips @@ -12820,7 +12826,7 @@ tasks: mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_arm64_openssl3_800rc15_fips + - name: e2e_tests_linux_arm64_openssl3_800rc17_fips tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12836,10 +12842,10 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "1" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_arm64_openssl3_800rc15 + - name: e2e_tests_linux_arm64_openssl3_800rc17 tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12855,7 +12861,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - name: e2e_tests_linux_arm64_openssl3_60x_fips @@ -12948,7 +12954,7 @@ tasks: mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_ppc64le_800rc15_fips + - name: e2e_tests_linux_ppc64le_800rc17_fips tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12964,10 +12970,10 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "1" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_ppc64le_800rc15 + - name: e2e_tests_linux_ppc64le_800rc17 tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -12983,7 +12989,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - name: e2e_tests_linux_ppc64le_60x_fips @@ -13076,7 +13082,7 @@ tasks: mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_s390x_800rc15_fips + - name: e2e_tests_linux_s390x_800rc17_fips tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -13092,10 +13098,10 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "1" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_linux_s390x_800rc15 + - name: e2e_tests_linux_s390x_800rc17 tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -13111,7 +13117,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - name: e2e_tests_linux_s390x_60x_fips @@ -13204,7 +13210,7 @@ tasks: mongosh_server_test_version: "stable-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_win32_800rc15_fips + - name: e2e_tests_win32_800rc17_fips tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -13220,10 +13226,10 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "1" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - - name: e2e_tests_win32_800rc15 + - name: e2e_tests_win32_800rc17 tags: ["e2e-test"] depends_on: - name: compile_artifact @@ -13239,7 +13245,7 @@ tasks: - func: run_e2e_tests vars: node_js_version: "20.16.0" - mongosh_server_test_version: "8.0.0-rc15-enterprise" + mongosh_server_test_version: "8.0.0-rc17-enterprise" mongosh_test_e2e_force_fips: "" disable_openssl_shared_config_for_bundled_openssl: ${disable_openssl_shared_config_for_bundled_openssl|false} - name: e2e_tests_win32_60x_fips @@ -17844,8 +17850,8 @@ buildvariants: run_on: ubuntu2404-small tags: ["nightly-driver"] tasks: - - name: e2e_tests_linux_x64_800rc15 - - name: e2e_tests_linux_x64_openssl3_800rc15 + - name: e2e_tests_linux_x64_800rc17 + - name: e2e_tests_linux_x64_openssl3_800rc17 - name: e2e_debian10_x64 display_name: "Debian 10 x64 (E2E Tests)" run_on: debian10-small @@ -17907,8 +17913,8 @@ buildvariants: display_name: "Ubuntu 24.04 arm64 (E2E Tests)" run_on: ubuntu2404-arm64-small tasks: - - name: e2e_tests_linux_arm64_800rc15 - - name: e2e_tests_linux_arm64_openssl3_800rc15 + - name: e2e_tests_linux_arm64_800rc17 + - name: e2e_tests_linux_arm64_openssl3_800rc17 - name: e2e_amazon2_arm64 display_name: "Amazon Linux 2 arm64 (E2E Tests)" run_on: amazon2-arm64-large @@ -18305,6 +18311,8 @@ buildvariants: - name: pkg_smoke_tests_rhel72_s390x display_name: "package smoke tests (RHEL 7.2 s390x)" run_on: rhel72-zseries-small + expansions: + no_automatic_encryption_support: 1 tasks: - name: pkg_test_rpmextract_rpm_s390x - name: pkg_smoke_tests_rhel83_s390x diff --git a/.evergreen/evergreen.yml.in b/.evergreen/evergreen.yml.in index 1d97f83df..dee4394da 100644 --- a/.evergreen/evergreen.yml.in +++ b/.evergreen/evergreen.yml.in @@ -15,8 +15,8 @@ const MONGODB_VERSIONS = [ { shortName: '60xe', versionSpec: '6.0.x-enterprise' }, { shortName: '70xc', versionSpec: '7.0.x' }, { shortName: '70xe', versionSpec: '7.0.x-enterprise' }, - { shortName: '80xc', versionSpec: '>= 8.0.0-rc15' }, - { shortName: '80xe', versionSpec: '>= 8.0.0-rc15-enterprise' }, + { shortName: '80xc', versionSpec: '8.0.0-rc17' }, + { shortName: '80xe', versionSpec: '8.0.0-rc17-enterprise' }, { shortName: 'latest', versionSpec: 'latest-alpha-enterprise' } ]; const NODE_VERSIONS = [ @@ -743,6 +743,8 @@ functions: params: working_dir: src shell: bash + env: + MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT: ${no_automatic_encryption_support|} script: | set -e set -x @@ -769,6 +771,8 @@ functions: params: working_dir: src shell: bash + env: + MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT: ${no_automatic_encryption_support|} script: | set -e set -x @@ -794,6 +798,8 @@ functions: params: working_dir: src shell: bash + env: + MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT: ${no_automatic_encryption_support|} script: | set -e set -x @@ -1192,7 +1198,7 @@ tasks: # E2E TESTS ### <% for (const { executableOsId, compileBuildVariant } of RELEASE_PACKAGE_MATRIX) { - for (const mVersion of ['stable', '8.0.0-rc15', '6.0.x']) { + for (const mVersion of ['stable', '8.0.0-rc17', '6.0.x']) { for (const fipsVariant of ['fips', 'nofips']) { %> - name: e2e_tests_<% out(executableOsId.replace(/-/g, '_')) %><% out(mVersion === 'stable' ? '' : '_' + mVersion.replace(/[^a-zA-Z0-9]/g, '')) %><% out(fipsVariant === 'fips' ? '_fips' : '') %> @@ -1732,8 +1738,8 @@ buildvariants: run_on: ubuntu2404-small tags: ["nightly-driver"] tasks: - - name: e2e_tests_linux_x64_800rc15 - - name: e2e_tests_linux_x64_openssl3_800rc15 + - name: e2e_tests_linux_x64_800rc17 + - name: e2e_tests_linux_x64_openssl3_800rc17 - name: e2e_debian10_x64 display_name: "Debian 10 x64 (E2E Tests)" run_on: debian10-small @@ -1795,8 +1801,8 @@ buildvariants: display_name: "Ubuntu 24.04 arm64 (E2E Tests)" run_on: ubuntu2404-arm64-small tasks: - - name: e2e_tests_linux_arm64_800rc15 - - name: e2e_tests_linux_arm64_openssl3_800rc15 + - name: e2e_tests_linux_arm64_800rc17 + - name: e2e_tests_linux_arm64_openssl3_800rc17 - name: e2e_amazon2_arm64 display_name: "Amazon Linux 2 arm64 (E2E Tests)" run_on: amazon2-arm64-large @@ -1965,6 +1971,8 @@ buildvariants: - name: pkg_smoke_tests_rhel72_s390x display_name: "package smoke tests (RHEL 7.2 s390x)" run_on: rhel72-zseries-small + expansions: + no_automatic_encryption_support: 1 tasks: - name: pkg_test_rpmextract_rpm_s390x - name: pkg_smoke_tests_rhel83_s390x diff --git a/package-lock.json b/package-lock.json index 705375b7b..ada522257 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23017,20 +23017,48 @@ } }, "node_modules/mongodb-client-encryption": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/mongodb-client-encryption/-/mongodb-client-encryption-6.0.0.tgz", - "integrity": "sha512-GtqkqlSq19acX006/U1odA3l+gwhvABeoTUlvvgtvSs6qcN3qSHPnur3Z5N4oKOv6fZ7EtT8rIsWP2riI0+Eyg==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/mongodb-client-encryption/-/mongodb-client-encryption-6.1.0.tgz", + "integrity": "sha512-Y3Hakre82nXD/pNDUzBjxfgwWSj5E1ar9ZLkqyXDfvirv4huHMbg8Q2qVO/TXlNJuf1B2bzrEDXsTqHKQSQLtw==", "hasInstallScript": true, + "license": "Apache-2.0", "optional": true, "dependencies": { "bindings": "^1.5.0", "node-addon-api": "^4.3.0", - "prebuild-install": "^7.1.1" + "prebuild-install": "^7.1.2" }, "engines": { "node": ">=16.20.1" } }, + "node_modules/mongodb-client-encryption/node_modules/prebuild-install": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz", + "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/mongodb-connection-string-url": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.1.tgz", @@ -31708,7 +31736,7 @@ "node": ">=14.15.1" }, "optionalDependencies": { - "mongodb-client-encryption": "^6.0.0" + "mongodb-client-encryption": "^6.1.0" } }, "packages/service-provider-server": { @@ -31740,7 +31768,7 @@ }, "optionalDependencies": { "kerberos": "^2.1.0", - "mongodb-client-encryption": "^6.0.0" + "mongodb-client-encryption": "^6.1.0" } }, "packages/shell-api": { @@ -38004,7 +38032,7 @@ "eslint": "^7.25.0", "mongodb": "^6.8.0", "mongodb-build-info": "^1.7.2", - "mongodb-client-encryption": "^6.0.0", + "mongodb-client-encryption": "^6.1.0", "mongodb-connection-string-url": "^3.0.1", "prettier": "^2.8.8" } @@ -38026,7 +38054,7 @@ "eslint": "^7.25.0", "kerberos": "^2.1.0", "mongodb": "^6.8.0", - "mongodb-client-encryption": "^6.0.0", + "mongodb-client-encryption": "^6.1.0", "mongodb-connection-string-url": "^3.0.1", "prettier": "^2.8.8", "socks": "^2.8.3" @@ -50492,14 +50520,36 @@ } }, "mongodb-client-encryption": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/mongodb-client-encryption/-/mongodb-client-encryption-6.0.0.tgz", - "integrity": "sha512-GtqkqlSq19acX006/U1odA3l+gwhvABeoTUlvvgtvSs6qcN3qSHPnur3Z5N4oKOv6fZ7EtT8rIsWP2riI0+Eyg==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/mongodb-client-encryption/-/mongodb-client-encryption-6.1.0.tgz", + "integrity": "sha512-Y3Hakre82nXD/pNDUzBjxfgwWSj5E1ar9ZLkqyXDfvirv4huHMbg8Q2qVO/TXlNJuf1B2bzrEDXsTqHKQSQLtw==", "optional": true, "requires": { "bindings": "^1.5.0", "node-addon-api": "^4.3.0", - "prebuild-install": "^7.1.1" + "prebuild-install": "^7.1.2" + }, + "dependencies": { + "prebuild-install": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz", + "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==", + "optional": true, + "requires": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + } + } } }, "mongodb-connection-string-url": { diff --git a/package.json b/package.json index 63c3938a9..f03f29dd5 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "generate-error-overview": "npm run generate-error-overview --workspace @mongosh/errors", "update-authors": "ts-node -P configs/tsconfig-mongosh/tsconfig.common.json scripts/generate-authors.ts", "create-dependency-sbom-lists": "npm run webpack-build -w packages/cli-repl && npm run write-node-js-dep && npm run create-purls-file && npm run create-first-party-dependency-lists", - "create-first-party-dependency-lists": "mongodb-sbom-tools fetch-codeql-results --first-party-deps-list-dest=.sbom/first-party-deps.json --dependencies=.sbom/dependencies.json --exclude-repos=mongodb-js/kerberos", + "create-first-party-dependency-lists": "mongodb-sbom-tools fetch-codeql-results --first-party-deps-list-dest=.sbom/first-party-deps.json --dependencies=.sbom/dependencies.json --exclude-repos=mongodb-js/kerberos,mongodb-client-encryption", "create-purls-file": "node scripts/create-purls.js .sbom/dependencies.json .sbom/node-js-dep.json > .sbom/purls.txt", "preupdate-third-party-notices": "npm run create-dependency-sbom-lists", "update-third-party-notices": "mongodb-sbom-tools generate-3rd-party-notices --product='mongosh' --dependencies=.sbom/dependencies.json > THIRD_PARTY_NOTICES.md", diff --git a/packages/build/src/compile/signable-compiler.ts b/packages/build/src/compile/signable-compiler.ts index 8acac4b74..cf591129a 100644 --- a/packages/build/src/compile/signable-compiler.ts +++ b/packages/build/src/compile/signable-compiler.ts @@ -32,7 +32,7 @@ async function preCompileHook(nodeSourceTree: string) { env: { ...process.env, FLE_NODE_SOURCE_PATH: nodeSourceTree, - LIBMONGOCRYPT_VERSION: `node-v${fleAddonVersion}`, + MONGODB_CLIENT_ENCRYPTION_VERSION: `v${fleAddonVersion}`, }, stdio: 'inherit', } diff --git a/packages/build/src/packaging/download-crypt-library.ts b/packages/build/src/packaging/download-crypt-library.ts index a0b235321..4c582d927 100644 --- a/packages/build/src/packaging/download-crypt-library.ts +++ b/packages/build/src/packaging/download-crypt-library.ts @@ -34,13 +34,12 @@ export async function downloadCryptLibrary( ); // Download mongodb for latest server version, including rapid releases // (for the platforms that they exist for, i.e. for ppc64le/s390x only pick stable releases). - let versionSpec = 'continuous'; - if (/ppc64/.test(opts.arch || process.arch)) { - versionSpec = 'stable'; - } - if (/s390x/.test(opts.arch || process.arch)) { - versionSpec = '6.0.x'; // The 7.x+ server releases don't have RHEL7-compatible crypt_shared libraries - } + // TODO(MONGOSH-1833): The current 'continuous' release is not compatible with 8.x rc server releases. So we are using + // 8.0.0-rc17 (current latest) for now and once 8.0 is released we should switch back to continuous. + const versionSpec = '8.0.0-rc17'; + //if (/ppc64|s390x/.test(opts.arch || process.arch)) { + // versionSpec = 'stable'; + //} const { downloadedBinDir: libdir, version } = await downloadMongoDbWithVersionInfo(cryptTmpTargetDir, versionSpec, opts); const cryptLibrary = path.join( @@ -73,11 +72,10 @@ function lookupReleaseDistro(packageVariant: PackageVariant): { case 'ppc64le': return { platform: 'linux', distro: 'rhel81' }; case 's390x': - return { platform: 'linux', distro: 'rhel72' }; + return { platform: 'linux', distro: 'rhel83' }; case 'arm64': - return { platform: 'linux', distro: 'amazon2' }; case 'x64': - return { platform: 'linux', distro: 'rhel70' }; + return { platform: 'linux', distro: 'rhel8' }; default: break; } diff --git a/packages/cli-repl/src/smoke-tests-fle.ts b/packages/cli-repl/src/smoke-tests-fle.ts index 938bbc0dc..d5d4e2ff8 100644 --- a/packages/cli-repl/src/smoke-tests-fle.ts +++ b/packages/cli-repl/src/smoke-tests-fle.ts @@ -13,7 +13,8 @@ const assert = function(value, message) { } }; if (db.version().startsWith('4.0.') || - !db.runCommand({buildInfo:1}).modules.includes('enterprise')) { + !db.runCommand({buildInfo:1}).modules.includes('enterprise') || + !!process.env.MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT) { // No FLE on mongod < 4.2 or community print('Test skipped') process.exit(0) diff --git a/packages/cli-repl/src/smoke-tests.ts b/packages/cli-repl/src/smoke-tests.ts index 2d2938ad3..22fe82155 100644 --- a/packages/cli-repl/src/smoke-tests.ts +++ b/packages/cli-repl/src/smoke-tests.ts @@ -73,8 +73,13 @@ export async function runSmokeTests({ const expectFipsSupport = !!process.env.MONGOSH_SMOKE_TEST_OS_HAS_FIPS_SUPPORT && (await buildInfo()).sharedOpenssl; + const expectAutomaticEncryptionSupport = + !process.env.MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT; if (!wantPerformanceTesting) { - console.log('FIPS support required to pass?', { expectFipsSupport }); + console.log('FIPS/FLE support required to pass?', { + expectFipsSupport, + expectAutomaticEncryptionSupport, + }); } const perfResults: PerfTestResult[] = []; diff --git a/packages/e2e-tests/test/e2e-fle.spec.ts b/packages/e2e-tests/test/e2e-fle.spec.ts index 4246ca071..49dae636e 100644 --- a/packages/e2e-tests/test/e2e-fle.spec.ts +++ b/packages/e2e-tests/test/e2e-fle.spec.ts @@ -27,6 +27,15 @@ describe('FLE tests', function () { let cryptLibrary: string; before(async function () { + if (process.platform === 'linux') { + const [major, minor] = (process.report as any) + .getReport() + .header.glibcVersionRuntime.split('.'); + expect(major).to.equal('2'); + // All crypt_shared versions that we use require at least glibc 2.28 + if (+minor < 28) return this.skip(); + } + kmsServer = makeFakeHTTPServer(fakeAWSHandlers); kmsServer.listen(0); await once(kmsServer, 'listening'); @@ -630,6 +639,60 @@ describe('FLE tests', function () { expect(collections).to.not.include('collfle2'); }); + it('creates an encrypted collection and generates data encryption keys automatically per encrypted fields', async function () { + const shell = TestShell.start({ args: ['--nodb'] }); + const uri = JSON.stringify(await testServer.connectionString()); + await shell.waitForPrompt(); + await shell.executeLine( + 'local = { key: BinData(0, "kh4Gv2N8qopZQMQYMEtww/AkPsIrXNmEMxTrs3tUoTQZbZu4msdRUaR8U5fXD7A7QXYHcEvuu4WctJLoT+NvvV3eeIg3MD+K8H9SR794m/safgRHdIfy6PD+rFpvmFbY") }' + ); + await shell.executeLine(`keyMongo = Mongo(${uri}, { + keyVaultNamespace: '${dbname}.keyVault', + kmsProviders: { local }, + explicitEncryptionOnly: true + });`); + await shell.executeLine(`secretDB = keyMongo.getDB('${dbname}')`); + await shell.executeLine(`var { collection, encryptedFields } = secretDB.createEncryptedCollection('secretCollection', { + provider: 'local', + createCollectionOptions: { + encryptedFields: { + fields: [{ + keyId: null, + path: 'secretField', + bsonType: 'string' + }] + } + } + });`); + + await shell.executeLine(`plainMongo = Mongo(${uri});`); + const collections = await shell.executeLine( + `plainMongo.getDB('${dbname}').getCollectionNames()` + ); + expect(collections).to.include('enxcol_.secretCollection.esc'); + expect(collections).to.include('enxcol_.secretCollection.ecoc'); + expect(collections).to.include('secretCollection'); + + const dekCount = await shell.executeLine( + `plainMongo.getDB('${dbname}').getCollection('keyVault').countDocuments()` + ); + // Since there is only one field to be encrypted hence there would only be one DEK in our keyvault collection + expect(parseInt(dekCount.trim(), 10)).to.equal(1); + }); + }); + + context('8.0+', function () { + skipIfServerVersion(testServer, '< 8.0'); // Queryable Encryption v2 only available on 7.0+ + + const rangeType = 'range'; + const rangeAlgorithm = 'Range'; + const rangeOptions = `{ + sparsity: Long(1), + trimFactor: 1, + min: new Date('1970'), + max: new Date('2100') + }`; + it('allows compactStructuredEncryptionData command when mongo instance configured with auto encryption', async function () { const shell = TestShell.start({ args: ['--nodb', `--cryptSharedLibPath=${cryptLibrary}`], @@ -716,152 +779,139 @@ describe('FLE tests', function () { expect(parseInt(dekCount.trim(), 10)).to.equal(1); }); - context('using rangePreview algorithm', function () { - // TODO(MONGOSH-1742): Server 8.0 drops "rangePreview" algorithm and adds - // "range". Re-enable these when the change is finalized - skipIfServerVersion(testServer, '>= 8.0.0-alpha'); - - it('allows explicit range encryption with bypassQueryAnalysis', async function () { - // No --cryptSharedLibPath since bypassQueryAnalysis is also a community edition feature - const shell = TestShell.start({ args: ['--nodb'] }); - const uri = JSON.stringify(await testServer.connectionString()); - - await shell.waitForPrompt(); - - await shell.executeLine(`{ - client = Mongo(${uri}, { - keyVaultNamespace: '${dbname}.keyVault', - kmsProviders: { local: { key: 'A'.repeat(128) } }, - bypassQueryAnalysis: true - }); - - keyVault = client.getKeyVault(); - clientEncryption = client.getClientEncryption(); - - // Create necessary data key - dataKey = keyVault.createKey('local'); - - rangeOptions = { - sparsity: Long(1), - min: new Date('1970'), - max: new Date('2100') - }; - coll = client.getDB('${dbname}').encryptiontest; - client.getDB('${dbname}').createCollection('encryptiontest', { - encryptedFields: { - fields: [{ - keyId: dataKey, - path: 'v', - bsonType: 'date', - queries: [{ - queryType: 'rangePreview', - contention: 4, - ...rangeOptions - }] + it('allows explicit range encryption with bypassQueryAnalysis', async function () { + // No --cryptSharedLibPath since bypassQueryAnalysis is also a community edition feature + const shell = TestShell.start({ args: ['--nodb'] }); + const uri = JSON.stringify(await testServer.connectionString()); + + await shell.waitForPrompt(); + + await shell.executeLine(`{ + client = Mongo(${uri}, { + keyVaultNamespace: '${dbname}.keyVault', + kmsProviders: { local: { key: 'A'.repeat(128) } }, + bypassQueryAnalysis: true + }); + + keyVault = client.getKeyVault(); + clientEncryption = client.getClientEncryption(); + + // Create necessary data key + dataKey = keyVault.createKey('local'); + + coll = client.getDB('${dbname}').encryptiontest; + client.getDB('${dbname}').createCollection('encryptiontest', { + encryptedFields: { + fields: [{ + keyId: dataKey, + path: 'v', + bsonType: 'date', + queries: [{ + queryType: '${rangeType}', + contention: 4, + ...${rangeOptions} }] - } - }); - - // Encrypt and insert data encrypted with specified data key - for (let year = 1990; year < 2010; year++) { - const insertPayload = clientEncryption.encrypt( - dataKey, - new Date(year + '-02-02T12:45:16.277Z'), - { - algorithm: 'RangePreview', - contentionFactor: 4, - rangeOptions - }); - coll.insertOne({ v: insertPayload, year }); + }] } - }`); - expect( - await shell.executeLine('({ count: coll.countDocuments() })') - ).to.include('{ count: 20 }'); - - await shell.executeLine(`{ - findPayload = clientEncryption.encryptExpression(dataKey, { - $and: [ { v: {$gt: new Date('1992')} }, { v: {$lt: new Date('1999')} } ] - }, { - algorithm: 'RangePreview', - queryType: 'rangePreview', - contentionFactor: 4, - rangeOptions }); - }`); - - // Make sure the find payload allows searching for the encrypted values - const out = await shell.executeLine( - "\ - coll.find(findPayload) \ - .toArray() \ - .map(d => d.year) \ - .sort() \ - .join(',')" - ); - expect(out).to.include('1992,1993,1994,1995,1996,1997,1998'); - }); - it('allows automatic range encryption', async function () { - // TODO(MONGOSH-1550): On s390x, we are using the 6.0.x RHEL7 shared library, - // which does not support QE rangePreview. That's just fine for preview, but - // we should switch to the 7.0.x RHEL8 shared library for Range GA. - if (process.arch === 's390x') { - return this.skip(); + // Encrypt and insert data encrypted with specified data key + for (let year = 1990; year < 2010; year++) { + const insertPayload = clientEncryption.encrypt( + dataKey, + new Date(year + '-02-02T12:45:16.277Z'), + { + algorithm: '${rangeAlgorithm}', + contentionFactor: 4, + rangeOptions: ${rangeOptions} + }); + coll.insertOne({ v: insertPayload, year }); } + }`); + expect( + await shell.executeLine('({ count: coll.countDocuments() })') + ).to.include('{ count: 20 }'); - const shell = TestShell.start({ - args: ['--nodb', `--cryptSharedLibPath=${cryptLibrary}`], + await shell.executeLine(`{ + findPayload = clientEncryption.encryptExpression(dataKey, { + $and: [ { v: {$gt: new Date('1992')} }, { v: {$lt: new Date('1999')} } ] + }, { + algorithm: '${rangeAlgorithm}', + queryType: '${rangeType}', + contentionFactor: 4, + rangeOptions: ${rangeOptions} + }); + }`); + + // Make sure the find payload allows searching for the encrypted values + const out = await shell.executeLine( + "\ + coll.find(findPayload) \ + .toArray() \ + .map(d => d.year) \ + .sort() \ + .join(',')" + ); + expect(out).to.include('1992,1993,1994,1995,1996,1997,1998'); + }); + + it('allows automatic range encryption', async function () { + // TODO(MONGOSH-1550): On s390x, we are using the 6.0.x RHEL7 shared library, + // which does not support QE rangePreview/range. That's just fine for preview, but + // we should switch to the 7.0.x RHEL8 shared library for Range GA. + if (process.arch === 's390x') { + return this.skip(); + } + + const shell = TestShell.start({ + args: ['--nodb', `--cryptSharedLibPath=${cryptLibrary}`], + }); + const uri = JSON.stringify(await testServer.connectionString()); + + await shell.waitForPrompt(); + + await shell.executeLine(`{ + client = Mongo(${uri}, { + keyVaultNamespace: '${dbname}.keyVault', + kmsProviders: { local: { key: 'A'.repeat(128) } } }); - const uri = JSON.stringify(await testServer.connectionString()); - - await shell.waitForPrompt(); - - await shell.executeLine(`{ - client = Mongo(${uri}, { - keyVaultNamespace: '${dbname}.keyVault', - kmsProviders: { local: { key: 'A'.repeat(128) } } - }); - - dataKey = client.getKeyVault().createKey('local'); - - coll = client.getDB('${dbname}').encryptiontest; - client.getDB('${dbname}').createCollection('encryptiontest', { - encryptedFields: { - fields: [{ - keyId: dataKey, - path: 'v', - bsonType: 'date', - queries: [{ - queryType: 'rangePreview', - contention: 4, - sparsity: 1, - min: new Date('1970'), - max: new Date('2100') - }] + + dataKey = client.getKeyVault().createKey('local'); + + coll = client.getDB('${dbname}').encryptiontest; + client.getDB('${dbname}').createCollection('encryptiontest', { + encryptedFields: { + fields: [{ + keyId: dataKey, + path: 'v', + bsonType: 'date', + queries: [{ + queryType: '${rangeType}', + contention: 4, + ...${rangeOptions} }] - } - }); - - for (let year = 1990; year < 2010; year++) { - coll.insertOne({ v: new Date(year + '-02-02T12:45:16.277Z'), year }) + }] } - }`); - expect( - await shell.executeLine('({ count: coll.countDocuments() })') - ).to.include('{ count: 20 }'); - - // Make sure the find payload allows searching for the encrypted values - const out = await shell.executeLine( - "\ - coll.find({ v: {$gt: new Date('1992'), $lt: new Date('1999') } }) \ - .toArray() \ - .map(d => d.year) \ - .sort() \ - .join(',')" - ); - expect(out).to.include('1992,1993,1994,1995,1996,1997,1998'); - }); + }); + + for (let year = 1990; year < 2010; year++) { + coll.insertOne({ v: new Date(year + '-02-02T12:45:16.277Z'), year }) + } + }`); + expect( + await shell.executeLine('({ count: coll.countDocuments() })') + ).to.include('{ count: 20 }'); + + // Make sure the find payload allows searching for the encrypted values + const out = await shell.executeLine( + "\ + coll.find({ v: {$gt: new Date('1992'), $lt: new Date('1999') } }) \ + .toArray() \ + .map(d => d.year) \ + .sort() \ + .join(',')" + ); + expect(out).to.include('1992,1993,1994,1995,1996,1997,1998'); }); }); diff --git a/packages/service-provider-core/package.json b/packages/service-provider-core/package.json index aa670c9cd..fdd3b51a3 100644 --- a/packages/service-provider-core/package.json +++ b/packages/service-provider-core/package.json @@ -51,7 +51,7 @@ "mongodb-connection-string-url": "^3.0.1" }, "optionalDependencies": { - "mongodb-client-encryption": "^6.0.0" + "mongodb-client-encryption": "^6.1.0" }, "devDependencies": { "@mongodb-js/eslint-config-mongosh": "^1.0.0", diff --git a/packages/service-provider-server/package.json b/packages/service-provider-server/package.json index 5334877a9..4ffaeb1ab 100644 --- a/packages/service-provider-server/package.json +++ b/packages/service-provider-server/package.json @@ -59,7 +59,7 @@ }, "optionalDependencies": { "kerberos": "^2.1.0", - "mongodb-client-encryption": "^6.0.0" + "mongodb-client-encryption": "^6.1.0" }, "devDependencies": { "@mongodb-js/eslint-config-mongosh": "^1.0.0", diff --git a/packages/shell-api/src/field-level-encryption.spec.ts b/packages/shell-api/src/field-level-encryption.spec.ts index 53cf176ab..deb925e13 100644 --- a/packages/shell-api/src/field-level-encryption.spec.ts +++ b/packages/shell-api/src/field-level-encryption.spec.ts @@ -268,13 +268,13 @@ describe('Field Level Encryption', function () { }; const options = { - algorithm: 'RangePreview', - queryType: 'rangePreview', + algorithm: 'Range', + queryType: 'range', contentionFactor: 0, rangeOptions: { sparsity: new bson.Long(1), }, - } as const; + } as any; // TODO Needs a driver update to get correct types. it('calls encryptExpression with algorithm on libmongoc', async function () { libmongoc.encryptExpression.resolves(); diff --git a/scripts/docker/centos7-epel-rpm.Dockerfile b/scripts/docker/centos7-epel-rpm.Dockerfile index 94c258d75..ea8f7544c 100644 --- a/scripts/docker/centos7-epel-rpm.Dockerfile +++ b/scripts/docker/centos7-epel-rpm.Dockerfile @@ -12,5 +12,5 @@ RUN yum install -y epel-release RUN yum repolist RUN yum install -y /tmp/*mongosh*.rpm RUN /usr/bin/mongosh --build-info -RUN env MONGOSH_RUN_NODE_SCRIPT=1 mongosh /usr/share/mongodb-crypt-library-version/node_modules/.bin/mongodb-crypt-library-version /usr/lib64/mongosh_crypt_v1.so | grep -Eq '^mongo_(crypt|csfle)_v1-' +ENV MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT=1 ENTRYPOINT [ "mongosh" ] diff --git a/scripts/docker/centos7-rpm.Dockerfile b/scripts/docker/centos7-rpm.Dockerfile index f08ae9ece..e6e4aada2 100644 --- a/scripts/docker/centos7-rpm.Dockerfile +++ b/scripts/docker/centos7-rpm.Dockerfile @@ -9,5 +9,5 @@ RUN yum-config-manager --enable C7.8.2003-base C7.8.2003-extras C7.8.2003-update RUN yum repolist RUN yum install -y /tmp/*mongosh*.rpm RUN /usr/bin/mongosh --build-info -RUN env MONGOSH_RUN_NODE_SCRIPT=1 mongosh /usr/share/mongodb-crypt-library-version/node_modules/.bin/mongodb-crypt-library-version /usr/lib64/mongosh_crypt_v1.so | grep -Eq '^mongo_(crypt|csfle)_v1-' +ENV MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT=1 ENTRYPOINT [ "mongosh" ] diff --git a/scripts/docker/suse12-rpm.Dockerfile b/scripts/docker/suse12-rpm.Dockerfile index 8b6c1f292..23b66957a 100644 --- a/scripts/docker/suse12-rpm.Dockerfile +++ b/scripts/docker/suse12-rpm.Dockerfile @@ -9,5 +9,5 @@ RUN zypper --no-gpg-checks --non-interactive refresh # using rpm directly seems to fix the issue RUN rpm -i /tmp/*mongosh*.rpm RUN /usr/bin/mongosh --build-info -RUN env MONGOSH_RUN_NODE_SCRIPT=1 mongosh /usr/share/mongodb-crypt-library-version/node_modules/.bin/mongodb-crypt-library-version /usr/lib64/mongosh_crypt_v1.so | grep -Eq '^mongo_(crypt|csfle)_v1-' +ENV MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT=1 ENTRYPOINT [ "mongosh" ] diff --git a/scripts/prep-fle-addon.sh b/scripts/prep-fle-addon.sh index d167dd48a..b072b27b6 100755 --- a/scripts/prep-fle-addon.sh +++ b/scripts/prep-fle-addon.sh @@ -26,69 +26,55 @@ BUILDROOT="$MONGOSH_ROOT_DIR"/tmp/fle-buildroot rm -rf "$BUILDROOT" mkdir -p "$BUILDROOT" cd "$BUILDROOT" -PREBUILT_OSNAME='' -[ -z "$LIBMONGOCRYPT_VERSION" ] && LIBMONGOCRYPT_VERSION=latest +[ -z "$MONGODB_CLIENT_ENCRYPTION_VERSION" ] && MONGODB_CLIENT_ENCRYPTION_VERSION=main -echo Using libmongocrypt at git tag "$LIBMONGOCRYPT_VERSION" +echo Using mongodb-client-encryption at git tag "$MONGODB_CLIENT_ENCRYPTION_VERSION" -if [ x"$FLE_NODE_SOURCE_PATH" != x"" -a -z "$BUILD_FLE_FROM_SOURCE" ]; then - # Use prebuilt binaries where available. - case `uname -a` in - Darwin*x86_64*) PREBUILT_OSNAME=macos;; - Linux*x86_64*) PREBUILT_OSNAME=rhel-70-64-bit;; - Linux*s390x*) PREBUILT_OSNAME=rhel72-zseries-test;; - Linux*aarch64*) PREBUILT_OSNAME=ubuntu1804-arm64;; - Linux*ppc64le*) PREBUILT_OSNAME=rhel-71-ppc64el;; - CYGWIN*|MINGW32*|MSYS*|MINGW*) PREBUILT_OSNAME=windows-test;; - esac -fi - -if [ x"$PREBUILT_OSNAME" != x"" ]; then - if [ $LIBMONGOCRYPT_VERSION != latest ]; then - # Replace LIBMONGOCRYPT_VERSION through its git SHA - git clone https://github.com/mongodb/libmongocrypt --branch $LIBMONGOCRYPT_VERSION --depth 2 - LIBMONGOCRYPT_VERSION=$(cd libmongocrypt && git rev-parse HEAD) - rm -rf libmongocrypt - fi +git clone https://github.com/mongodb-js/mongodb-client-encryption --branch "$MONGODB_CLIENT_ENCRYPTION_VERSION" --depth 2 - # Download and extract prebuilt binaries. - curl -sSfLO https://s3.amazonaws.com/mciuploads/libmongocrypt/$PREBUILT_OSNAME/master/$LIBMONGOCRYPT_VERSION/libmongocrypt.tar.gz - if tar -tzf libmongocrypt.tar.gz lib64; then LIB=lib64; else LIB=lib; fi - mkdir -p prebuilts - tar -xzvf libmongocrypt.tar.gz -C prebuilts nocrypto/ $LIB/ - mkdir -p lib - mv -v prebuilts/nocrypto/$LIB/* lib - mv -v prebuilts/nocrypto/include include - mv -v prebuilts/$LIB/*bson* lib - rm -rf prebuilts -else - if [ `uname` = Darwin ]; then - export CFLAGS="-mmacosx-version-min=10.15"; - fi +cd mongodb-client-encryption - if [ -z "$CMAKE" ]; then CMAKE=cmake; fi +unset IS_WINDOWS +case $(uname -a) in + CYGWIN*|MINGW32*|MSYS*|MINGW*) IS_WINDOWS="true";; +esac - # libmongocrypt currently determines its own version at build time by using - # `git describe`, so there's no way to do anything but a full checkout of the - # repository at this point. - git clone https://github.com/mongodb/libmongocrypt - if [ $LIBMONGOCRYPT_VERSION != "latest" ]; then - (cd libmongocrypt && git checkout $LIBMONGOCRYPT_VERSION) - fi +if [[ $IS_WINDOWS == "true" ]]; then + CMAKE_VERSION="3.25.1" + archive="cmake-$CMAKE_VERSION-windows-x86_64.zip" + url="https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-windows-x86_64.zip" + extract_dir="cmake_$CMAKE_VERSION" + curl --retry 5 -LsS --max-time 120 --fail --output "$archive" "$url" + unzip -o -qq "$archive" -d "cmake_$CMAKE_VERSION" + mv -- $extract_dir/cmake-$CMAKE_VERSION-*/* "$extract_dir" + chmod +x $extract_dir/bin/* - # build libmongocrypt - cd libmongocrypt - mkdir -p cmake-build - cd cmake-build - "$CMAKE" -DCMAKE_INSTALL_PREFIX="$BUILDROOT" -DCMAKE_PREFIX_PATH="$BUILDROOT" -DDISABLE_NATIVE_CRYPTO=1 .. - make -j8 install - cd ../../ + PATH=$PWD/$extract_dir/bin:$PATH + export PATH + hash -r + which cmake + cmake --version fi +# The script in `mongodb-js/mongodb-client-encryption` will download or build the libmongocrypt version specified in +# mongodb-client-encryption's package.json at "mongodb:libmongocrypt" +npm run install:libmongocrypt -- --skip-bindings ${IS_WINDOWS:+--build} + +# The "deps" directory will be populated +# Structure: +# deps/ +# include/kms_message +# include/mongocrypt +# lib/ +# libbson-static-for-libmongocrypt.a +# libkms_message-static.a +# libmongocrypt-static.a + if [ x"$FLE_NODE_SOURCE_PATH" != x"" ]; then mkdir -p "$FLE_NODE_SOURCE_PATH"/deps/lib mkdir -p "$FLE_NODE_SOURCE_PATH"/deps/include - cp -rv "$BUILDROOT"/lib*/*-static* "$FLE_NODE_SOURCE_PATH"/deps/lib - cp -rv "$BUILDROOT"/include/*{kms,mongocrypt}* "$FLE_NODE_SOURCE_PATH"/deps/include + cp -rv ./deps/lib*/*-static* "$FLE_NODE_SOURCE_PATH"/deps/lib + cp -rv ./deps/include/*kms* "$FLE_NODE_SOURCE_PATH"/deps/include + cp -rv ./deps/include/*mongocrypt* "$FLE_NODE_SOURCE_PATH"/deps/include fi