merge main

.evergreen/ocsp-requirements.txt

@@ -1,3 +1,3 @@
 asn1crypto==1.3.0
 bottle==0.12.20
-oscrypto==1.2.0
+oscrypto==1.2.0

.evergreen/run-deployed-lambda-aws-tests.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+set -o errexit  # Exit the script with error if any of the commands fail.
+
+source ${DRIVERS_TOOLS}/.evergreen/atlas/secrets-export.sh
+
+VARLIST=(
+	AWS_REGION
+	DRIVERS_TOOLS
+	DRIVERS_ATLAS_PUBLIC_API_KEY
+	DRIVERS_ATLAS_PRIVATE_API_KEY
+	DRIVERS_ATLAS_LAMBDA_USER
+	DRIVERS_ATLAS_LAMBDA_PASSWORD
+	DRIVERS_ATLAS_GROUP_ID
+	LAMBDA_STACK_NAME
+	PROJECT_DIRECTORY
+	TEST_LAMBDA_DIRECTORY
+)
+
+# Ensure that all variables required to run the test are set, otherwise throw
+# an error.
+for VARNAME in ${VARLIST[*]}; do
+	[[ -z "${!VARNAME}" ]] && echo "ERROR: $VARNAME not set" && exit 1;
+done
+
+echo "Starting deployment"
+. ${DRIVERS_TOOLS}/.evergreen/aws_lambda/run-deployed-lambda-aws-tests.sh

.evergreen/run-fuzz.sh

@@ -44,7 +44,7 @@ do
 			for CORPUS_FILE in $PARENTDIR/testdata/fuzz/$FUNC/*
 			do
 				# Check to see if the value for CORPUS_FILE is in cset.
-				if [[ ! " ${cset[@]} " =~ " ${CORPUS_FILE} " ]]; then
+				if [[ ! " ${cset[*]} " =~ " ${CORPUS_FILE} " ]]; then
 					# Create the directory if it doesn't exist.
 					if [ ! -d $PROJECT_DIRECTORY/fuzz/$FUNC ]; then
 						mkdir -p $PROJECT_DIRECTORY/fuzz/$FUNC
@@ -64,5 +64,8 @@ done
 if [ -d $PROJECT_DIRECTORY/fuzz ]; then
 	echo "Tarring up fuzz directory"
 	tar -czf $PROJECT_DIRECTORY/fuzz.tgz $PROJECT_DIRECTORY/fuzz
-fi
 
+	# Exit with code 1 to indicate that errors occurred in fuzz tests, resulting in corpus files being generated.
+	# This will trigger a notification to be sent to the Go Driver team.
+	exit 1
+fi

.evergreen/run-mongodb-aws-test.sh

@@ -12,24 +12,13 @@ set -o errexit  # Exit the script with error if any of the commands fail
 #                 mechanism.
 
 echo "Running MONGODB-AWS authentication tests"
-# ensure no secrets are printed in log files
-set +x
 
-# load the script
-shopt -s expand_aliases # needed for `urlencode` alias
-[ -s "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" ] && source "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-
-MONGODB_URI=${MONGODB_URI:-"mongodb://localhost"}
-MONGODB_URI="${MONGODB_URI}/aws?authMechanism=MONGODB-AWS"
-if [[ -n ${SESSION_TOKEN} ]]; then
-    MONGODB_URI="${MONGODB_URI}&authMechanismProperties=AWS_SESSION_TOKEN:${SESSION_TOKEN}"
-fi
-
-export MONGODB_URI="$MONGODB_URI"
+# Handle credentials and environment setup.
+. $DRIVERS_TOOLS/.evergreen/auth_aws/aws_setup.sh $1
 
 # show test output
 set -x
 
 # For Go 1.16+, Go builds requires a go.mod file in the current working directory or a parent
 # directory. Spawn a new subshell, "cd" to the project directory, then run "go run".
-(cd ${PROJECT_DIRECTORY} && go run "./cmd/testaws/main.go")
+(cd ${PROJECT_DIRECTORY} && go run "./internal/cmd/testaws/main.go" | tee test.suite)

.evergreen/run-tests.sh

@@ -4,9 +4,13 @@ set -o errexit
 
 export GOPATH=$(dirname $(dirname $(dirname `pwd`)))
 export GOCACHE="$(pwd)/.cache"
-export DRIVERS_TOOLS="$(pwd)/../drivers-tools"
+export DRIVERS_TOOLS=${DRIVERS_TOOLS:-""}
 
-if [ "Windows_NT" = "$OS" ]; then
+if [ -z $DRIVERS_TOOLS ]; then
+  export DRIVERS_TOOLS="$(dirname $(dirname $(dirname `pwd`)))/drivers-tools"
+fi
+
+if [ "Windows_NT" = "${OS:-}" ]; then
     export GOPATH=$(cygpath -m $GOPATH)
     export GOCACHE=$(cygpath -m $GOCACHE)
     export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
@@ -15,9 +19,16 @@ fi
 export GOROOT="${GOROOT}"
 export PATH="${GOROOT}/bin:${GCC_PATH}:$GOPATH/bin:$PATH"
 export PROJECT="${project}"
-export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig
-export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib
-export GOFLAGS=-mod=vendor
+export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig
+export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib64
+
+if [ "$(uname -s)" = "Darwin" ]; then
+  export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib/pkgconfig
+  export DYLD_FALLBACK_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib
+else
+  export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig
+  export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib64
+fi
 
 SSL=${SSL:-nossl}
 if [ "$SSL" != "nossl" -a -z "${SERVERLESS+x}" ]; then
@@ -34,33 +45,8 @@ if [ "$SSL" != "nossl" -a -z "${SERVERLESS+x}" ]; then
     fi
 fi
 
-if [ -z ${AWS_ACCESS_KEY_ID+x} ]; then
-  export AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}"
-  export AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}"
-fi
-
-# Set temp credentials for AWS if python3 is available.
-#
-# Using python3-venv in Ubuntu 14.04 (an OS required for legacy server version
-# tasks) requires the use of apt-get, which we wish to avoid. So, we do not set
-# a python3 binary on Ubuntu 14.04. Setting AWS temp credentials for legacy
-# server version tasks is unneccesary, as temp credentials are only needed on 4.2+.
-if [ ! -z ${PYTHON3_BINARY} ]; then
-  export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}"
-  export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}"
-  export AWS_DEFAULT_REGION="us-east-1"
-  ${PYTHON3_BINARY} -m venv ./venv
-
-  # Set the PYTHON environment variable to point to the active python3 binary. This is used by the
-  # set-temp-creds.sh script.
-  if [ "Windows_NT" = "$OS" ]; then
-    export PYTHON="$(pwd)/venv/Scripts/python"
-  else
-    export PYTHON="$(pwd)/venv/bin/python"
-  fi
-
-  ./venv/${VENV_BIN_DIR:-bin}/pip3 install boto3
-  . ${DRIVERS_TOOLS}/.evergreen/csfle/set-temp-creds.sh
+if [ -f "secrets-export.sh" ]; then
+    source $(pwd)/secrets-export.sh
 fi
 
 # If GO_BUILD_TAGS is not set, set the default Go build tags to "cse" to enable
@@ -69,22 +55,16 @@ if [ -z ${GO_BUILD_TAGS+x} ]; then
   GO_BUILD_TAGS="cse"
 fi
 
-# Ensure mock KMS servers are running before starting tests.
-await_server() {
-  for i in $(seq 300); do
-      # Exit code 7: "Failed to connect to host".
-      if curl -s "localhost:$2"; test $? -ne 7; then
-        return 0
-      else
-        sleep 1
-      fi
-  done
-  echo "could not detect '$1' server on port $2"
-}
-# * List servers to await here ...
-await_server "KMS", 5698
-
-echo "finished awaiting servers"
+if [[ $GO_BUILD_TAGS == *"cse"* ]]; then
+  if [ "Windows_NT" = "$OS" ]; then
+    if [ ! -d /cygdrive/c/libmongocrypt/bin ]; then
+      bash $(pwd)/etc/install-libmongocrypt.sh
+    fi
+    export PATH=$PATH:/cygdrive/c/libmongocrypt/bin
+  elif [ ! -d "$PKG_CONFIG_PATH" ]; then
+    bash $(pwd)/etc/install-libmongocrypt.sh
+  fi
+fi
 
 if [ "${SKIP_CRYPT_SHARED_LIB}" = "true" ]; then
   CRYPT_SHARED_LIB_PATH=""
@@ -96,16 +76,13 @@ else
   echo "crypt_shared library will be loaded from path: $CRYPT_SHARED_LIB_PATH"
 fi
 
-CSFLE_TLS_CA_FILE="$(pwd)/testdata/kmip-certs/ca-ec.pem"
-CSFLE_TLS_CERTIFICATE_KEY_FILE="$(pwd)/testdata/kmip-certs/client-ec.pem"
-
-if [ "Windows_NT" = "$OS" ]; then
-  CSFLE_TLS_CA_FILE=$(cygpath -m $CSFLE_TLS_CA_FILE)
-  CSFLE_TLS_CERTIFICATE_KEY_FILE=$(cygpath -m $CSFLE_TLS_CERTIFICATE_KEY_FILE)
-fi
-
 if [ -z ${MAKEFILE_TARGET+x} ]; then
-  MAKEFILE_TARGET="evg-test"
+  if [ "$(uname -s)" = "Darwin" ]; then
+      # Run a subset of the tests on Darwin
+      MAKEFILE_TARGET="evg-test-load-balancers"
+  else
+    MAKEFILE_TARGET="evg-test"
+  fi
 fi
 
 AUTH=${AUTH} \
@@ -117,21 +94,9 @@ MONGO_GO_DRIVER_PKCS8_UNENCRYPTED_KEY_FILE=${MONGO_GO_DRIVER_PKCS8_UNENCRYPTED_K
 MONGODB_URI="${MONGODB_URI}" \
 TOPOLOGY=${TOPOLOGY} \
 MONGO_GO_DRIVER_COMPRESSOR=${MONGO_GO_DRIVER_COMPRESSOR} \
-BUILD_TAGS="-tags ${GO_BUILD_TAGS}" \
-AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \
-AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \
-AWS_DEFAULT_REGION="us-east-1" \
-CSFLE_AWS_TEMP_ACCESS_KEY_ID="$CSFLE_AWS_TEMP_ACCESS_KEY_ID" \
-CSFLE_AWS_TEMP_SECRET_ACCESS_KEY="$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY" \
-CSFLE_AWS_TEMP_SESSION_TOKEN="$CSFLE_AWS_TEMP_SESSION_TOKEN" \
-AZURE_TENANT_ID="${cse_azure_tenant_id}" \
-AZURE_CLIENT_ID="${cse_azure_client_id}" \
-AZURE_CLIENT_SECRET="${cse_azure_client_secret}" \
-GCP_EMAIL="${cse_gcp_email}" \
-GCP_PRIVATE_KEY="${cse_gcp_private_key}" \
-CSFLE_TLS_CA_FILE="$CSFLE_TLS_CA_FILE" \
-CSFLE_TLS_CERTIFICATE_KEY_FILE="$CSFLE_TLS_CERTIFICATE_KEY_FILE" \
+BUILD_TAGS="${RACE} -tags=${GO_BUILD_TAGS}" \
 CRYPT_SHARED_LIB_PATH=$CRYPT_SHARED_LIB_PATH \
-make $MAKEFILE_TARGET \
 PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
-LD_LIBRARY_PATH=$LD_LIBRARY_PATH
+LD_LIBRARY_PATH=$LD_LIBRARY_PATH \
+MACOS_LIBRARY_PATH=$DYLD_FALLBACK_LIBRARY_PATH \
+make $MAKEFILE_TARGET

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      actions:
+        patterns:
+          - "*"
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: "weekly"

.github/labeler.yml

@@ -0,0 +1,15 @@
+
+priority-3-low:
+- changed-files:
+  - any-glob-to-any-file: '*'
+
+documentation:
+- changed-files:
+  - any-glob-to-any-file: 
+    - docs/**
+    - examples/**
+
+dependencies:
+- changed-files:
+  - any-glob-to-any-file: 
+    - go.mod

.github/reviewers.txt

@@ -0,0 +1,4 @@
+qingyang-hu
+matthewdale
+prestonvasquez
+blink1073

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+name: GoDriver Tests
+
+on:
+  push:
+  pull_request:
+
+concurrency:
+  group: test-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash -eux {0}
+
+jobs:
+  pre_commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+      - uses: pre-commit/[email protected]

.gitignore

@@ -11,3 +11,28 @@ driver-test-data.tar.gz
 perf
 **mongocryptd.pid
 *.test
+.DS_Store
+install
+main.so
+.cache
+install
+libmongocrypt
+venv
+test.suite
+go.work.sum
+
+# AWS SAM-generated files
+internal/cmd/faas/awslambda/.aws-sam
+internal/cmd/faas/awslambda/events/event.json
+
+# Ignore compiled binaries from the compilecheck
+internal/cmd/compilecheck/compilecheck
+internal/cmd/compilecheck/compilecheck.so
+
+# Ignore api report files
+api-report.md
+api-report.txt
+
+# Ignore secrets files
+secrets-expansion.yml
+secrets-export.sh

.golangci.yml

@@ -1,12 +1,20 @@
 run:
   timeout: 5m
+  skip-dirs-use-default: false
+  skip-dirs:
+    - (^|/)vendor($|/)
+    - (^|/)testdata($|/)
+    - (^|/)etc($|/)
+    # Disable all linters for "golang.org/x/exp/rand" package in internal/rand.
+    - internal/rand
 
 linters:
   disable-all: true
   # TODO(GODRIVER-2156): Enable all commented-out linters.
   enable:
     - errcheck
     # - errorlint
+    - exportloopref
     - gocritic
     - goimports
     - gosimple
@@ -82,21 +90,12 @@ issues:
     - "ineffectual assignment to wm"
     - "ineffectual assignment to rem"
 
-  skip-dirs-use-default: false
-  skip-dirs:
-    - (^|/)vendor($|/)
-    - (^|/)testdata($|/)
-    - (^|/)etc($|/)
   exclude-rules:
     # Ignore some linters for example code that is intentionally simplified.
     - path: examples/
       linters:
         - revive
         - errcheck
-    # Disable unused code linters for the copy/pasted "awsv4" package.
-    - path: x/mongo/driver/auth/internal/awsv4
-      linters:
-        - unused
     # Disable "unused" linter for code files that depend on the "mongocrypt.MongoCrypt" type because
     # the linter build doesn't work correctly with CGO enabled. As a result, all calls to a
     # "mongocrypt.MongoCrypt" API appear to always panic (see mongocrypt_not_enabled.go), leading
@@ -117,7 +116,3 @@ issues:
     # Ignore missing package comments for directories that aren't frequently used by external users.
     - path: (internal\/|benchmark\/|x\/|cmd\/|mongo\/integration\/)
       text: should have a package comment
-    # Disable unused linter for "golang.org/x/exp/rand" package in internal/randutil/rand.
-    - path: internal/randutil/rand
-      linters:
-        - unused