From ad0460bff35a898098575e1323460857d517a266 Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Mon, 18 Nov 2024 18:19:54 +0000 Subject: [PATCH] Add PROPFIND/list test for public & authed path-style buckets --- CMakeLists.txt | 2 +- test/CMakeLists.txt | 10 ++++++ test/s3-list-test.sh | 74 ++++++++++++++++++++++++++++++++++++++++++ test/s3-setup.sh | 23 ++++++++++--- test/s3-stress-test.sh | 2 +- test/s3-test.sh | 27 ++++++++++++--- 6 files changed, 127 insertions(+), 11 deletions(-) create mode 100755 test/s3-list-test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index c6a709e..a271f6d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required( VERSION 3.13 ) project( xrootd-http/s3 ) -option( XROOTD_PLUGINS_BUILD_UNITTESTS "Build the scitokens-cpp unit tests" OFF ) +option( XROOTD_PLUGINS_BUILD_UNITTESTS "Build the XRootD plugins unit tests" OFF ) option( XROOTD_PLUGINS_EXTERNAL_GTEST "Use an external/pre-installed copy of GTest" OFF ) set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 21d5c8a..c175738 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -118,6 +118,9 @@ add_test(NAME S3::s3_basic::test add_test(NAME S3::s3_basic::stress_test COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/s3-stress-test.sh" s3_basic) +add_test(NAME S3::s3_basic::list_test + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/s3-list-test.sh" s3_basic) + list(APPEND S3_BASIC_TEST_LOGS ${CMAKE_CURRENT_BINARY_DIR}/tests/s3_basic/server.log) list(APPEND S3_BASIC_TEST_LOGS ${CMAKE_CURRENT_BINARY_DIR}/tests/s3_basic/client.log) @@ -134,3 +137,10 @@ set_tests_properties(S3::s3_basic::stress_test ENVIRONMENT "BINARY_DIR=${CMAKE_BINARY_DIR}" ATTACHED_FILES_ON_FAIL "${S3_BASIC_TEST_LOGS}" ) + +set_tests_properties(S3::s3_basic::list_test + PROPERTIES + FIXTURES_REQUIRED S3::s3_basic + ENVIRONMENT "BINARY_DIR=${CMAKE_BINARY_DIR}" + ATTACHED_FILES_ON_FAIL "${S3_BASIC_TEST_LOGS}" +) diff --git a/test/s3-list-test.sh b/test/s3-list-test.sh new file mode 100755 index 0000000..9958f5d --- /dev/null +++ b/test/s3-list-test.sh @@ -0,0 +1,74 @@ +#!/bin/sh + +############################ +# Test Setup # +############################ +TEST_NAME=$1 + +if [ -z "$BINARY_DIR" ]; then + echo "\$BINARY_DIR environment variable is not set; cannot run test" + exit 1 +fi +if [ ! -d "$BINARY_DIR" ]; then + echo "$BINARY_DIR is not a directory; cannot run test" + exit 1 +fi + +echo "Running $TEST_NAME - PROPFIND test" + +if [ ! -f "$BINARY_DIR/tests/$TEST_NAME/setup.sh" ]; then + echo "Test environment file $BINARY_DIR/tests/$TEST_NAME/setup.sh does not exist - cannot run test" + exit 1 +fi +. "$BINARY_DIR/tests/$TEST_NAME/setup.sh" + +if [ -z "$XROOTD_URL" ]; then + echo "XRootD URL is not set; cannot test" + exit 1 +fi + +############################ +# Start the tests # +############################ + +# PROPFIND against public bucket with `hello` prefix +RESPONSE_PUBLIC=$(curl --cacert "$X509_CA_FILE" -v --fail -X PROPFIND "$XROOTD_URL/test-public/hello" \ + 2>"$BINARY_DIR/tests/$TEST_NAME/propfind-client-public.log") +CURL_EXIT_PUBLIC=$? + +if [ $CURL_EXIT_PUBLIC -ne 0 ]; then + echo "PROPFIND request against public bucket failed: CURL exit code $CURL_EXIT_PUBLIC" + exit 1 +fi + +# TODO: Adjust this XML when when lists are fixed -- this will fail until then +# Validate the public bucket response contains the expected XML +EXPECTED_XML_PUBLIC='/test/hello_world.txt' +if [ "$RESPONSE_PUBLIC" != "$EXPECTED_XML_PUBLIC" ]; then + echo "PROPFIND response for public bucket does not match expected output" + echo "Actual response: $RESPONSE_PUBLIC" + exit 1 +fi + +echo "Public PROPFIND test passed" + +# PROPFIND against authed bucket with `hello` prefix +RESPONSE_AUTHED=$(curl --cacert "$X509_CA_FILE" -v --fail -X PROPFIND "$XROOTD_URL/test-authed/hello" \ + 2>"$BINARY_DIR/tests/$TEST_NAME/propfind-client-authed.log") +CURL_EXIT_AUTHED=$? + +if [ $CURL_EXIT_AUTHED -ne 0 ]; then + echo "PROPFIND request against authed bucket failed" + exit 1 +fi + +# TODO: Adjust this XML when when lists are fixed -- this will fail until then +# Validate the authed bucket response contains the expected XML +EXPECTED_XML_AUTHED='/test/hello_world.txt' +if [ "$RESPONSE_AUTHED" != "$EXPECTED_XML_AUTHED" ]; then + echo "PROPFIND response for authed bucket does not match expected output" + echo "Actual response: $RESPONSE_AUTHED" + exit 1 +fi + +echo "Authed PROPFIND test passed" diff --git a/test/s3-setup.sh b/test/s3-setup.sh index 1665ad2..879625a 100755 --- a/test/s3-setup.sh +++ b/test/s3-setup.sh @@ -188,14 +188,18 @@ echo "Starting configuration of minio" "$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" admin user add adminminio "$MINIO_USER" "$MINIO_PASSWORD" "$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" alias set userminio "$MINIO_URL" "$MINIO_USER" "$MINIO_PASSWORD" "$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" admin policy attach adminminio readwrite --user "$MINIO_USER" -"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" mb userminio/test-bucket +"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" mb userminio/test-bucket-authed +"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" mb userminio/test-bucket-public +"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" anonymous set public userminio/test-bucket-public + if [ $? -ne 0 ]; then echo "Failed to create test bucket in minio server" exit 1 fi echo "Hello, World" > "$RUNDIR/hello_world.txt" -"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" cp "$RUNDIR/hello_world.txt" userminio/test-bucket/hello_world.txt +"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" cp "$RUNDIR/hello_world.txt" userminio/test-bucket-authed/hello_world.txt +"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" cp "$RUNDIR/hello_world.txt" userminio/test-bucket-public/hello_world.txt #### # Starting XRootD config with S3 backend @@ -234,9 +238,10 @@ ofs.osslib $BINARY_DIR/libXrdS3.so s3.trace debug +# Setup the auth'ed bucket s3.begin -s3.path_name /test -s3.bucket_name test-bucket +s3.path_name /test-authed +s3.bucket_name test-bucket-authed s3.service_url $MINIO_URL s3.service_name $(hostname) s3.url_style path @@ -245,6 +250,16 @@ s3.access_key_file $XROOTD_CONFIGDIR/access_key s3.secret_key_file $XROOTD_CONFIGDIR/secret_key s3.end +# And the public/anonymous bucket +s3.begin +s3.path_name /test-public +s3.bucket_name test-bucket-public +s3.service_url $MINIO_URL +s3.service_name $(hostname) +s3.url_style path +s3.region us-east-1 +s3.end + EOF cat > $XROOTD_CONFIGDIR/authdb < "$BINARY_DIR/tests/$TEST_NAME/client-$IDX.log" > "$BINARY_DIR/tests/$TEST_NAME/client-$IDX.out" & + curl --cacert $X509_CA_FILE -v --fail "$XROOTD_URL/test-authed/hello_world.txt" 2> "$BINARY_DIR/tests/$TEST_NAME/client-$IDX.log" > "$BINARY_DIR/tests/$TEST_NAME/client-$IDX.out" & export CURL_${IDX}_PID=$! done diff --git a/test/s3-test.sh b/test/s3-test.sh index 736ca72..752f9bb 100755 --- a/test/s3-test.sh +++ b/test/s3-test.sh @@ -11,7 +11,7 @@ if [ ! -d "$BINARY_DIR" ]; then exit 1 fi -echo "Running $TEST_NAME - simple download" +echo "Running $TEST_NAME - simple download, authed bucket" if [ ! -f "$BINARY_DIR/tests/$TEST_NAME/setup.sh" ]; then echo "Test environment file $BINARY_DIR/tests/$TEST_NAME/setup.sh does not exist - cannot run test" @@ -24,22 +24,39 @@ if [ -z "$XROOTD_URL" ]; then exit 1 fi -CONTENTS=$(curl --cacert $X509_CA_FILE -v --fail "$XROOTD_URL/test/hello_world.txt" 2> "$BINARY_DIR/tests/$TEST_NAME/client.log") +# Hit the authed bucket +CONTENTS=$(curl --cacert $X509_CA_FILE -v --fail "$XROOTD_URL/test-authed/hello_world.txt" 2> "$BINARY_DIR/tests/$TEST_NAME/client.log") CURL_EXIT=$? if [ $CURL_EXIT -ne 0 ]; then - echo "Download of hello-world text failed" + echo "Download of hello-world text from authed bucket failed" exit 1 fi if [ "$CONTENTS" != "Hello, World" ]; then - echo "Downloaded hello-world text is incorrect: $CONTENTS" + echo "Downloaded hello-world text from authed bucket is incorrect: $CONTENTS" + exit 1 +fi + +# Hit the public/anonymous bucket +echo "Running $TEST_NAME - simple download, public bucket" +# We still pass the CA file to curl to avoid having to pass -k for insecure https connections +CONTENTS=$(curl --cacert $X509_CA_FILE -v --fail "$XROOTD_URL/test-public/hello_world.txt" 2>> "$BINARY_DIR/tests/$TEST_NAME/client.log") +CURL_EXIT=$? + +if [ $CURL_EXIT -ne 0 ]; then + echo "Download of hello-world text from public bucket failed" + exit 1 +fi + +if [ "$CONTENTS" != "Hello, World" ]; then + echo "Downloaded hello-world text from public bucket is incorrect: $CONTENTS" exit 1 fi echo "Running $TEST_NAME - missing object" -HTTP_CODE=$(curl --cacert $X509_CA_FILE --output /dev/null -v --write-out '%{http_code}' "$XROOTD_URL/test/missing.txt" 2>> "$BINARY_DIR/tests/$TEST_NAME/client.log") +HTTP_CODE=$(curl --cacert $X509_CA_FILE --output /dev/null -v --write-out '%{http_code}' "$XROOTD_URL/test-authed/missing.txt" 2>> "$BINARY_DIR/tests/$TEST_NAME/client.log") if [ "$HTTP_CODE" -ne 404 ]; then echo "Expected HTTP code is 404; actual was $HTTP_CODE" exit 1