From de970fec30a2016d71a204259fcdc390fdb4ab17 Mon Sep 17 00:00:00 2001
From: wlandau-lilly <will.landau@lilly.com>
Date: Fri, 10 Nov 2023 11:43:31 -0500
Subject: [PATCH] Switch to ETags for AWS S3 targets

---
 R/class_aws.R                 |  4 ++--
 R/utils_aws.R                 |  2 +-
 tests/aws/test-aws_endpoint.R |  8 ++++----
 tests/aws/test-aws_hpc.R      |  2 +-
 tests/aws/test-aws_long.R     |  2 +-
 tests/aws/test-aws_meta.R     | 14 +++++++-------
 tests/gcp/test-gcp_hpc.R      |  2 +-
 tests/gcp/test-gcp_long.R     |  2 +-
 tests/gcp/test-gcp_meta.R     | 14 +++++++-------
 9 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/R/class_aws.R b/R/class_aws.R
index aae074599..b8ab7a1e3 100644
--- a/R/class_aws.R
+++ b/R/class_aws.R
@@ -224,7 +224,6 @@ store_upload_object_aws <- function(store) {
       bucket = bucket,
       region = store_aws_region(store$file$path),
       endpoint = store_aws_endpoint(store$file$path),
-      metadata = list("targets-hash" = store$file$hash),
       part_size = aws$part_size,
       args = aws$args,
       max_tries = aws$max_tries,
@@ -247,6 +246,7 @@ store_upload_object_aws <- function(store) {
     invert = TRUE
   )
   store$file$path <- c(path, paste0("version=", head$VersionId))
+  store$file$hash <- digest_chr64(head$ETag)
   invisible()
 }
 
@@ -271,7 +271,7 @@ store_aws_hash <- function(store) {
     close_connection = aws$close_connection,
     s3_force_path_style = aws$s3_force_path_style
   )
-  head$Metadata[["targets-hash"]]
+  digest_chr64(head$ETag)
 }
 # nocov end
 
diff --git a/R/utils_aws.R b/R/utils_aws.R
index 3e4ab47cc..65fd4ee14 100644
--- a/R/utils_aws.R
+++ b/R/utils_aws.R
@@ -102,7 +102,7 @@ aws_s3_list_etags <- function(
   out <- list()
   for (page in pages) {
     for (object in page$Contents) {
-      out[[object$Key]] <- eval(parse(text = object$ETag))
+      out[[object$Key]] <- object$ETag
     }
   }
   out
diff --git a/tests/aws/test-aws_endpoint.R b/tests/aws/test-aws_endpoint.R
index 9dd30b46a..9c9f522f4 100644
--- a/tests/aws/test-aws_endpoint.R
+++ b/tests/aws/test-aws_endpoint.R
@@ -39,7 +39,7 @@ tar_test("aws qs format data gets stored", {
   })
   expr <- tar_tidy_eval(expr, environment(), TRUE)
   eval(as.call(list(`tar_script`, expr, ask = FALSE)))
-  tar_make(callr_function = NULL)
+  tar_make(callr_function = NULL, reporter = "silent")
   expect_true(
     aws_s3_exists(
       bucket = bucket_name,
@@ -97,10 +97,10 @@ tar_test("aws qs format invalidation", {
   })
   expr <- tar_tidy_eval(expr, environment(), TRUE)
   eval(as.call(list(`tar_script`, expr, ask = FALSE)))
-  tar_make(callr_function = NULL)
+  tar_make(callr_function = NULL, reporter = "silent")
   expect_equal(tar_progress(x)$progress, "built")
   expect_equal(tar_progress(y)$progress, "built")
-  tar_make(callr_function = NULL)
+  tar_make(callr_function = NULL, reporter = "silent")
   progress <- tar_progress()
   progress <- progress[progress$progress != "skipped", ]
   expect_equal(nrow(progress), 0L)
@@ -122,7 +122,7 @@ tar_test("aws qs format invalidation", {
   })
   expr <- tar_tidy_eval(expr, environment(), TRUE)
   eval(as.call(list(`tar_script`, expr, ask = FALSE)))
-  tar_make(callr_function = NULL)
+  tar_make(callr_function = NULL, reporter = "silent")
   expect_equal(tar_progress(x)$progress, "built")
   expect_equal(tar_progress(y)$progress, "built")
   expect_equal(tar_read(x), "x_value2")
diff --git a/tests/aws/test-aws_hpc.R b/tests/aws/test-aws_hpc.R
index b4062d19a..3235cbb4b 100644
--- a/tests/aws/test-aws_hpc.R
+++ b/tests/aws/test-aws_hpc.R
@@ -29,6 +29,6 @@ tar_test("AWS S3 + HPC", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make_future()
+  tar_make_future(reporter = "silent")
   expect_equal(tar_read(c), 2L)
 })
diff --git a/tests/aws/test-aws_long.R b/tests/aws/test-aws_long.R
index 781933d1c..690ab871e 100644
--- a/tests/aws/test-aws_long.R
+++ b/tests/aws/test-aws_long.R
@@ -39,7 +39,7 @@ tar_test("pipeline continuously uploads metadata", {
   expr <- tar_tidy_eval(expr, environment(), TRUE)
   eval(as.call(list(`tar_script`, expr, ask = FALSE)))
   R.utils::withTimeout(
-    expr = tar_make(seconds_meta_upload = 1),
+    expr = tar_make(seconds_meta_upload = 1, reporter = "silent"),
     timeout = 30,
     onTimeout = "silent"
   )
diff --git a/tests/aws/test-aws_meta.R b/tests/aws/test-aws_meta.R
index 830cca86e..17e30e067 100644
--- a/tests/aws/test-aws_meta.R
+++ b/tests/aws/test-aws_meta.R
@@ -29,7 +29,7 @@ tar_test("AWS meta", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -52,7 +52,7 @@ tar_test("AWS meta", {
   }
   unlink(path_meta(path_store_default()))
   expect_equal(sort(tar_outdated()), sort(c("a", "b", "c")))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "skipped"))
   tar_destroy()
   for (file in c("meta", "process", "progress", "crew")) {
@@ -106,7 +106,7 @@ tar_test("AWS tar_meta_delete()", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -179,7 +179,7 @@ tar_test("AWS tar_meta_upload()", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -244,7 +244,7 @@ tar_test("AWS tar_meta_download()", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -297,7 +297,7 @@ tar_test("AWS tar_meta_sync() upload", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -362,7 +362,7 @@ tar_test("AWS tar_meta_sync() download", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
diff --git a/tests/gcp/test-gcp_hpc.R b/tests/gcp/test-gcp_hpc.R
index 50fe7fc23..3fa63950d 100644
--- a/tests/gcp/test-gcp_hpc.R
+++ b/tests/gcp/test-gcp_hpc.R
@@ -31,6 +31,6 @@ tar_test("gcp + HPC", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make_future()
+  tar_make_future(reporter = "silent")
   expect_equal(tar_read(c), 2L)
 })
diff --git a/tests/gcp/test-gcp_long.R b/tests/gcp/test-gcp_long.R
index 4dd4b76d3..0eb36a5f4 100644
--- a/tests/gcp/test-gcp_long.R
+++ b/tests/gcp/test-gcp_long.R
@@ -41,7 +41,7 @@ tar_test("pipeline continuously uploads metadata", {
   expr <- tar_tidy_eval(expr, environment(), TRUE)
   eval(as.call(list(`tar_script`, expr, ask = FALSE)))
   R.utils::withTimeout(
-    expr = tar_make(seconds_meta_upload = 1),
+    expr = tar_make(seconds_meta_upload = 1, reporter = "silent"),
     timeout = 30,
     onTimeout = "silent"
   )
diff --git a/tests/gcp/test-gcp_meta.R b/tests/gcp/test-gcp_meta.R
index e64d8b804..a69e25dcd 100644
--- a/tests/gcp/test-gcp_meta.R
+++ b/tests/gcp/test-gcp_meta.R
@@ -30,7 +30,7 @@ tar_test("gcp meta", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -53,7 +53,7 @@ tar_test("gcp meta", {
   }
   unlink(path_meta(path_store_default()))
   expect_equal(sort(tar_outdated()), sort(c("a", "b", "c")))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "skipped"))
   tar_destroy()
   for (file in c("meta", "process", "progress", "crew")) {
@@ -108,7 +108,7 @@ tar_test("gcp tar_meta_delete()", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -182,7 +182,7 @@ tar_test("gcp tar_meta_upload()", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -248,7 +248,7 @@ tar_test("gcp tar_meta_download()", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -302,7 +302,7 @@ tar_test("gcp tar_meta_sync() upload", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {
@@ -368,7 +368,7 @@ tar_test("gcp tar_meta_sync() download", {
     )
   }, env = list(bucket_name = bucket_name))
   do.call(tar_script, list(code = code))
-  tar_make()
+  tar_make(reporter = "silent")
   expect_true(all(tar_progress()$progress == "built"))
   expect_equal(tar_read(c), 2L)
   for (file in c("meta", "process", "progress", "crew")) {